diff --git a/amplify/backend.ts b/amplify/backend.ts index 297062102..86e3136a6 100644 --- a/amplify/backend.ts +++ b/amplify/backend.ts @@ -1,25 +1,13 @@ -import { defineBackend } from '@aws-amplify/backend'; -import { goFunctions } from './functions/go-functions'; -import { rustFunctions } from './functions/rust-functions'; -import { pythonFunctions } from './functions/python-functions'; -import { defineData } from '@aws-amplify/backend'; -import { schema } from './data/resource'; +import { defineBackend } from '@aws-amplify/backend' +import { goFunction } from './functions/go/resource.ts' +import { pythonFunction } from './functions/python/resource.ts' +import { rustFunction } from './functions/rust/resource.ts' + +defineBackend({ + goFunction, + pythonFunction, + rustFunction +}) -const data = defineData({ - schema, - authorizationModes: { - defaultAuthorizationMode: 'userPool', - }, -}); - -const backend = defineBackend({ - data, - ...goFunctions, - ...rustFunctions, - ...pythonFunctions, -}); - -// Export backend for use in frontend -export default backend; diff --git a/amplify/data/resource.ts b/amplify/data/resource.ts deleted file mode 100644 index 296d8d9f1..000000000 --- a/amplify/data/resource.ts +++ /dev/null @@ -1,203 +0,0 @@ -import { type ClientSchema, a, defineData } from '@aws-amplify/backend'; - -const schema = a.schema({ - // Data Models - User: a - .model({ - id: a.id(), - email: a.string().required(), - name: a.string().required(), - profileImage: a.string(), - preferences: a.json(), - createdAt: a.datetime(), - updatedAt: a.datetime(), - // Relations - posts: a.hasMany('Post', 'userId'), - analytics: a.hasMany('UserAnalytics', 'userId'), - }) - .authorization((allow) => [ - allow.owner(), - allow.authenticated().to(['read']), - ]), - - Post: a - .model({ - id: a.id(), - title: a.string().required(), - content: a.string().required(), - imageUrl: a.string(), - processedImageUrl: a.string(), - status: a.enum(['draft', 'published', 'archived']), - tags: a.string().array(), - metadata: a.json(), - userId: a.id().required(), - createdAt: a.datetime(), - updatedAt: a.datetime(), - // Relations - user: a.belongsTo('User', 'userId'), - analytics: a.hasMany('PostAnalytics', 'postId'), - }) - .authorization((allow) => [ - allow.owner(), - allow.authenticated().to(['read']), - allow.guest().to(['read']), - ]), - - UserAnalytics: a - .model({ - id: a.id(), - userId: a.id().required(), - sessionData: a.json(), - behaviorMetrics: a.json(), - timestamp: a.datetime(), - // Relations - user: a.belongsTo('User', 'userId'), - }) - .authorization((allow) => [ - allow.owner(), - ]), - - PostAnalytics: a - .model({ - id: a.id(), - postId: a.id().required(), - views: a.integer().default(0), - likes: a.integer().default(0), - shares: a.integer().default(0), - engagementData: a.json(), - timestamp: a.datetime(), - // Relations - post: a.belongsTo('Post', 'postId'), - }) - .authorization((allow) => [ - allow.authenticated().to(['read', 'update']), - allow.owner().to(['create', 'read', 'update', 'delete']), - ]), - - ProcessingJob: a - .model({ - id: a.id(), - type: a.enum(['image_processing', 'data_analysis', 'ml_inference', 'crypto_operation']), - status: a.enum(['pending', 'processing', 'completed', 'failed']), - inputData: a.json(), - outputData: a.json(), - errorMessage: a.string(), - processingTime: a.integer(), - userId: a.id().required(), - createdAt: a.datetime(), - updatedAt: a.datetime(), - }) - .authorization((allow) => [ - allow.owner(), - ]), - - // Custom Types - ApiResponse: a.customType({ - success: a.boolean().required(), - message: a.string(), - data: a.json(), - timestamp: a.datetime(), - }), - - ProcessingResult: a.customType({ - jobId: a.string().required(), - status: a.string().required(), - result: a.json(), - processingTime: a.integer(), - error: a.string(), - }), - - AnalyticsData: a.customType({ - metrics: a.json().required(), - insights: a.json(), - recommendations: a.string().array(), - generatedAt: a.datetime(), - }), - - // Custom Queries - Go Functions - processData: a - .query() - .arguments({ - input: a.json().required(), - processingType: a.string().required(), - }) - .returns(a.ref('ProcessingResult')) - .authorization((allow) => [allow.authenticated()]) - .handler(a.handler.function('goDataProcessor')), - - getApiHealth: a - .query() - .arguments({}) - .returns(a.ref('ApiResponse')) - .authorization((allow) => [allow.authenticated(), allow.guest()]) - .handler(a.handler.function('goApiHandler')), - - // Custom Mutations - Rust Functions - processImageRust: a - .mutation() - .arguments({ - imageUrl: a.string().required(), - operations: a.string().array().required(), - quality: a.integer().default(80), - }) - .returns(a.ref('ProcessingResult')) - .authorization((allow) => [allow.authenticated()]) - .handler(a.handler.function('rustImageProcessor')), - - // Custom Queries - Python Functions - processImagePython: a - .mutation() - .arguments({ - imageUrl: a.string().required(), - operations: a.string().array().required(), - quality: a.integer().default(80), - }) - .returns(a.ref('ProcessingResult')) - .authorization((allow) => [allow.authenticated()]) - .handler(a.handler.function('pythonImageProcessing')), - - analyzeUserData: a - .query() - .arguments({ - userId: a.string(), - dateRange: a.json(), - metrics: a.string().array(), - }) - .returns(a.ref('AnalyticsData')) - .authorization((allow) => [allow.authenticated()]) - .handler(a.handler.function('pythonDataAnalytics')), - - // Subscriptions for real-time updates - onProcessingJobUpdate: a - .subscription() - .for(a.ref('ProcessingJob')) - .authorization((allow) => [allow.owner()]), - - onPostAnalyticsUpdate: a - .subscription() - .for(a.ref('PostAnalytics')) - .authorization((allow) => [allow.authenticated()]), - - // Custom Subscriptions - onJobStatusChange: a - .subscription() - .arguments({ - jobId: a.string().required(), - }) - .returns(a.ref('ProcessingResult')) - .authorization((allow) => [allow.authenticated()]), -}); - -export type Schema = ClientSchema; - -export const data = defineData({ - schema, - authorizationModes: { - defaultAuthorizationMode: 'userPool', - // API Key mode for guest access - apiKeyAuthorizationMode: { - expiresInDays: 30, - }, - }, -}); - diff --git a/amplify/functions/go-functions.ts b/amplify/functions/go-functions.ts deleted file mode 100644 index 84fded1d5..000000000 --- a/amplify/functions/go-functions.ts +++ /dev/null @@ -1,24 +0,0 @@ -import { defineFunction } from '@aws-amplify/backend'; - -export const goApiHandler = defineFunction({ - name: 'go-api-handler', - entry: './functions/go/api-handler', - runtime: 'provided.al2023', - architecture: 'arm64', - timeout: 30, - memoryMB: 512, -}); - -export const goDataProcessor = defineFunction({ - name: 'go-data-processor', - entry: './functions/go/data-processor', - runtime: 'provided.al2023', - architecture: 'arm64', - timeout: 60, - memoryMB: 1024, -}); - -export const goFunctions = { - goApiHandler, - goDataProcessor, -}; diff --git a/amplify/functions/go/resource.ts b/amplify/functions/go/resource.ts new file mode 100644 index 000000000..6cab69fd9 --- /dev/null +++ b/amplify/functions/go/resource.ts @@ -0,0 +1,9 @@ +import { defineFunction } from '@aws-amplify/backend' + +export const goFunction = defineFunction({ + name: 'go-functions', + entry: '../../../functions/go-functions/main.go', // ← Points to your Go file + runtime: 'provided.al2023' // ← Custom runtime for Go +}) + + diff --git a/amplify/functions/python-functions.ts b/amplify/functions/python-functions.ts deleted file mode 100644 index 48212947f..000000000 --- a/amplify/functions/python-functions.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { defineFunction } from '@aws-amplify/backend'; - -export const pythonImageProcessing = defineFunction({ - name: 'python-image-processing', - entry: './functions/python/image-processing', - runtime: 'python3.11', - timeout: 300, - memoryMB: 3008, -}); - -export const pythonDataAnalytics = defineFunction({ - name: 'python-data-analytics', - entry: './functions/python/data-analytics', - runtime: 'python3.11', - timeout: 900, - memoryMB: 2048, -}); - -export const pythonFunctions = { - pythonImageProcessing, - pythonDataAnalytics, -}; diff --git a/amplify/functions/python/resource.ts b/amplify/functions/python/resource.ts new file mode 100644 index 000000000..e57a6065e --- /dev/null +++ b/amplify/functions/python/resource.ts @@ -0,0 +1,8 @@ +import { defineFunction } from '@aws-amplify/backend' + +export const pythonFunction = defineFunction({ + name: 'python-function', + entry: '../../../functions/python-function/handler.py', // ← Points to external folder + runtime: 'python3.12' +}) + diff --git a/amplify/functions/rust-functions.ts b/amplify/functions/rust-functions.ts deleted file mode 100644 index 639fc8e8e..000000000 --- a/amplify/functions/rust-functions.ts +++ /dev/null @@ -1,14 +0,0 @@ -import { defineFunction } from '@aws-amplify/backend'; - -export const rustImageProcessor = defineFunction({ - name: 'rust-image-processor', - entry: './functions/rust/image-processor', - runtime: 'provided.al2023', - architecture: 'arm64', - timeout: 300, - memoryMB: 2048, -}); - -export const rustFunctions = { - rustImageProcessor -}; diff --git a/amplify/functions/rust/resource.ts b/amplify/functions/rust/resource.ts new file mode 100644 index 000000000..785ce2421 --- /dev/null +++ b/amplify/functions/rust/resource.ts @@ -0,0 +1,8 @@ +import { defineFunction } from '@aws-amplify/backend' + +export const rustFunction = defineFunction({ + name: 'rust-function', + entry: '../../../functions/rust-function/src/main.rs', // ← Points to external folder + runtime: 'provided.al2' +}) + diff --git a/dist/go/api-handler.zip b/dist/go/api-handler.zip deleted file mode 100644 index cec6629a0..000000000 Binary files a/dist/go/api-handler.zip and /dev/null differ diff --git a/dist/python/data-processor.zip b/dist/python/data-processor.zip deleted file mode 100644 index 6ec35fa56..000000000 Binary files a/dist/python/data-processor.zip and /dev/null differ diff --git a/dist/rust/performance-handler.zip b/dist/rust/performance-handler.zip deleted file mode 100644 index 1ccbfc89c..000000000 Binary files a/dist/rust/performance-handler.zip and /dev/null differ diff --git a/functions/go/api-handler/main.go b/functions/go/api-handler/main.go deleted file mode 100644 index 2fba7507a..000000000 --- a/functions/go/api-handler/main.go +++ /dev/null @@ -1,45 +0,0 @@ -package main - -import ( - "context" - "encoding/json" - "log" - - "github.com/aws/aws-lambda-go/events" - "github.com/aws/aws-lambda-go/lambda" -) - -type Response struct { - StatusCode int `json:"statusCode"` - Headers map[string]string `json:"headers"` - Body string `json:"body"` -} - -func handler(ctx context.Context, request events.APIGatewayProxyRequest) (Response, error) { - log.Printf("Received request: %+v", request) - - headers := map[string]string{ - "Access-Control-Allow-Origin": "*", - "Access-Control-Allow-Headers": "Content-Type", - "Access-Control-Allow-Methods": "GET, POST, PUT, DELETE, OPTIONS", - "Content-Type": "application/json", - } - - responseBody := map[string]interface{}{ - "message": "Hello from reorganized Go Lambda!", - "path": request.Path, - "method": request.HTTPMethod, - } - - body, _ := json.Marshal(responseBody) - - return Response{ - StatusCode: 200, - Headers: headers, - Body: string(body), - }, nil -} - -func main() { - lambda.Start(handler) -} diff --git a/functions/go/api-handler/go.mod b/functions/go/go.mod similarity index 72% rename from functions/go/api-handler/go.mod rename to functions/go/go.mod index 3ecec50f4..65998c6bd 100644 --- a/functions/go/api-handler/go.mod +++ b/functions/go/go.mod @@ -1,5 +1,6 @@ -module api-handler +module go-functions go 1.21 require github.com/aws/aws-lambda-go v1.41.0 + diff --git a/functions/go/api-handler/go.sum b/functions/go/go.sum similarity index 100% rename from functions/go/api-handler/go.sum rename to functions/go/go.sum diff --git a/functions/go/main.go b/functions/go/main.go new file mode 100644 index 000000000..f04dd87c3 --- /dev/null +++ b/functions/go/main.go @@ -0,0 +1,26 @@ +package main + +import ( + "context" + "fmt" + + "github.com/aws/aws-lambda-go/lambda" +) + +type MyEvent struct { + Name string `json:"name"` +} + +type MyResponse struct { + Message string `json:"message"` +} + +func HandleRequest(ctx context.Context, event MyEvent) (MyResponse, error) { + return MyResponse{ + Message: fmt.Sprintf("Hello %s from Go Lambda!", event.Name), + }, nil +} + +func main() { + lambda.Start(HandleRequest) +} diff --git a/functions/go/shared/crud/health.go b/functions/go/shared/crud/health.go deleted file mode 100644 index fe0119ed3..000000000 --- a/functions/go/shared/crud/health.go +++ /dev/null @@ -1,11 +0,0 @@ -package crud - -import "github.com/gjcampbell777/personal_site/backend-go/internal/database" - -func Health(db *database.Client) map[string]string { - return map[string]string{ - "status": "ok", - "service": "portfolio-api", - "database": db.Status(), - } -} diff --git a/functions/go/shared/database/client.go b/functions/go/shared/database/client.go deleted file mode 100644 index e962c5cef..000000000 --- a/functions/go/shared/database/client.go +++ /dev/null @@ -1,11 +0,0 @@ -package database - -type Client struct{} - -func NewClient() *Client { - return &Client{} -} - -func (c *Client) Status() string { - return "ready" -} diff --git a/functions/python/data-processor/__pycache__/typing_extensions.cpython-314.pyc b/functions/python/data-processor/__pycache__/typing_extensions.cpython-314.pyc deleted file mode 100644 index 645d5c947..000000000 Binary files a/functions/python/data-processor/__pycache__/typing_extensions.cpython-314.pyc and /dev/null differ diff --git a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/INSTALLER b/functions/python/data-processor/annotated_doc-0.0.4.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/METADATA b/functions/python/data-processor/annotated_doc-0.0.4.dist-info/METADATA deleted file mode 100644 index 9bf7a9e80..000000000 --- a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/METADATA +++ /dev/null @@ -1,145 +0,0 @@ -Metadata-Version: 2.4 -Name: annotated-doc -Version: 0.0.4 -Summary: Document parameters, class attributes, return types, and variables inline, with Annotated. -Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= -License-Expression: MIT -License-File: LICENSE -Classifier: Intended Audience :: Information Technology -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Application Frameworks -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: Software Development -Classifier: Typing :: Typed -Classifier: Development Status :: 4 - Beta -Classifier: Intended Audience :: Developers -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Project-URL: Homepage, https://github.com/fastapi/annotated-doc -Project-URL: Documentation, https://github.com/fastapi/annotated-doc -Project-URL: Repository, https://github.com/fastapi/annotated-doc -Project-URL: Issues, https://github.com/fastapi/annotated-doc/issues -Project-URL: Changelog, https://github.com/fastapi/annotated-doc/release-notes.md -Requires-Python: >=3.8 -Description-Content-Type: text/markdown - -# Annotated Doc - -Document parameters, class attributes, return types, and variables inline, with `Annotated`. - - - Test - - - Coverage - - - Package version - - - Supported Python versions - - -## Installation - -```bash -pip install annotated-doc -``` - -Or with `uv`: - -```Python -uv add annotated-doc -``` - -## Usage - -Import `Doc` and pass a single literal string with the documentation for the specific parameter, class attribute, return type, or variable. - -For example, to document a parameter `name` in a function `hi` you could do: - -```Python -from typing import Annotated - -from annotated_doc import Doc - -def hi(name: Annotated[str, Doc("Who to say hi to")]) -> None: - print(f"Hi, {name}!") -``` - -You can also use it to document class attributes: - -```Python -from typing import Annotated - -from annotated_doc import Doc - -class User: - name: Annotated[str, Doc("The user's name")] - age: Annotated[int, Doc("The user's age")] -``` - -The same way, you could document return types and variables, or anything that could have a type annotation with `Annotated`. - -## Who Uses This - -`annotated-doc` was made for: - -* [FastAPI](https://fastapi.tiangolo.com/) -* [Typer](https://typer.tiangolo.com/) -* [SQLModel](https://sqlmodel.tiangolo.com/) -* [Asyncer](https://asyncer.tiangolo.com/) - -`annotated-doc` is supported by [griffe-typingdoc](https://github.com/mkdocstrings/griffe-typingdoc), which powers reference documentation like the one in the [FastAPI Reference](https://fastapi.tiangolo.com/reference/). - -## Reasons not to use `annotated-doc` - -You are already comfortable with one of the existing docstring formats, like: - -* Sphinx -* numpydoc -* Google -* Keras - -Your team is already comfortable using them. - -You prefer having the documentation about parameters all together in a docstring, separated from the code defining them. - -You care about a specific set of users, using one specific editor, and that editor already has support for the specific docstring format you use. - -## Reasons to use `annotated-doc` - -* No micro-syntax to learn for newcomers, it’s **just Python** syntax. -* **Editing** would be already fully supported by default by any editor (current or future) supporting Python syntax, including syntax errors, syntax highlighting, etc. -* **Rendering** would be relatively straightforward to implement by static tools (tools that don't need runtime execution), as the information can be extracted from the AST they normally already create. -* **Deduplication of information**: the name of a parameter would be defined in a single place, not duplicated inside of a docstring. -* **Elimination** of the possibility of having **inconsistencies** when removing a parameter or class variable and **forgetting to remove** its documentation. -* **Minimization** of the probability of adding a new parameter or class variable and **forgetting to add its documentation**. -* **Elimination** of the possibility of having **inconsistencies** between the **name** of a parameter in the **signature** and the name in the docstring when it is renamed. -* **Access** to the documentation string for each symbol at **runtime**, including existing (older) Python versions. -* A more formalized way to document other symbols, like type aliases, that could use Annotated. -* **Support** for apps using FastAPI, Typer and others. -* **AI Accessibility**: AI tools will have an easier way understanding each parameter as the distance from documentation to parameter is much closer. - -## History - -I ([@tiangolo](https://github.com/tiangolo)) originally wanted for this to be part of the Python standard library (in [PEP 727](https://peps.python.org/pep-0727/)), but the proposal was withdrawn as there was a fair amount of negative feedback and opposition. - -The conclusion was that this was better done as an external effort, in a third-party library. - -So, here it is, with a simpler approach, as a third-party library, in a way that can be used by others, starting with FastAPI and friends. - -## License - -This project is licensed under the terms of the MIT license. diff --git a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/RECORD b/functions/python/data-processor/annotated_doc-0.0.4.dist-info/RECORD deleted file mode 100644 index f6281a13f..000000000 --- a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/RECORD +++ /dev/null @@ -1,11 +0,0 @@ -annotated_doc-0.0.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -annotated_doc-0.0.4.dist-info/METADATA,sha256=Irm5KJua33dY2qKKAjJ-OhKaVBVIfwFGej_dSe3Z1TU,6566 -annotated_doc-0.0.4.dist-info/RECORD,, -annotated_doc-0.0.4.dist-info/WHEEL,sha256=9P2ygRxDrTJz3gsagc0Z96ukrxjr-LFBGOgv3AuKlCA,90 -annotated_doc-0.0.4.dist-info/entry_points.txt,sha256=6OYgBcLyFCUgeqLgnvMyOJxPCWzgy7se4rLPKtNonMs,34 -annotated_doc-0.0.4.dist-info/licenses/LICENSE,sha256=__Fwd5pqy_ZavbQFwIfxzuF4ZpHkqWpANFF-SlBKDN8,1086 -annotated_doc/__init__.py,sha256=VuyxxUe80kfEyWnOrCx_Bk8hybo3aKo6RYBlkBBYW8k,52 -annotated_doc/__pycache__/__init__.cpython-314.pyc,, -annotated_doc/__pycache__/main.cpython-314.pyc,, -annotated_doc/main.py,sha256=5Zfvxv80SwwLqpRW73AZyZyiM4bWma9QWRbp_cgD20s,1075 -annotated_doc/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/WHEEL b/functions/python/data-processor/annotated_doc-0.0.4.dist-info/WHEEL deleted file mode 100644 index 045c8acde..000000000 --- a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: pdm-backend (2.4.5) -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/entry_points.txt b/functions/python/data-processor/annotated_doc-0.0.4.dist-info/entry_points.txt deleted file mode 100644 index c3ad4726d..000000000 --- a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/entry_points.txt +++ /dev/null @@ -1,4 +0,0 @@ -[console_scripts] - -[gui_scripts] - diff --git a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/licenses/LICENSE b/functions/python/data-processor/annotated_doc-0.0.4.dist-info/licenses/LICENSE deleted file mode 100644 index 7a254464c..000000000 --- a/functions/python/data-processor/annotated_doc-0.0.4.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2025 Sebastián Ramírez - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/functions/python/data-processor/annotated_doc/__init__.py b/functions/python/data-processor/annotated_doc/__init__.py deleted file mode 100644 index a0152a7d1..000000000 --- a/functions/python/data-processor/annotated_doc/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from .main import Doc as Doc - -__version__ = "0.0.4" diff --git a/functions/python/data-processor/annotated_doc/main.py b/functions/python/data-processor/annotated_doc/main.py deleted file mode 100644 index 7063c59e4..000000000 --- a/functions/python/data-processor/annotated_doc/main.py +++ /dev/null @@ -1,36 +0,0 @@ -class Doc: - """Define the documentation of a type annotation using `Annotated`, to be - used in class attributes, function and method parameters, return values, - and variables. - - The value should be a positional-only string literal to allow static tools - like editors and documentation generators to use it. - - This complements docstrings. - - The string value passed is available in the attribute `documentation`. - - Example: - - ```Python - from typing import Annotated - from annotated_doc import Doc - - def hi(name: Annotated[str, Doc("Who to say hi to")]) -> None: - print(f"Hi, {name}!") - ``` - """ - - def __init__(self, documentation: str, /) -> None: - self.documentation = documentation - - def __repr__(self) -> str: - return f"Doc({self.documentation!r})" - - def __hash__(self) -> int: - return hash(self.documentation) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Doc): - return NotImplemented - return self.documentation == other.documentation diff --git a/functions/python/data-processor/annotated_types-0.7.0.dist-info/INSTALLER b/functions/python/data-processor/annotated_types-0.7.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/annotated_types-0.7.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/annotated_types-0.7.0.dist-info/METADATA b/functions/python/data-processor/annotated_types-0.7.0.dist-info/METADATA deleted file mode 100644 index 3ac05cfd1..000000000 --- a/functions/python/data-processor/annotated_types-0.7.0.dist-info/METADATA +++ /dev/null @@ -1,295 +0,0 @@ -Metadata-Version: 2.3 -Name: annotated-types -Version: 0.7.0 -Summary: Reusable constraint types to use with typing.Annotated -Project-URL: Homepage, https://github.com/annotated-types/annotated-types -Project-URL: Source, https://github.com/annotated-types/annotated-types -Project-URL: Changelog, https://github.com/annotated-types/annotated-types/releases -Author-email: Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, Samuel Colvin , Zac Hatfield-Dodds -License-File: LICENSE -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Console -Classifier: Environment :: MacOS X -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Information Technology -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: POSIX :: Linux -Classifier: Operating System :: Unix -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Typing :: Typed -Requires-Python: >=3.8 -Requires-Dist: typing-extensions>=4.0.0; python_version < '3.9' -Description-Content-Type: text/markdown - -# annotated-types - -[![CI](https://github.com/annotated-types/annotated-types/workflows/CI/badge.svg?event=push)](https://github.com/annotated-types/annotated-types/actions?query=event%3Apush+branch%3Amain+workflow%3ACI) -[![pypi](https://img.shields.io/pypi/v/annotated-types.svg)](https://pypi.python.org/pypi/annotated-types) -[![versions](https://img.shields.io/pypi/pyversions/annotated-types.svg)](https://github.com/annotated-types/annotated-types) -[![license](https://img.shields.io/github/license/annotated-types/annotated-types.svg)](https://github.com/annotated-types/annotated-types/blob/main/LICENSE) - -[PEP-593](https://peps.python.org/pep-0593/) added `typing.Annotated` as a way of -adding context-specific metadata to existing types, and specifies that -`Annotated[T, x]` _should_ be treated as `T` by any tool or library without special -logic for `x`. - -This package provides metadata objects which can be used to represent common -constraints such as upper and lower bounds on scalar values and collection sizes, -a `Predicate` marker for runtime checks, and -descriptions of how we intend these metadata to be interpreted. In some cases, -we also note alternative representations which do not require this package. - -## Install - -```bash -pip install annotated-types -``` - -## Examples - -```python -from typing import Annotated -from annotated_types import Gt, Len, Predicate - -class MyClass: - age: Annotated[int, Gt(18)] # Valid: 19, 20, ... - # Invalid: 17, 18, "19", 19.0, ... - factors: list[Annotated[int, Predicate(is_prime)]] # Valid: 2, 3, 5, 7, 11, ... - # Invalid: 4, 8, -2, 5.0, "prime", ... - - my_list: Annotated[list[int], Len(0, 10)] # Valid: [], [10, 20, 30, 40, 50] - # Invalid: (1, 2), ["abc"], [0] * 20 -``` - -## Documentation - -_While `annotated-types` avoids runtime checks for performance, users should not -construct invalid combinations such as `MultipleOf("non-numeric")` or `Annotated[int, Len(3)]`. -Downstream implementors may choose to raise an error, emit a warning, silently ignore -a metadata item, etc., if the metadata objects described below are used with an -incompatible type - or for any other reason!_ - -### Gt, Ge, Lt, Le - -Express inclusive and/or exclusive bounds on orderable values - which may be numbers, -dates, times, strings, sets, etc. Note that the boundary value need not be of the -same type that was annotated, so long as they can be compared: `Annotated[int, Gt(1.5)]` -is fine, for example, and implies that the value is an integer x such that `x > 1.5`. - -We suggest that implementors may also interpret `functools.partial(operator.le, 1.5)` -as being equivalent to `Gt(1.5)`, for users who wish to avoid a runtime dependency on -the `annotated-types` package. - -To be explicit, these types have the following meanings: - -* `Gt(x)` - value must be "Greater Than" `x` - equivalent to exclusive minimum -* `Ge(x)` - value must be "Greater than or Equal" to `x` - equivalent to inclusive minimum -* `Lt(x)` - value must be "Less Than" `x` - equivalent to exclusive maximum -* `Le(x)` - value must be "Less than or Equal" to `x` - equivalent to inclusive maximum - -### Interval - -`Interval(gt, ge, lt, le)` allows you to specify an upper and lower bound with a single -metadata object. `None` attributes should be ignored, and non-`None` attributes -treated as per the single bounds above. - -### MultipleOf - -`MultipleOf(multiple_of=x)` might be interpreted in two ways: - -1. Python semantics, implying `value % multiple_of == 0`, or -2. [JSONschema semantics](https://json-schema.org/draft/2020-12/json-schema-validation.html#rfc.section.6.2.1), - where `int(value / multiple_of) == value / multiple_of`. - -We encourage users to be aware of these two common interpretations and their -distinct behaviours, especially since very large or non-integer numbers make -it easy to cause silent data corruption due to floating-point imprecision. - -We encourage libraries to carefully document which interpretation they implement. - -### MinLen, MaxLen, Len - -`Len()` implies that `min_length <= len(value) <= max_length` - lower and upper bounds are inclusive. - -As well as `Len()` which can optionally include upper and lower bounds, we also -provide `MinLen(x)` and `MaxLen(y)` which are equivalent to `Len(min_length=x)` -and `Len(max_length=y)` respectively. - -`Len`, `MinLen`, and `MaxLen` may be used with any type which supports `len(value)`. - -Examples of usage: - -* `Annotated[list, MaxLen(10)]` (or `Annotated[list, Len(max_length=10))`) - list must have a length of 10 or less -* `Annotated[str, MaxLen(10)]` - string must have a length of 10 or less -* `Annotated[list, MinLen(3))` (or `Annotated[list, Len(min_length=3))`) - list must have a length of 3 or more -* `Annotated[list, Len(4, 6)]` - list must have a length of 4, 5, or 6 -* `Annotated[list, Len(8, 8)]` - list must have a length of exactly 8 - -#### Changed in v0.4.0 - -* `min_inclusive` has been renamed to `min_length`, no change in meaning -* `max_exclusive` has been renamed to `max_length`, upper bound is now **inclusive** instead of **exclusive** -* The recommendation that slices are interpreted as `Len` has been removed due to ambiguity and different semantic - meaning of the upper bound in slices vs. `Len` - -See [issue #23](https://github.com/annotated-types/annotated-types/issues/23) for discussion. - -### Timezone - -`Timezone` can be used with a `datetime` or a `time` to express which timezones -are allowed. `Annotated[datetime, Timezone(None)]` must be a naive datetime. -`Timezone[...]` ([literal ellipsis](https://docs.python.org/3/library/constants.html#Ellipsis)) -expresses that any timezone-aware datetime is allowed. You may also pass a specific -timezone string or [`tzinfo`](https://docs.python.org/3/library/datetime.html#tzinfo-objects) -object such as `Timezone(timezone.utc)` or `Timezone("Africa/Abidjan")` to express that you only -allow a specific timezone, though we note that this is often a symptom of fragile design. - -#### Changed in v0.x.x - -* `Timezone` accepts [`tzinfo`](https://docs.python.org/3/library/datetime.html#tzinfo-objects) objects instead of - `timezone`, extending compatibility to [`zoneinfo`](https://docs.python.org/3/library/zoneinfo.html) and third party libraries. - -### Unit - -`Unit(unit: str)` expresses that the annotated numeric value is the magnitude of -a quantity with the specified unit. For example, `Annotated[float, Unit("m/s")]` -would be a float representing a velocity in meters per second. - -Please note that `annotated_types` itself makes no attempt to parse or validate -the unit string in any way. That is left entirely to downstream libraries, -such as [`pint`](https://pint.readthedocs.io) or -[`astropy.units`](https://docs.astropy.org/en/stable/units/). - -An example of how a library might use this metadata: - -```python -from annotated_types import Unit -from typing import Annotated, TypeVar, Callable, Any, get_origin, get_args - -# given a type annotated with a unit: -Meters = Annotated[float, Unit("m")] - - -# you can cast the annotation to a specific unit type with any -# callable that accepts a string and returns the desired type -T = TypeVar("T") -def cast_unit(tp: Any, unit_cls: Callable[[str], T]) -> T | None: - if get_origin(tp) is Annotated: - for arg in get_args(tp): - if isinstance(arg, Unit): - return unit_cls(arg.unit) - return None - - -# using `pint` -import pint -pint_unit = cast_unit(Meters, pint.Unit) - - -# using `astropy.units` -import astropy.units as u -astropy_unit = cast_unit(Meters, u.Unit) -``` - -### Predicate - -`Predicate(func: Callable)` expresses that `func(value)` is truthy for valid values. -Users should prefer the statically inspectable metadata above, but if you need -the full power and flexibility of arbitrary runtime predicates... here it is. - -For some common constraints, we provide generic types: - -* `IsLower = Annotated[T, Predicate(str.islower)]` -* `IsUpper = Annotated[T, Predicate(str.isupper)]` -* `IsDigit = Annotated[T, Predicate(str.isdigit)]` -* `IsFinite = Annotated[T, Predicate(math.isfinite)]` -* `IsNotFinite = Annotated[T, Predicate(Not(math.isfinite))]` -* `IsNan = Annotated[T, Predicate(math.isnan)]` -* `IsNotNan = Annotated[T, Predicate(Not(math.isnan))]` -* `IsInfinite = Annotated[T, Predicate(math.isinf)]` -* `IsNotInfinite = Annotated[T, Predicate(Not(math.isinf))]` - -so that you can write e.g. `x: IsFinite[float] = 2.0` instead of the longer -(but exactly equivalent) `x: Annotated[float, Predicate(math.isfinite)] = 2.0`. - -Some libraries might have special logic to handle known or understandable predicates, -for example by checking for `str.isdigit` and using its presence to both call custom -logic to enforce digit-only strings, and customise some generated external schema. -Users are therefore encouraged to avoid indirection like `lambda s: s.lower()`, in -favor of introspectable methods such as `str.lower` or `re.compile("pattern").search`. - -To enable basic negation of commonly used predicates like `math.isnan` without introducing introspection that makes it impossible for implementers to introspect the predicate we provide a `Not` wrapper that simply negates the predicate in an introspectable manner. Several of the predicates listed above are created in this manner. - -We do not specify what behaviour should be expected for predicates that raise -an exception. For example `Annotated[int, Predicate(str.isdigit)]` might silently -skip invalid constraints, or statically raise an error; or it might try calling it -and then propagate or discard the resulting -`TypeError: descriptor 'isdigit' for 'str' objects doesn't apply to a 'int' object` -exception. We encourage libraries to document the behaviour they choose. - -### Doc - -`doc()` can be used to add documentation information in `Annotated`, for function and method parameters, variables, class attributes, return types, and any place where `Annotated` can be used. - -It expects a value that can be statically analyzed, as the main use case is for static analysis, editors, documentation generators, and similar tools. - -It returns a `DocInfo` class with a single attribute `documentation` containing the value passed to `doc()`. - -This is the early adopter's alternative form of the [`typing-doc` proposal](https://github.com/tiangolo/fastapi/blob/typing-doc/typing_doc.md). - -### Integrating downstream types with `GroupedMetadata` - -Implementers may choose to provide a convenience wrapper that groups multiple pieces of metadata. -This can help reduce verbosity and cognitive overhead for users. -For example, an implementer like Pydantic might provide a `Field` or `Meta` type that accepts keyword arguments and transforms these into low-level metadata: - -```python -from dataclasses import dataclass -from typing import Iterator -from annotated_types import GroupedMetadata, Ge - -@dataclass -class Field(GroupedMetadata): - ge: int | None = None - description: str | None = None - - def __iter__(self) -> Iterator[object]: - # Iterating over a GroupedMetadata object should yield annotated-types - # constraint metadata objects which describe it as fully as possible, - # and may include other unknown objects too. - if self.ge is not None: - yield Ge(self.ge) - if self.description is not None: - yield Description(self.description) -``` - -Libraries consuming annotated-types constraints should check for `GroupedMetadata` and unpack it by iterating over the object and treating the results as if they had been "unpacked" in the `Annotated` type. The same logic should be applied to the [PEP 646 `Unpack` type](https://peps.python.org/pep-0646/), so that `Annotated[T, Field(...)]`, `Annotated[T, Unpack[Field(...)]]` and `Annotated[T, *Field(...)]` are all treated consistently. - -Libraries consuming annotated-types should also ignore any metadata they do not recongize that came from unpacking a `GroupedMetadata`, just like they ignore unrecognized metadata in `Annotated` itself. - -Our own `annotated_types.Interval` class is a `GroupedMetadata` which unpacks itself into `Gt`, `Lt`, etc., so this is not an abstract concern. Similarly, `annotated_types.Len` is a `GroupedMetadata` which unpacks itself into `MinLen` (optionally) and `MaxLen`. - -### Consuming metadata - -We intend to not be prescriptive as to _how_ the metadata and constraints are used, but as an example of how one might parse constraints from types annotations see our [implementation in `test_main.py`](https://github.com/annotated-types/annotated-types/blob/f59cf6d1b5255a0fe359b93896759a180bec30ae/tests/test_main.py#L94-L103). - -It is up to the implementer to determine how this metadata is used. -You could use the metadata for runtime type checking, for generating schemas or to generate example data, amongst other use cases. - -## Design & History - -This package was designed at the PyCon 2022 sprints by the maintainers of Pydantic -and Hypothesis, with the goal of making it as easy as possible for end-users to -provide more informative annotations for use by runtime libraries. - -It is deliberately minimal, and following PEP-593 allows considerable downstream -discretion in what (if anything!) they choose to support. Nonetheless, we expect -that staying simple and covering _only_ the most common use-cases will give users -and maintainers the best experience we can. If you'd like more constraints for your -types - follow our lead, by defining them and documenting them downstream! diff --git a/functions/python/data-processor/annotated_types-0.7.0.dist-info/RECORD b/functions/python/data-processor/annotated_types-0.7.0.dist-info/RECORD deleted file mode 100644 index 2aed74035..000000000 --- a/functions/python/data-processor/annotated_types-0.7.0.dist-info/RECORD +++ /dev/null @@ -1,10 +0,0 @@ -annotated_types-0.7.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -annotated_types-0.7.0.dist-info/METADATA,sha256=7ltqxksJJ0wCYFGBNIQCWTlWQGeAH0hRFdnK3CB895E,15046 -annotated_types-0.7.0.dist-info/RECORD,, -annotated_types-0.7.0.dist-info/WHEEL,sha256=zEMcRr9Kr03x1ozGwg5v9NQBKn3kndp6LSoSlVg-jhU,87 -annotated_types-0.7.0.dist-info/licenses/LICENSE,sha256=_hBJiEsaDZNCkB6I4H8ykl0ksxIdmXK2poBfuYJLCV0,1083 -annotated_types/__init__.py,sha256=RynLsRKUEGI0KimXydlD1fZEfEzWwDo0Uon3zOKhG1Q,13819 -annotated_types/__pycache__/__init__.cpython-314.pyc,, -annotated_types/__pycache__/test_cases.cpython-314.pyc,, -annotated_types/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -annotated_types/test_cases.py,sha256=zHFX6EpcMbGJ8FzBYDbO56bPwx_DYIVSKbZM-4B3_lg,6421 diff --git a/functions/python/data-processor/annotated_types-0.7.0.dist-info/WHEEL b/functions/python/data-processor/annotated_types-0.7.0.dist-info/WHEEL deleted file mode 100644 index 516596c76..000000000 --- a/functions/python/data-processor/annotated_types-0.7.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.24.2 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/annotated_types-0.7.0.dist-info/licenses/LICENSE b/functions/python/data-processor/annotated_types-0.7.0.dist-info/licenses/LICENSE deleted file mode 100644 index d99323a99..000000000 --- a/functions/python/data-processor/annotated_types-0.7.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2022 the contributors - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/functions/python/data-processor/annotated_types/__init__.py b/functions/python/data-processor/annotated_types/__init__.py deleted file mode 100644 index 74e0deeab..000000000 --- a/functions/python/data-processor/annotated_types/__init__.py +++ /dev/null @@ -1,432 +0,0 @@ -import math -import sys -import types -from dataclasses import dataclass -from datetime import tzinfo -from typing import TYPE_CHECKING, Any, Callable, Iterator, Optional, SupportsFloat, SupportsIndex, TypeVar, Union - -if sys.version_info < (3, 8): - from typing_extensions import Protocol, runtime_checkable -else: - from typing import Protocol, runtime_checkable - -if sys.version_info < (3, 9): - from typing_extensions import Annotated, Literal -else: - from typing import Annotated, Literal - -if sys.version_info < (3, 10): - EllipsisType = type(Ellipsis) - KW_ONLY = {} - SLOTS = {} -else: - from types import EllipsisType - - KW_ONLY = {"kw_only": True} - SLOTS = {"slots": True} - - -__all__ = ( - 'BaseMetadata', - 'GroupedMetadata', - 'Gt', - 'Ge', - 'Lt', - 'Le', - 'Interval', - 'MultipleOf', - 'MinLen', - 'MaxLen', - 'Len', - 'Timezone', - 'Predicate', - 'LowerCase', - 'UpperCase', - 'IsDigits', - 'IsFinite', - 'IsNotFinite', - 'IsNan', - 'IsNotNan', - 'IsInfinite', - 'IsNotInfinite', - 'doc', - 'DocInfo', - '__version__', -) - -__version__ = '0.7.0' - - -T = TypeVar('T') - - -# arguments that start with __ are considered -# positional only -# see https://peps.python.org/pep-0484/#positional-only-arguments - - -class SupportsGt(Protocol): - def __gt__(self: T, __other: T) -> bool: - ... - - -class SupportsGe(Protocol): - def __ge__(self: T, __other: T) -> bool: - ... - - -class SupportsLt(Protocol): - def __lt__(self: T, __other: T) -> bool: - ... - - -class SupportsLe(Protocol): - def __le__(self: T, __other: T) -> bool: - ... - - -class SupportsMod(Protocol): - def __mod__(self: T, __other: T) -> T: - ... - - -class SupportsDiv(Protocol): - def __div__(self: T, __other: T) -> T: - ... - - -class BaseMetadata: - """Base class for all metadata. - - This exists mainly so that implementers - can do `isinstance(..., BaseMetadata)` while traversing field annotations. - """ - - __slots__ = () - - -@dataclass(frozen=True, **SLOTS) -class Gt(BaseMetadata): - """Gt(gt=x) implies that the value must be greater than x. - - It can be used with any type that supports the ``>`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - gt: SupportsGt - - -@dataclass(frozen=True, **SLOTS) -class Ge(BaseMetadata): - """Ge(ge=x) implies that the value must be greater than or equal to x. - - It can be used with any type that supports the ``>=`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - ge: SupportsGe - - -@dataclass(frozen=True, **SLOTS) -class Lt(BaseMetadata): - """Lt(lt=x) implies that the value must be less than x. - - It can be used with any type that supports the ``<`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - lt: SupportsLt - - -@dataclass(frozen=True, **SLOTS) -class Le(BaseMetadata): - """Le(le=x) implies that the value must be less than or equal to x. - - It can be used with any type that supports the ``<=`` operator, - including numbers, dates and times, strings, sets, and so on. - """ - - le: SupportsLe - - -@runtime_checkable -class GroupedMetadata(Protocol): - """A grouping of multiple objects, like typing.Unpack. - - `GroupedMetadata` on its own is not metadata and has no meaning. - All of the constraints and metadata should be fully expressable - in terms of the `BaseMetadata`'s returned by `GroupedMetadata.__iter__()`. - - Concrete implementations should override `GroupedMetadata.__iter__()` - to add their own metadata. - For example: - - >>> @dataclass - >>> class Field(GroupedMetadata): - >>> gt: float | None = None - >>> description: str | None = None - ... - >>> def __iter__(self) -> Iterable[object]: - >>> if self.gt is not None: - >>> yield Gt(self.gt) - >>> if self.description is not None: - >>> yield Description(self.gt) - - Also see the implementation of `Interval` below for an example. - - Parsers should recognize this and unpack it so that it can be used - both with and without unpacking: - - - `Annotated[int, Field(...)]` (parser must unpack Field) - - `Annotated[int, *Field(...)]` (PEP-646) - """ # noqa: trailing-whitespace - - @property - def __is_annotated_types_grouped_metadata__(self) -> Literal[True]: - return True - - def __iter__(self) -> Iterator[object]: - ... - - if not TYPE_CHECKING: - __slots__ = () # allow subclasses to use slots - - def __init_subclass__(cls, *args: Any, **kwargs: Any) -> None: - # Basic ABC like functionality without the complexity of an ABC - super().__init_subclass__(*args, **kwargs) - if cls.__iter__ is GroupedMetadata.__iter__: - raise TypeError("Can't subclass GroupedMetadata without implementing __iter__") - - def __iter__(self) -> Iterator[object]: # noqa: F811 - raise NotImplementedError # more helpful than "None has no attribute..." type errors - - -@dataclass(frozen=True, **KW_ONLY, **SLOTS) -class Interval(GroupedMetadata): - """Interval can express inclusive or exclusive bounds with a single object. - - It accepts keyword arguments ``gt``, ``ge``, ``lt``, and/or ``le``, which - are interpreted the same way as the single-bound constraints. - """ - - gt: Union[SupportsGt, None] = None - ge: Union[SupportsGe, None] = None - lt: Union[SupportsLt, None] = None - le: Union[SupportsLe, None] = None - - def __iter__(self) -> Iterator[BaseMetadata]: - """Unpack an Interval into zero or more single-bounds.""" - if self.gt is not None: - yield Gt(self.gt) - if self.ge is not None: - yield Ge(self.ge) - if self.lt is not None: - yield Lt(self.lt) - if self.le is not None: - yield Le(self.le) - - -@dataclass(frozen=True, **SLOTS) -class MultipleOf(BaseMetadata): - """MultipleOf(multiple_of=x) might be interpreted in two ways: - - 1. Python semantics, implying ``value % multiple_of == 0``, or - 2. JSONschema semantics, where ``int(value / multiple_of) == value / multiple_of`` - - We encourage users to be aware of these two common interpretations, - and libraries to carefully document which they implement. - """ - - multiple_of: Union[SupportsDiv, SupportsMod] - - -@dataclass(frozen=True, **SLOTS) -class MinLen(BaseMetadata): - """ - MinLen() implies minimum inclusive length, - e.g. ``len(value) >= min_length``. - """ - - min_length: Annotated[int, Ge(0)] - - -@dataclass(frozen=True, **SLOTS) -class MaxLen(BaseMetadata): - """ - MaxLen() implies maximum inclusive length, - e.g. ``len(value) <= max_length``. - """ - - max_length: Annotated[int, Ge(0)] - - -@dataclass(frozen=True, **SLOTS) -class Len(GroupedMetadata): - """ - Len() implies that ``min_length <= len(value) <= max_length``. - - Upper bound may be omitted or ``None`` to indicate no upper length bound. - """ - - min_length: Annotated[int, Ge(0)] = 0 - max_length: Optional[Annotated[int, Ge(0)]] = None - - def __iter__(self) -> Iterator[BaseMetadata]: - """Unpack a Len into zone or more single-bounds.""" - if self.min_length > 0: - yield MinLen(self.min_length) - if self.max_length is not None: - yield MaxLen(self.max_length) - - -@dataclass(frozen=True, **SLOTS) -class Timezone(BaseMetadata): - """Timezone(tz=...) requires a datetime to be aware (or ``tz=None``, naive). - - ``Annotated[datetime, Timezone(None)]`` must be a naive datetime. - ``Timezone[...]`` (the ellipsis literal) expresses that the datetime must be - tz-aware but any timezone is allowed. - - You may also pass a specific timezone string or tzinfo object such as - ``Timezone(timezone.utc)`` or ``Timezone("Africa/Abidjan")`` to express that - you only allow a specific timezone, though we note that this is often - a symptom of poor design. - """ - - tz: Union[str, tzinfo, EllipsisType, None] - - -@dataclass(frozen=True, **SLOTS) -class Unit(BaseMetadata): - """Indicates that the value is a physical quantity with the specified unit. - - It is intended for usage with numeric types, where the value represents the - magnitude of the quantity. For example, ``distance: Annotated[float, Unit('m')]`` - or ``speed: Annotated[float, Unit('m/s')]``. - - Interpretation of the unit string is left to the discretion of the consumer. - It is suggested to follow conventions established by python libraries that work - with physical quantities, such as - - - ``pint`` : - - ``astropy.units``: - - For indicating a quantity with a certain dimensionality but without a specific unit - it is recommended to use square brackets, e.g. `Annotated[float, Unit('[time]')]`. - Note, however, ``annotated_types`` itself makes no use of the unit string. - """ - - unit: str - - -@dataclass(frozen=True, **SLOTS) -class Predicate(BaseMetadata): - """``Predicate(func: Callable)`` implies `func(value)` is truthy for valid values. - - Users should prefer statically inspectable metadata, but if you need the full - power and flexibility of arbitrary runtime predicates... here it is. - - We provide a few predefined predicates for common string constraints: - ``IsLower = Predicate(str.islower)``, ``IsUpper = Predicate(str.isupper)``, and - ``IsDigits = Predicate(str.isdigit)``. Users are encouraged to use methods which - can be given special handling, and avoid indirection like ``lambda s: s.lower()``. - - Some libraries might have special logic to handle certain predicates, e.g. by - checking for `str.isdigit` and using its presence to both call custom logic to - enforce digit-only strings, and customise some generated external schema. - - We do not specify what behaviour should be expected for predicates that raise - an exception. For example `Annotated[int, Predicate(str.isdigit)]` might silently - skip invalid constraints, or statically raise an error; or it might try calling it - and then propagate or discard the resulting exception. - """ - - func: Callable[[Any], bool] - - def __repr__(self) -> str: - if getattr(self.func, "__name__", "") == "": - return f"{self.__class__.__name__}({self.func!r})" - if isinstance(self.func, (types.MethodType, types.BuiltinMethodType)) and ( - namespace := getattr(self.func.__self__, "__name__", None) - ): - return f"{self.__class__.__name__}({namespace}.{self.func.__name__})" - if isinstance(self.func, type(str.isascii)): # method descriptor - return f"{self.__class__.__name__}({self.func.__qualname__})" - return f"{self.__class__.__name__}({self.func.__name__})" - - -@dataclass -class Not: - func: Callable[[Any], bool] - - def __call__(self, __v: Any) -> bool: - return not self.func(__v) - - -_StrType = TypeVar("_StrType", bound=str) - -LowerCase = Annotated[_StrType, Predicate(str.islower)] -""" -Return True if the string is a lowercase string, False otherwise. - -A string is lowercase if all cased characters in the string are lowercase and there is at least one cased character in the string. -""" # noqa: E501 -UpperCase = Annotated[_StrType, Predicate(str.isupper)] -""" -Return True if the string is an uppercase string, False otherwise. - -A string is uppercase if all cased characters in the string are uppercase and there is at least one cased character in the string. -""" # noqa: E501 -IsDigit = Annotated[_StrType, Predicate(str.isdigit)] -IsDigits = IsDigit # type: ignore # plural for backwards compatibility, see #63 -""" -Return True if the string is a digit string, False otherwise. - -A string is a digit string if all characters in the string are digits and there is at least one character in the string. -""" # noqa: E501 -IsAscii = Annotated[_StrType, Predicate(str.isascii)] -""" -Return True if all characters in the string are ASCII, False otherwise. - -ASCII characters have code points in the range U+0000-U+007F. Empty string is ASCII too. -""" - -_NumericType = TypeVar('_NumericType', bound=Union[SupportsFloat, SupportsIndex]) -IsFinite = Annotated[_NumericType, Predicate(math.isfinite)] -"""Return True if x is neither an infinity nor a NaN, and False otherwise.""" -IsNotFinite = Annotated[_NumericType, Predicate(Not(math.isfinite))] -"""Return True if x is one of infinity or NaN, and False otherwise""" -IsNan = Annotated[_NumericType, Predicate(math.isnan)] -"""Return True if x is a NaN (not a number), and False otherwise.""" -IsNotNan = Annotated[_NumericType, Predicate(Not(math.isnan))] -"""Return True if x is anything but NaN (not a number), and False otherwise.""" -IsInfinite = Annotated[_NumericType, Predicate(math.isinf)] -"""Return True if x is a positive or negative infinity, and False otherwise.""" -IsNotInfinite = Annotated[_NumericType, Predicate(Not(math.isinf))] -"""Return True if x is neither a positive or negative infinity, and False otherwise.""" - -try: - from typing_extensions import DocInfo, doc # type: ignore [attr-defined] -except ImportError: - - @dataclass(frozen=True, **SLOTS) - class DocInfo: # type: ignore [no-redef] - """ " - The return value of doc(), mainly to be used by tools that want to extract the - Annotated documentation at runtime. - """ - - documentation: str - """The documentation string passed to doc().""" - - def doc( - documentation: str, - ) -> DocInfo: - """ - Add documentation to a type annotation inside of Annotated. - - For example: - - >>> def hi(name: Annotated[int, doc("The name of the user")]) -> None: ... - """ - return DocInfo(documentation) diff --git a/functions/python/data-processor/annotated_types/py.typed b/functions/python/data-processor/annotated_types/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/annotated_types/test_cases.py b/functions/python/data-processor/annotated_types/test_cases.py deleted file mode 100644 index d9164d688..000000000 --- a/functions/python/data-processor/annotated_types/test_cases.py +++ /dev/null @@ -1,151 +0,0 @@ -import math -import sys -from datetime import date, datetime, timedelta, timezone -from decimal import Decimal -from typing import Any, Dict, Iterable, Iterator, List, NamedTuple, Set, Tuple - -if sys.version_info < (3, 9): - from typing_extensions import Annotated -else: - from typing import Annotated - -import annotated_types as at - - -class Case(NamedTuple): - """ - A test case for `annotated_types`. - """ - - annotation: Any - valid_cases: Iterable[Any] - invalid_cases: Iterable[Any] - - -def cases() -> Iterable[Case]: - # Gt, Ge, Lt, Le - yield Case(Annotated[int, at.Gt(4)], (5, 6, 1000), (4, 0, -1)) - yield Case(Annotated[float, at.Gt(0.5)], (0.6, 0.7, 0.8, 0.9), (0.5, 0.0, -0.1)) - yield Case( - Annotated[datetime, at.Gt(datetime(2000, 1, 1))], - [datetime(2000, 1, 2), datetime(2000, 1, 3)], - [datetime(2000, 1, 1), datetime(1999, 12, 31)], - ) - yield Case( - Annotated[datetime, at.Gt(date(2000, 1, 1))], - [date(2000, 1, 2), date(2000, 1, 3)], - [date(2000, 1, 1), date(1999, 12, 31)], - ) - yield Case( - Annotated[datetime, at.Gt(Decimal('1.123'))], - [Decimal('1.1231'), Decimal('123')], - [Decimal('1.123'), Decimal('0')], - ) - - yield Case(Annotated[int, at.Ge(4)], (4, 5, 6, 1000, 4), (0, -1)) - yield Case(Annotated[float, at.Ge(0.5)], (0.5, 0.6, 0.7, 0.8, 0.9), (0.4, 0.0, -0.1)) - yield Case( - Annotated[datetime, at.Ge(datetime(2000, 1, 1))], - [datetime(2000, 1, 2), datetime(2000, 1, 3)], - [datetime(1998, 1, 1), datetime(1999, 12, 31)], - ) - - yield Case(Annotated[int, at.Lt(4)], (0, -1), (4, 5, 6, 1000, 4)) - yield Case(Annotated[float, at.Lt(0.5)], (0.4, 0.0, -0.1), (0.5, 0.6, 0.7, 0.8, 0.9)) - yield Case( - Annotated[datetime, at.Lt(datetime(2000, 1, 1))], - [datetime(1999, 12, 31), datetime(1999, 12, 31)], - [datetime(2000, 1, 2), datetime(2000, 1, 3)], - ) - - yield Case(Annotated[int, at.Le(4)], (4, 0, -1), (5, 6, 1000)) - yield Case(Annotated[float, at.Le(0.5)], (0.5, 0.0, -0.1), (0.6, 0.7, 0.8, 0.9)) - yield Case( - Annotated[datetime, at.Le(datetime(2000, 1, 1))], - [datetime(2000, 1, 1), datetime(1999, 12, 31)], - [datetime(2000, 1, 2), datetime(2000, 1, 3)], - ) - - # Interval - yield Case(Annotated[int, at.Interval(gt=4)], (5, 6, 1000), (4, 0, -1)) - yield Case(Annotated[int, at.Interval(gt=4, lt=10)], (5, 6), (4, 10, 1000, 0, -1)) - yield Case(Annotated[float, at.Interval(ge=0.5, le=1)], (0.5, 0.9, 1), (0.49, 1.1)) - yield Case( - Annotated[datetime, at.Interval(gt=datetime(2000, 1, 1), le=datetime(2000, 1, 3))], - [datetime(2000, 1, 2), datetime(2000, 1, 3)], - [datetime(2000, 1, 1), datetime(2000, 1, 4)], - ) - - yield Case(Annotated[int, at.MultipleOf(multiple_of=3)], (0, 3, 9), (1, 2, 4)) - yield Case(Annotated[float, at.MultipleOf(multiple_of=0.5)], (0, 0.5, 1, 1.5), (0.4, 1.1)) - - # lengths - - yield Case(Annotated[str, at.MinLen(3)], ('123', '1234', 'x' * 10), ('', '1', '12')) - yield Case(Annotated[str, at.Len(3)], ('123', '1234', 'x' * 10), ('', '1', '12')) - yield Case(Annotated[List[int], at.MinLen(3)], ([1, 2, 3], [1, 2, 3, 4], [1] * 10), ([], [1], [1, 2])) - yield Case(Annotated[List[int], at.Len(3)], ([1, 2, 3], [1, 2, 3, 4], [1] * 10), ([], [1], [1, 2])) - - yield Case(Annotated[str, at.MaxLen(4)], ('', '1234'), ('12345', 'x' * 10)) - yield Case(Annotated[str, at.Len(0, 4)], ('', '1234'), ('12345', 'x' * 10)) - yield Case(Annotated[List[str], at.MaxLen(4)], ([], ['a', 'bcdef'], ['a', 'b', 'c']), (['a'] * 5, ['b'] * 10)) - yield Case(Annotated[List[str], at.Len(0, 4)], ([], ['a', 'bcdef'], ['a', 'b', 'c']), (['a'] * 5, ['b'] * 10)) - - yield Case(Annotated[str, at.Len(3, 5)], ('123', '12345'), ('', '1', '12', '123456', 'x' * 10)) - yield Case(Annotated[str, at.Len(3, 3)], ('123',), ('12', '1234')) - - yield Case(Annotated[Dict[int, int], at.Len(2, 3)], [{1: 1, 2: 2}], [{}, {1: 1}, {1: 1, 2: 2, 3: 3, 4: 4}]) - yield Case(Annotated[Set[int], at.Len(2, 3)], ({1, 2}, {1, 2, 3}), (set(), {1}, {1, 2, 3, 4})) - yield Case(Annotated[Tuple[int, ...], at.Len(2, 3)], ((1, 2), (1, 2, 3)), ((), (1,), (1, 2, 3, 4))) - - # Timezone - - yield Case( - Annotated[datetime, at.Timezone(None)], [datetime(2000, 1, 1)], [datetime(2000, 1, 1, tzinfo=timezone.utc)] - ) - yield Case( - Annotated[datetime, at.Timezone(...)], [datetime(2000, 1, 1, tzinfo=timezone.utc)], [datetime(2000, 1, 1)] - ) - yield Case( - Annotated[datetime, at.Timezone(timezone.utc)], - [datetime(2000, 1, 1, tzinfo=timezone.utc)], - [datetime(2000, 1, 1), datetime(2000, 1, 1, tzinfo=timezone(timedelta(hours=6)))], - ) - yield Case( - Annotated[datetime, at.Timezone('Europe/London')], - [datetime(2000, 1, 1, tzinfo=timezone(timedelta(0), name='Europe/London'))], - [datetime(2000, 1, 1), datetime(2000, 1, 1, tzinfo=timezone(timedelta(hours=6)))], - ) - - # Quantity - - yield Case(Annotated[float, at.Unit(unit='m')], (5, 4.2), ('5m', '4.2m')) - - # predicate types - - yield Case(at.LowerCase[str], ['abc', 'foobar'], ['', 'A', 'Boom']) - yield Case(at.UpperCase[str], ['ABC', 'DEFO'], ['', 'a', 'abc', 'AbC']) - yield Case(at.IsDigit[str], ['123'], ['', 'ab', 'a1b2']) - yield Case(at.IsAscii[str], ['123', 'foo bar'], ['£100', '😊', 'whatever 👀']) - - yield Case(Annotated[int, at.Predicate(lambda x: x % 2 == 0)], [0, 2, 4], [1, 3, 5]) - - yield Case(at.IsFinite[float], [1.23], [math.nan, math.inf, -math.inf]) - yield Case(at.IsNotFinite[float], [math.nan, math.inf], [1.23]) - yield Case(at.IsNan[float], [math.nan], [1.23, math.inf]) - yield Case(at.IsNotNan[float], [1.23, math.inf], [math.nan]) - yield Case(at.IsInfinite[float], [math.inf], [math.nan, 1.23]) - yield Case(at.IsNotInfinite[float], [math.nan, 1.23], [math.inf]) - - # check stacked predicates - yield Case(at.IsInfinite[Annotated[float, at.Predicate(lambda x: x > 0)]], [math.inf], [-math.inf, 1.23, math.nan]) - - # doc - yield Case(Annotated[int, at.doc("A number")], [1, 2], []) - - # custom GroupedMetadata - class MyCustomGroupedMetadata(at.GroupedMetadata): - def __iter__(self) -> Iterator[at.Predicate]: - yield at.Predicate(lambda x: float(x).is_integer()) - - yield Case(Annotated[float, MyCustomGroupedMetadata()], [0, 2.0], [0.01, 1.5]) diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/INSTALLER b/functions/python/data-processor/anyio-4.13.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/METADATA b/functions/python/data-processor/anyio-4.13.0.dist-info/METADATA deleted file mode 100644 index 2de1cd552..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/METADATA +++ /dev/null @@ -1,105 +0,0 @@ -Metadata-Version: 2.4 -Name: anyio -Version: 4.13.0 -Summary: High-level concurrency and networking framework on top of asyncio or Trio -Author-email: Alex Grönholm -License-Expression: MIT -Project-URL: Documentation, https://anyio.readthedocs.io/en/latest/ -Project-URL: Changelog, https://anyio.readthedocs.io/en/stable/versionhistory.html -Project-URL: Source code, https://github.com/agronholm/anyio -Project-URL: Issue tracker, https://github.com/agronholm/anyio/issues -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Framework :: AnyIO -Classifier: Typing :: Typed -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Requires-Python: >=3.10 -Description-Content-Type: text/x-rst -License-File: LICENSE -Requires-Dist: exceptiongroup>=1.0.2; python_version < "3.11" -Requires-Dist: idna>=2.8 -Requires-Dist: typing_extensions>=4.5; python_version < "3.13" -Provides-Extra: trio -Requires-Dist: trio>=0.32.0; extra == "trio" -Dynamic: license-file - -.. image:: https://github.com/agronholm/anyio/actions/workflows/test.yml/badge.svg - :target: https://github.com/agronholm/anyio/actions/workflows/test.yml - :alt: Build Status -.. image:: https://coveralls.io/repos/github/agronholm/anyio/badge.svg?branch=master - :target: https://coveralls.io/github/agronholm/anyio?branch=master - :alt: Code Coverage -.. image:: https://readthedocs.org/projects/anyio/badge/?version=latest - :target: https://anyio.readthedocs.io/en/latest/?badge=latest - :alt: Documentation -.. image:: https://badges.gitter.im/gitterHQ/gitter.svg - :target: https://gitter.im/python-trio/AnyIO - :alt: Gitter chat -.. image:: https://tidelift.com/badges/package/pypi/anyio - :target: https://tidelift.com/subscription/pkg/pypi-anyio - :alt: Tidelift - -AnyIO is an asynchronous networking and concurrency library that works on top of either asyncio_ or -Trio_. It implements Trio-like `structured concurrency`_ (SC) on top of asyncio and works in harmony -with the native SC of Trio itself. - -Applications and libraries written against AnyIO's API will run unmodified on either asyncio_ or -Trio_. AnyIO can also be adopted into a library or application incrementally – bit by bit, no full -refactoring necessary. It will blend in with the native libraries of your chosen backend. - -To find out why you might want to use AnyIO's APIs instead of asyncio's, you can read about it -`here `_. - -Documentation -------------- - -View full documentation at: https://anyio.readthedocs.io/ - -Features --------- - -AnyIO offers the following functionality: - -* Task groups (nurseries_ in trio terminology) -* High-level networking (TCP, UDP and UNIX sockets) - - * `Happy eyeballs`_ algorithm for TCP connections (more robust than that of asyncio on Python - 3.8) - * async/await style UDP sockets (unlike asyncio where you still have to use Transports and - Protocols) - -* A versatile API for byte streams and object streams -* Inter-task synchronization and communication (locks, conditions, events, semaphores, object - streams) -* Worker threads -* Subprocesses -* Subinterpreter support for code parallelization (on Python 3.13 and later) -* Asynchronous file I/O (using worker threads) -* Signal handling -* Asynchronous version of the functools_ module - -AnyIO also comes with its own pytest_ plugin which also supports asynchronous fixtures. -It even works with the popular Hypothesis_ library. - -.. _asyncio: https://docs.python.org/3/library/asyncio.html -.. _Trio: https://github.com/python-trio/trio -.. _structured concurrency: https://en.wikipedia.org/wiki/Structured_concurrency -.. _nurseries: https://trio.readthedocs.io/en/stable/reference-core.html#nurseries-and-spawning -.. _Happy eyeballs: https://en.wikipedia.org/wiki/Happy_Eyeballs -.. _pytest: https://docs.pytest.org/en/latest/ -.. _functools: https://docs.python.org/3/library/functools.html -.. _Hypothesis: https://hypothesis.works/ - -Security contact information ----------------------------- - -To report a security vulnerability, please use the `Tidelift security contact`_. -Tidelift will coordinate the fix and disclosure. - -.. _Tidelift security contact: https://tidelift.com/security diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/RECORD b/functions/python/data-processor/anyio-4.13.0.dist-info/RECORD deleted file mode 100644 index a7ece7c19..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/RECORD +++ /dev/null @@ -1,92 +0,0 @@ -anyio-4.13.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -anyio-4.13.0.dist-info/METADATA,sha256=F0EYfiPlmTRwmJN2JktNxJg1GNnl0wHhzOWmz7pFvjM,4513 -anyio-4.13.0.dist-info/RECORD,, -anyio-4.13.0.dist-info/WHEEL,sha256=aeYiig01lYGDzBgS8HxWXOg3uV61G9ijOsup-k9o1sk,91 -anyio-4.13.0.dist-info/entry_points.txt,sha256=_d6Yu6uiaZmNe0CydowirE9Cmg7zUL2g08tQpoS3Qvc,39 -anyio-4.13.0.dist-info/licenses/LICENSE,sha256=U2GsncWPLvX9LpsJxoKXwX8ElQkJu8gCO9uC6s8iwrA,1081 -anyio-4.13.0.dist-info/top_level.txt,sha256=QglSMiWX8_5dpoVAEIHdEYzvqFMdSYWmCj6tYw2ITkQ,6 -anyio/__init__.py,sha256=7iDVqMUprUuKNY91FuoKqayAhR-OY136YDPI6P78HHk,6170 -anyio/__pycache__/__init__.cpython-314.pyc,, -anyio/__pycache__/from_thread.cpython-314.pyc,, -anyio/__pycache__/functools.cpython-314.pyc,, -anyio/__pycache__/lowlevel.cpython-314.pyc,, -anyio/__pycache__/pytest_plugin.cpython-314.pyc,, -anyio/__pycache__/to_interpreter.cpython-314.pyc,, -anyio/__pycache__/to_process.cpython-314.pyc,, -anyio/__pycache__/to_thread.cpython-314.pyc,, -anyio/_backends/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/_backends/__pycache__/__init__.cpython-314.pyc,, -anyio/_backends/__pycache__/_asyncio.cpython-314.pyc,, -anyio/_backends/__pycache__/_trio.cpython-314.pyc,, -anyio/_backends/_asyncio.py,sha256=kuqlg2sBUsFdgY80xSDAw60Gx_4WNCl9iSL5XlY6lCU,99476 -anyio/_backends/_trio.py,sha256=l9U-TsKRxzmTQxSMvOhn0bNeFn_iRx3Ho30jvR5Bdu0,41366 -anyio/_core/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/_core/__pycache__/__init__.cpython-314.pyc,, -anyio/_core/__pycache__/_asyncio_selector_thread.cpython-314.pyc,, -anyio/_core/__pycache__/_contextmanagers.cpython-314.pyc,, -anyio/_core/__pycache__/_eventloop.cpython-314.pyc,, -anyio/_core/__pycache__/_exceptions.cpython-314.pyc,, -anyio/_core/__pycache__/_fileio.cpython-314.pyc,, -anyio/_core/__pycache__/_resources.cpython-314.pyc,, -anyio/_core/__pycache__/_signals.cpython-314.pyc,, -anyio/_core/__pycache__/_sockets.cpython-314.pyc,, -anyio/_core/__pycache__/_streams.cpython-314.pyc,, -anyio/_core/__pycache__/_subprocesses.cpython-314.pyc,, -anyio/_core/__pycache__/_synchronization.cpython-314.pyc,, -anyio/_core/__pycache__/_tasks.cpython-314.pyc,, -anyio/_core/__pycache__/_tempfile.cpython-314.pyc,, -anyio/_core/__pycache__/_testing.cpython-314.pyc,, -anyio/_core/__pycache__/_typedattr.cpython-314.pyc,, -anyio/_core/_asyncio_selector_thread.py,sha256=2PdxFM3cs02Kp6BSppbvmRT7q7asreTW5FgBxEsflBo,5626 -anyio/_core/_contextmanagers.py,sha256=YInBCabiEeS-UaP_Jdxa1CaFC71ETPW8HZTHIM8Rsc8,7215 -anyio/_core/_eventloop.py,sha256=c2EdcBX-xnKwxPcC4Pjn3_qG9I-x4IWFO2R9RqCGjM4,6448 -anyio/_core/_exceptions.py,sha256=Y3aq-Wxd7Q2HqwSg7nZPvRsHEuGazv_qeet6gqEBdPk,4407 -anyio/_core/_fileio.py,sha256=CKi1gFNiW2G4knWeBE7He7-rptQwgYjDUWfG8DSlvLs,25665 -anyio/_core/_resources.py,sha256=NbmU5O5UX3xEyACnkmYX28Fmwdl-f-ny0tHym26e0w0,435 -anyio/_core/_signals.py,sha256=mjTBB2hTKNPRlU0IhnijeQedpWOGERDiMjSlJQsFrug,1016 -anyio/_core/_sockets.py,sha256=RBXHcUqZt5gg_-OOfgHVv8uq2FSKk1uVUzTdpjBoI1o,34977 -anyio/_core/_streams.py,sha256=FczFwIgDpnkK0bODWJXMpsUJYdvAD04kaUaGzJU8DK0,1806 -anyio/_core/_subprocesses.py,sha256=tkmkPKEkEaiMD8C9WRZBlmgjOYRDRbZdte6e-unay2E,7916 -anyio/_core/_synchronization.py,sha256=9G3fvRsPNrrWJ_Z6gD_80wXq8I8qgAyhwM8PvHQnT2c,21061 -anyio/_core/_tasks.py,sha256=pVB7K6AAulzUM8YgXAeqNZG44nSyZ1bYJjH8GznC00I,5435 -anyio/_core/_tempfile.py,sha256=jE2w59FRF3yRo4vjkjfZF2YcqsBZvc66VWRwrJGDYGk,19624 -anyio/_core/_testing.py,sha256=u7MPqGXwpTxqI7hclSdNA30z2GH1Nw258uwKvy_RfBg,2340 -anyio/_core/_typedattr.py,sha256=P4ozZikn3-DbpoYcvyghS_FOYAgbmUxeoU8-L_07pZM,2508 -anyio/abc/__init__.py,sha256=6mWhcl_pGXhrgZVHP_TCfMvIXIOp9mroEFM90fYCU_U,2869 -anyio/abc/__pycache__/__init__.cpython-314.pyc,, -anyio/abc/__pycache__/_eventloop.cpython-314.pyc,, -anyio/abc/__pycache__/_resources.cpython-314.pyc,, -anyio/abc/__pycache__/_sockets.cpython-314.pyc,, -anyio/abc/__pycache__/_streams.cpython-314.pyc,, -anyio/abc/__pycache__/_subprocesses.cpython-314.pyc,, -anyio/abc/__pycache__/_tasks.cpython-314.pyc,, -anyio/abc/__pycache__/_testing.cpython-314.pyc,, -anyio/abc/_eventloop.py,sha256=39lYnmtvoHaZw22sWBKOTA_zv7bamOnr8O49PqgDXdw,10629 -anyio/abc/_resources.py,sha256=DrYvkNN1hH6Uvv5_5uKySvDsnknGVDe8FCKfko0VtN8,783 -anyio/abc/_sockets.py,sha256=OmVDrfemVvF9c5K1tpBgQyV6fn5v0XyCExLAqBOGz9o,13124 -anyio/abc/_streams.py,sha256=HYvna1iZbWcwLROTO6IhLX79RTRLPShZMWe0sG1q54I,7481 -anyio/abc/_subprocesses.py,sha256=cumAPJTktOQtw63IqG0lDpyZqu_l1EElvQHMiwJgL08,2067 -anyio/abc/_tasks.py,sha256=KC7wrciE48AINOI-AhPutnFhe1ewfP7QnamFlDzqesQ,3721 -anyio/abc/_testing.py,sha256=tBJUzkSfOXJw23fe8qSJ03kJlShOYjjaEyFB6k6MYT8,1821 -anyio/from_thread.py,sha256=L-0w1HxJ6BSb-KuVi57k5Tkc3yzQrx3QK5tAxMPcY-0,19141 -anyio/functools.py,sha256=5AWM1iYTKkTzptvUhQDdLSh5GvbBW-vcs-SAUfIfA9A,12076 -anyio/lowlevel.py,sha256=AyKLVK3LaWSoK39LkCKxE4_GDMLKZBNqTrLUgk63y80,5158 -anyio/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/pytest_plugin.py,sha256=t6h4KJstqIxfxwTZ1YO8vpUVuB99nfCLltn0NHfatHo,12775 -anyio/streams/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -anyio/streams/__pycache__/__init__.cpython-314.pyc,, -anyio/streams/__pycache__/buffered.cpython-314.pyc,, -anyio/streams/__pycache__/file.cpython-314.pyc,, -anyio/streams/__pycache__/memory.cpython-314.pyc,, -anyio/streams/__pycache__/stapled.cpython-314.pyc,, -anyio/streams/__pycache__/text.cpython-314.pyc,, -anyio/streams/__pycache__/tls.cpython-314.pyc,, -anyio/streams/buffered.py,sha256=2R3PeJhe4EXrdYqz44Y6-Eg9R6DrmlsYrP36Ir43-po,6263 -anyio/streams/file.py,sha256=msnrotVKGMQomUu_Rj2qz9MvIdUp6d3JGr7MOEO8kV4,4428 -anyio/streams/memory.py,sha256=F0zwzvFJKAhX_LRZGoKzzqDC2oMM-f-yyTBrEYEGOaU,10740 -anyio/streams/stapled.py,sha256=T8Xqwf8K6EgURPxbt1N4i7A8BAk-gScv-GRhjLXIf_o,4390 -anyio/streams/text.py,sha256=BcVAGJw1VRvtIqnv-o0Rb0pwH7p8vwlvl21xHq522ag,5765 -anyio/streams/tls.py,sha256=DQVkXUvsTEYKkBO8dlVU7j_5H8QOtLy4sGi1Wrjqevo,15303 -anyio/to_interpreter.py,sha256=_mLngrMy97TMR6VbW4Y6YzDUk9ZuPcQMPlkuyRh3C9k,7100 -anyio/to_process.py,sha256=J7gAA_YOuoHqnpDAf5fm1Qu6kOmTzdFbiDNvnV755vk,9798 -anyio/to_thread.py,sha256=f6h_k2d743GBv9FhAnhM_YpTvWgIrzBy9cOE0eJ1UJw,2693 diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/WHEEL b/functions/python/data-processor/anyio-4.13.0.dist-info/WHEEL deleted file mode 100644 index 14a883f29..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (82.0.1) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/entry_points.txt b/functions/python/data-processor/anyio-4.13.0.dist-info/entry_points.txt deleted file mode 100644 index 44dd9bdc3..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[pytest11] -anyio = anyio.pytest_plugin diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/licenses/LICENSE b/functions/python/data-processor/anyio-4.13.0.dist-info/licenses/LICENSE deleted file mode 100644 index 104eebf5a..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,20 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Alex Grönholm - -Permission is hereby granted, free of charge, to any person obtaining a copy of -this software and associated documentation files (the "Software"), to deal in -the Software without restriction, including without limitation the rights to -use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of -the Software, and to permit persons to whom the Software is furnished to do so, -subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS -FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR -COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER -IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN -CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/functions/python/data-processor/anyio-4.13.0.dist-info/top_level.txt b/functions/python/data-processor/anyio-4.13.0.dist-info/top_level.txt deleted file mode 100644 index c77c069ec..000000000 --- a/functions/python/data-processor/anyio-4.13.0.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -anyio diff --git a/functions/python/data-processor/anyio/__init__.py b/functions/python/data-processor/anyio/__init__.py deleted file mode 100644 index d23c5a5a2..000000000 --- a/functions/python/data-processor/anyio/__init__.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -from ._core._contextmanagers import AsyncContextManagerMixin as AsyncContextManagerMixin -from ._core._contextmanagers import ContextManagerMixin as ContextManagerMixin -from ._core._eventloop import current_time as current_time -from ._core._eventloop import get_all_backends as get_all_backends -from ._core._eventloop import get_available_backends as get_available_backends -from ._core._eventloop import get_cancelled_exc_class as get_cancelled_exc_class -from ._core._eventloop import run as run -from ._core._eventloop import sleep as sleep -from ._core._eventloop import sleep_forever as sleep_forever -from ._core._eventloop import sleep_until as sleep_until -from ._core._exceptions import BrokenResourceError as BrokenResourceError -from ._core._exceptions import BrokenWorkerInterpreter as BrokenWorkerInterpreter -from ._core._exceptions import BrokenWorkerProcess as BrokenWorkerProcess -from ._core._exceptions import BusyResourceError as BusyResourceError -from ._core._exceptions import ClosedResourceError as ClosedResourceError -from ._core._exceptions import ConnectionFailed as ConnectionFailed -from ._core._exceptions import DelimiterNotFound as DelimiterNotFound -from ._core._exceptions import EndOfStream as EndOfStream -from ._core._exceptions import IncompleteRead as IncompleteRead -from ._core._exceptions import NoEventLoopError as NoEventLoopError -from ._core._exceptions import RunFinishedError as RunFinishedError -from ._core._exceptions import TypedAttributeLookupError as TypedAttributeLookupError -from ._core._exceptions import WouldBlock as WouldBlock -from ._core._fileio import AsyncFile as AsyncFile -from ._core._fileio import Path as Path -from ._core._fileio import open_file as open_file -from ._core._fileio import wrap_file as wrap_file -from ._core._resources import aclose_forcefully as aclose_forcefully -from ._core._signals import open_signal_receiver as open_signal_receiver -from ._core._sockets import TCPConnectable as TCPConnectable -from ._core._sockets import UNIXConnectable as UNIXConnectable -from ._core._sockets import as_connectable as as_connectable -from ._core._sockets import connect_tcp as connect_tcp -from ._core._sockets import connect_unix as connect_unix -from ._core._sockets import create_connected_udp_socket as create_connected_udp_socket -from ._core._sockets import ( - create_connected_unix_datagram_socket as create_connected_unix_datagram_socket, -) -from ._core._sockets import create_tcp_listener as create_tcp_listener -from ._core._sockets import create_udp_socket as create_udp_socket -from ._core._sockets import create_unix_datagram_socket as create_unix_datagram_socket -from ._core._sockets import create_unix_listener as create_unix_listener -from ._core._sockets import getaddrinfo as getaddrinfo -from ._core._sockets import getnameinfo as getnameinfo -from ._core._sockets import notify_closing as notify_closing -from ._core._sockets import wait_readable as wait_readable -from ._core._sockets import wait_socket_readable as wait_socket_readable -from ._core._sockets import wait_socket_writable as wait_socket_writable -from ._core._sockets import wait_writable as wait_writable -from ._core._streams import create_memory_object_stream as create_memory_object_stream -from ._core._subprocesses import open_process as open_process -from ._core._subprocesses import run_process as run_process -from ._core._synchronization import CapacityLimiter as CapacityLimiter -from ._core._synchronization import ( - CapacityLimiterStatistics as CapacityLimiterStatistics, -) -from ._core._synchronization import Condition as Condition -from ._core._synchronization import ConditionStatistics as ConditionStatistics -from ._core._synchronization import Event as Event -from ._core._synchronization import EventStatistics as EventStatistics -from ._core._synchronization import Lock as Lock -from ._core._synchronization import LockStatistics as LockStatistics -from ._core._synchronization import ResourceGuard as ResourceGuard -from ._core._synchronization import Semaphore as Semaphore -from ._core._synchronization import SemaphoreStatistics as SemaphoreStatistics -from ._core._tasks import TASK_STATUS_IGNORED as TASK_STATUS_IGNORED -from ._core._tasks import CancelScope as CancelScope -from ._core._tasks import create_task_group as create_task_group -from ._core._tasks import current_effective_deadline as current_effective_deadline -from ._core._tasks import fail_after as fail_after -from ._core._tasks import move_on_after as move_on_after -from ._core._tempfile import NamedTemporaryFile as NamedTemporaryFile -from ._core._tempfile import SpooledTemporaryFile as SpooledTemporaryFile -from ._core._tempfile import TemporaryDirectory as TemporaryDirectory -from ._core._tempfile import TemporaryFile as TemporaryFile -from ._core._tempfile import gettempdir as gettempdir -from ._core._tempfile import gettempdirb as gettempdirb -from ._core._tempfile import mkdtemp as mkdtemp -from ._core._tempfile import mkstemp as mkstemp -from ._core._testing import TaskInfo as TaskInfo -from ._core._testing import get_current_task as get_current_task -from ._core._testing import get_running_tasks as get_running_tasks -from ._core._testing import wait_all_tasks_blocked as wait_all_tasks_blocked -from ._core._typedattr import TypedAttributeProvider as TypedAttributeProvider -from ._core._typedattr import TypedAttributeSet as TypedAttributeSet -from ._core._typedattr import typed_attribute as typed_attribute - -# Re-export imports so they look like they live directly in this package -for __value in list(locals().values()): - if getattr(__value, "__module__", "").startswith("anyio."): - __value.__module__ = __name__ - - -del __value - - -def __getattr__(attr: str) -> type[BrokenWorkerInterpreter]: - """Support deprecated aliases.""" - if attr == "BrokenWorkerIntepreter": - import warnings - - warnings.warn( - "The 'BrokenWorkerIntepreter' alias is deprecated, use 'BrokenWorkerInterpreter' instead.", - DeprecationWarning, - stacklevel=2, - ) - return BrokenWorkerInterpreter - - raise AttributeError(f"module {__name__!r} has no attribute {attr!r}") diff --git a/functions/python/data-processor/anyio/_backends/__init__.py b/functions/python/data-processor/anyio/_backends/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/anyio/_backends/_asyncio.py b/functions/python/data-processor/anyio/_backends/_asyncio.py deleted file mode 100644 index 9f1ddc242..000000000 --- a/functions/python/data-processor/anyio/_backends/_asyncio.py +++ /dev/null @@ -1,2996 +0,0 @@ -from __future__ import annotations - -import array -import asyncio -import concurrent.futures -import contextvars -import math -import os -import socket -import sys -import threading -import weakref -from asyncio import ( - AbstractEventLoop, - CancelledError, - all_tasks, - create_task, - current_task, - get_running_loop, - sleep, -) -from asyncio.base_events import _run_until_complete_cb # type: ignore[attr-defined] -from collections import OrderedDict, deque -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Iterable, - Sequence, -) -from concurrent.futures import Future -from contextlib import AbstractContextManager, suppress -from contextvars import Context, copy_context -from dataclasses import dataclass, field -from functools import partial, wraps -from inspect import ( - CORO_RUNNING, - CORO_SUSPENDED, - getcoroutinestate, - iscoroutine, -) -from io import IOBase -from os import PathLike -from queue import Queue -from signal import Signals -from socket import AddressFamily, SocketKind -from threading import Thread -from types import CodeType, TracebackType -from typing import ( - IO, - TYPE_CHECKING, - Any, - ParamSpec, - TypeVar, - cast, -) -from weakref import WeakKeyDictionary - -from .. import ( - CapacityLimiterStatistics, - EventStatistics, - LockStatistics, - TaskInfo, - abc, -) -from .._core._eventloop import ( - claim_worker_thread, - set_current_async_library, - threadlocals, -) -from .._core._exceptions import ( - BrokenResourceError, - BusyResourceError, - ClosedResourceError, - EndOfStream, - RunFinishedError, - WouldBlock, -) -from .._core._sockets import convert_ipv6_sockaddr -from .._core._streams import create_memory_object_stream -from .._core._synchronization import ( - CapacityLimiter as BaseCapacityLimiter, -) -from .._core._synchronization import Event as BaseEvent -from .._core._synchronization import Lock as BaseLock -from .._core._synchronization import ( - ResourceGuard, - SemaphoreStatistics, -) -from .._core._synchronization import Semaphore as BaseSemaphore -from .._core._tasks import CancelScope as BaseCancelScope -from ..abc import ( - AsyncBackend, - IPSockAddrType, - SocketListener, - UDPPacketType, - UNIXDatagramPacketType, -) -from ..abc._eventloop import StrOrBytesPath -from ..lowlevel import RunVar -from ..streams.memory import MemoryObjectReceiveStream, MemoryObjectSendStream - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike -else: - FileDescriptorLike = object - -if sys.version_info >= (3, 11): - from asyncio import Runner - from typing import TypeVarTuple, Unpack -else: - import contextvars - import enum - import signal - from asyncio import coroutines, events, exceptions, tasks - - from exceptiongroup import BaseExceptionGroup - from typing_extensions import TypeVarTuple, Unpack - - class _State(enum.Enum): - CREATED = "created" - INITIALIZED = "initialized" - CLOSED = "closed" - - class Runner: - # Copied from CPython 3.11 - def __init__( - self, - *, - debug: bool | None = None, - loop_factory: Callable[[], AbstractEventLoop] | None = None, - ): - self._state = _State.CREATED - self._debug = debug - self._loop_factory = loop_factory - self._loop: AbstractEventLoop | None = None - self._context = None - self._interrupt_count = 0 - self._set_event_loop = False - - def __enter__(self) -> Runner: - self._lazy_init() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def close(self) -> None: - """Shutdown and close event loop.""" - loop = self._loop - if self._state is not _State.INITIALIZED or loop is None: - return - try: - _cancel_all_tasks(loop) - loop.run_until_complete(loop.shutdown_asyncgens()) - if hasattr(loop, "shutdown_default_executor"): - loop.run_until_complete(loop.shutdown_default_executor()) - else: - loop.run_until_complete(_shutdown_default_executor(loop)) - finally: - if self._set_event_loop: - events.set_event_loop(None) - loop.close() - self._loop = None - self._state = _State.CLOSED - - def get_loop(self) -> AbstractEventLoop: - """Return embedded event loop.""" - self._lazy_init() - return self._loop - - def run(self, coro: Coroutine[T_Retval], *, context=None) -> T_Retval: - """Run a coroutine inside the embedded event loop.""" - if not coroutines.iscoroutine(coro): - raise ValueError(f"a coroutine was expected, got {coro!r}") - - if events._get_running_loop() is not None: - # fail fast with short traceback - raise RuntimeError( - "Runner.run() cannot be called from a running event loop" - ) - - self._lazy_init() - - if context is None: - context = self._context - task = context.run(self._loop.create_task, coro) - - if ( - threading.current_thread() is threading.main_thread() - and signal.getsignal(signal.SIGINT) is signal.default_int_handler - ): - sigint_handler = partial(self._on_sigint, main_task=task) - try: - signal.signal(signal.SIGINT, sigint_handler) - except ValueError: - # `signal.signal` may throw if `threading.main_thread` does - # not support signals (e.g. embedded interpreter with signals - # not registered - see gh-91880) - sigint_handler = None - else: - sigint_handler = None - - self._interrupt_count = 0 - try: - return self._loop.run_until_complete(task) - except exceptions.CancelledError: - if self._interrupt_count > 0: - uncancel = getattr(task, "uncancel", None) - if uncancel is not None and uncancel() == 0: - raise KeyboardInterrupt # noqa: B904 - raise # CancelledError - finally: - if ( - sigint_handler is not None - and signal.getsignal(signal.SIGINT) is sigint_handler - ): - signal.signal(signal.SIGINT, signal.default_int_handler) - - def _lazy_init(self) -> None: - if self._state is _State.CLOSED: - raise RuntimeError("Runner is closed") - if self._state is _State.INITIALIZED: - return - if self._loop_factory is None: - self._loop = events.new_event_loop() - if not self._set_event_loop: - # Call set_event_loop only once to avoid calling - # attach_loop multiple times on child watchers - events.set_event_loop(self._loop) - self._set_event_loop = True - else: - self._loop = self._loop_factory() - if self._debug is not None: - self._loop.set_debug(self._debug) - self._context = contextvars.copy_context() - self._state = _State.INITIALIZED - - def _on_sigint(self, signum, frame, main_task: asyncio.Task) -> None: - self._interrupt_count += 1 - if self._interrupt_count == 1 and not main_task.done(): - main_task.cancel() - # wakeup loop if it is blocked by select() with long timeout - self._loop.call_soon_threadsafe(lambda: None) - return - raise KeyboardInterrupt() - - def _cancel_all_tasks(loop: AbstractEventLoop) -> None: - to_cancel = tasks.all_tasks(loop) - if not to_cancel: - return - - for task in to_cancel: - task.cancel() - - loop.run_until_complete(tasks.gather(*to_cancel, return_exceptions=True)) - - for task in to_cancel: - if task.cancelled(): - continue - if task.exception() is not None: - loop.call_exception_handler( - { - "message": "unhandled exception during asyncio.run() shutdown", - "exception": task.exception(), - "task": task, - } - ) - - async def _shutdown_default_executor(loop: AbstractEventLoop) -> None: - """Schedule the shutdown of the default executor.""" - - def _do_shutdown(future: asyncio.futures.Future) -> None: - try: - loop._default_executor.shutdown(wait=True) # type: ignore[attr-defined] - loop.call_soon_threadsafe(future.set_result, None) - except Exception as ex: - loop.call_soon_threadsafe(future.set_exception, ex) - - loop._executor_shutdown_called = True - if loop._default_executor is None: - return - future = loop.create_future() - thread = threading.Thread(target=_do_shutdown, args=(future,)) - thread.start() - try: - await future - finally: - thread.join() - - -T_Retval = TypeVar("T_Retval") -T_contra = TypeVar("T_contra", contravariant=True) -PosArgsT = TypeVarTuple("PosArgsT") -P = ParamSpec("P") - -_root_task: RunVar[asyncio.Task | None] = RunVar("_root_task") - - -def find_root_task() -> asyncio.Task: - root_task = _root_task.get(None) - if root_task is not None and not root_task.done(): - return root_task - - # Look for a task that has been started via run_until_complete() - for task in all_tasks(): - if task._callbacks and not task.done(): - callbacks = [cb for cb, context in task._callbacks] - for cb in callbacks: - if ( - cb is _run_until_complete_cb - or getattr(cb, "__module__", None) == "uvloop.loop" - ): - _root_task.set(task) - return task - - # Look up the topmost task in the AnyIO task tree, if possible - task = cast(asyncio.Task, current_task()) - state = _task_states.get(task) - if state: - cancel_scope = state.cancel_scope - while cancel_scope and cancel_scope._parent_scope is not None: - cancel_scope = cancel_scope._parent_scope - - if cancel_scope is not None: - return cast(asyncio.Task, cancel_scope._host_task) - - return task - - -def get_callable_name(func: Callable) -> str: - module = getattr(func, "__module__", None) - qualname = getattr(func, "__qualname__", None) - return ".".join([x for x in (module, qualname) if x]) - - -# -# Event loop -# - -_run_vars: WeakKeyDictionary[asyncio.AbstractEventLoop, Any] = WeakKeyDictionary() - - -def _task_started(task: asyncio.Task) -> bool: - """Return ``True`` if the task has been started and has not finished.""" - # The task coro should never be None here, as we never add finished tasks to the - # task list - coro = task.get_coro() - assert coro is not None - try: - return getcoroutinestate(coro) in (CORO_RUNNING, CORO_SUSPENDED) - except AttributeError: - # task coro is async_genenerator_asend https://bugs.python.org/issue37771 - raise Exception(f"Cannot determine if task {task} has started or not") from None - - -# -# Timeouts and cancellation -# - - -def is_anyio_cancellation(exc: CancelledError) -> bool: - # Sometimes third party frameworks catch a CancelledError and raise a new one, so as - # a workaround we have to look at the previous ones in __context__ too for a - # matching cancel message - while True: - if ( - exc.args - and isinstance(exc.args[0], str) - and exc.args[0].startswith("Cancelled via cancel scope ") - ): - return True - - if isinstance(exc.__context__, CancelledError): - exc = exc.__context__ - continue - - return False - - -class CancelScope(BaseCancelScope): - def __new__( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return object.__new__(cls) - - def __init__(self, deadline: float = math.inf, shield: bool = False): - self._deadline = deadline - self._shield = shield - self._parent_scope: CancelScope | None = None - self._child_scopes: set[CancelScope] = set() - self._cancel_called = False - self._cancel_reason: str | None = None - self._cancelled_caught = False - self._active = False - self._timeout_handle: asyncio.TimerHandle | None = None - self._cancel_handle: asyncio.Handle | None = None - self._tasks: set[asyncio.Task] = set() - self._host_task: asyncio.Task | None = None - if sys.version_info >= (3, 11): - self._pending_uncancellations: int | None = 0 - else: - self._pending_uncancellations = None - - def __enter__(self) -> CancelScope: - if self._active: - raise RuntimeError( - "Each CancelScope may only be used for a single 'with' block" - ) - - self._host_task = host_task = cast(asyncio.Task, current_task()) - self._tasks.add(host_task) - try: - task_state = _task_states[host_task] - except KeyError: - task_state = TaskState(None, self) - _task_states[host_task] = task_state - else: - self._parent_scope = task_state.cancel_scope - task_state.cancel_scope = self - if self._parent_scope is not None: - # If using an eager task factory, the parent scope may not even contain - # the host task - self._parent_scope._child_scopes.add(self) - self._parent_scope._tasks.discard(host_task) - - self._timeout() - self._active = True - - # Start cancelling the host task if the scope was cancelled before entering - if self._cancel_called: - self._deliver_cancellation(self) - - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - del exc_tb - - if not self._active: - raise RuntimeError("This cancel scope is not active") - if current_task() is not self._host_task: - raise RuntimeError( - "Attempted to exit cancel scope in a different task than it was " - "entered in" - ) - - assert self._host_task is not None - host_task_state = _task_states.get(self._host_task) - if host_task_state is None or host_task_state.cancel_scope is not self: - raise RuntimeError( - "Attempted to exit a cancel scope that isn't the current tasks's " - "current cancel scope" - ) - - try: - self._active = False - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._tasks.remove(self._host_task) - if self._parent_scope is not None: - self._parent_scope._child_scopes.remove(self) - self._parent_scope._tasks.add(self._host_task) - - host_task_state.cancel_scope = self._parent_scope - - # Restart the cancellation effort in the closest visible, cancelled parent - # scope if necessary - self._restart_cancellation_in_parent() - - # We only swallow the exception iff it was an AnyIO CancelledError, either - # directly as exc_val or inside an exception group and there are no cancelled - # parent cancel scopes visible to us here - if self._cancel_called and not self._parent_cancellation_is_visible_to_us: - # For each level-cancel() call made on the host task, call uncancel() - while self._pending_uncancellations: - self._host_task.uncancel() - self._pending_uncancellations -= 1 - - # Update cancelled_caught and check for exceptions we must not swallow - if isinstance(exc_val, BaseExceptionGroup): - cancelleds_caught, remaining = exc_val.split( - lambda exc: ( - isinstance(exc, CancelledError) - and is_anyio_cancellation(exc) - ) - ) - - if cancelleds_caught is None: - return False - - self._cancelled_caught = True - - if remaining is None: - return True - - context = remaining.__context__ - try: - # Preserve __cause__ and __suppress_context__ by avoiding `raise - # ... from ...` - raise remaining - finally: - # Preserve __context__ - remaining.__context__ = context - del context - else: - if isinstance(exc_val, CancelledError) and is_anyio_cancellation( - exc_val - ): - self._cancelled_caught = True - return True - else: - return False - else: - if self._pending_uncancellations: - assert self._parent_scope is not None - assert self._parent_scope._pending_uncancellations is not None - self._parent_scope._pending_uncancellations += ( - self._pending_uncancellations - ) - self._pending_uncancellations = 0 - - return False - finally: - self._host_task = None - del exc_val - - @property - def _effectively_cancelled(self) -> bool: - cancel_scope: CancelScope | None = self - while cancel_scope is not None: - if cancel_scope._cancel_called: - return True - - if cancel_scope.shield: - return False - - cancel_scope = cancel_scope._parent_scope - - return False - - @property - def _parent_cancellation_is_visible_to_us(self) -> bool: - return ( - self._parent_scope is not None - and not self.shield - and self._parent_scope._effectively_cancelled - ) - - def _timeout(self) -> None: - if self._deadline != math.inf: - loop = get_running_loop() - if loop.time() >= self._deadline: - self.cancel("deadline exceeded") - else: - self._timeout_handle = loop.call_at(self._deadline, self._timeout) - - def _deliver_cancellation(self, origin: CancelScope) -> bool: - """ - Deliver cancellation to directly contained tasks and nested cancel scopes. - - Schedule another run at the end if we still have tasks eligible for - cancellation. - - :param origin: the cancel scope that originated the cancellation - :return: ``True`` if the delivery needs to be retried on the next cycle - - """ - should_retry = False - current = current_task() - for task in self._tasks: - should_retry = True - if task._must_cancel: # type: ignore[attr-defined] - continue - - # The task is eligible for cancellation if it has started - if task is not current and (task is self._host_task or _task_started(task)): - waiter = task._fut_waiter # type: ignore[attr-defined] - if not isinstance(waiter, asyncio.Future) or not waiter.done(): - task.cancel(origin._cancel_reason) - if ( - task is origin._host_task - and origin._pending_uncancellations is not None - ): - origin._pending_uncancellations += 1 - - # Deliver cancellation to child scopes that aren't shielded or running their own - # cancellation callbacks - for scope in self._child_scopes: - if not scope._shield and not scope.cancel_called: - should_retry = scope._deliver_cancellation(origin) or should_retry - - # Schedule another callback if there are still tasks left - if origin is self: - if should_retry: - self._cancel_handle = get_running_loop().call_soon( - self._deliver_cancellation, origin - ) - else: - self._cancel_handle = None - - return should_retry - - def _restart_cancellation_in_parent(self) -> None: - """ - Restart the cancellation effort in the closest directly cancelled parent scope. - - """ - scope = self._parent_scope - while scope is not None: - if scope._cancel_called: - if scope._cancel_handle is None: - scope._deliver_cancellation(scope) - - break - - # No point in looking beyond any shielded scope - if scope._shield: - break - - scope = scope._parent_scope - - def cancel(self, reason: str | None = None) -> None: - if not self._cancel_called: - if self._timeout_handle: - self._timeout_handle.cancel() - self._timeout_handle = None - - self._cancel_called = True - self._cancel_reason = f"Cancelled via cancel scope {id(self):x}" - if task := current_task(): - self._cancel_reason += f" by {task}" - - if reason: - self._cancel_reason += f"; reason: {reason}" - - if self._host_task is not None: - self._deliver_cancellation(self) - - @property - def deadline(self) -> float: - return self._deadline - - @deadline.setter - def deadline(self, value: float) -> None: - self._deadline = float(value) - if self._timeout_handle is not None: - self._timeout_handle.cancel() - self._timeout_handle = None - - if self._active and not self._cancel_called: - self._timeout() - - @property - def cancel_called(self) -> bool: - return self._cancel_called - - @property - def cancelled_caught(self) -> bool: - return self._cancelled_caught - - @property - def shield(self) -> bool: - return self._shield - - @shield.setter - def shield(self, value: bool) -> None: - if self._shield != value: - self._shield = value - if not value: - self._restart_cancellation_in_parent() - - -# -# Task states -# - - -class TaskState: - """ - Encapsulates auxiliary task information that cannot be added to the Task instance - itself because there are no guarantees about its implementation. - """ - - __slots__ = "parent_id", "cancel_scope", "__weakref__" - - def __init__(self, parent_id: int | None, cancel_scope: CancelScope | None): - self.parent_id = parent_id - self.cancel_scope = cancel_scope - - -_task_states: WeakKeyDictionary[asyncio.Task, TaskState] = WeakKeyDictionary() - - -# -# Task groups -# - - -class _AsyncioTaskStatus(abc.TaskStatus): - def __init__(self, future: asyncio.Future, parent_id: int): - self._future = future - self._parent_id = parent_id - - def started(self, value: T_contra | None = None) -> None: - try: - self._future.set_result(value) - except asyncio.InvalidStateError: - if not self._future.cancelled(): - raise RuntimeError( - "called 'started' twice on the same task status" - ) from None - - task = cast(asyncio.Task, current_task()) - _task_states[task].parent_id = self._parent_id - - -if sys.version_info >= (3, 12): - _eager_task_factory_code: CodeType | None = asyncio.eager_task_factory.__code__ -else: - _eager_task_factory_code = None - - -class TaskGroup(abc.TaskGroup): - def __init__(self) -> None: - self.cancel_scope: CancelScope = CancelScope() - self._active = False - self._exceptions: list[BaseException] = [] - self._tasks: set[asyncio.Task] = set() - self._on_completed_fut: asyncio.Future[None] | None = None - - async def __aenter__(self) -> TaskGroup: - self.cancel_scope.__enter__() - self._active = True - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - try: - if exc_val is not None: - self.cancel_scope.cancel() - if not isinstance(exc_val, CancelledError): - self._exceptions.append(exc_val) - - loop = get_running_loop() - try: - if self._tasks: - with CancelScope() as wait_scope: - while self._tasks: - self._on_completed_fut = loop.create_future() - - try: - await self._on_completed_fut - except CancelledError as exc: - # Shield the scope against further cancellation attempts, - # as they're not productive (#695) - wait_scope.shield = True - self.cancel_scope.cancel() - - # Set exc_val from the cancellation exception if it was - # previously unset. However, we should not replace a native - # cancellation exception with one raise by a cancel scope. - if exc_val is None or ( - isinstance(exc_val, CancelledError) - and not is_anyio_cancellation(exc) - ): - exc_val = exc - - self._on_completed_fut = None - else: - # If there are no child tasks to wait on, run at least one checkpoint - # anyway - await AsyncIOBackend.cancel_shielded_checkpoint() - - self._active = False - if self._exceptions: - # The exception that got us here should already have been - # added to self._exceptions so it's ok to break exception - # chaining and avoid adding a "During handling of above..." - # for each nesting level. - raise BaseExceptionGroup( - "unhandled errors in a TaskGroup", self._exceptions - ) from None - elif exc_val: - raise exc_val - except BaseException as exc: - if self.cancel_scope.__exit__(type(exc), exc, exc.__traceback__): - return True - - raise - - return self.cancel_scope.__exit__(exc_type, exc_val, exc_tb) - finally: - del exc_val, exc_tb, self._exceptions - - def _spawn( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - args: tuple[Unpack[PosArgsT]], - name: object, - task_status_future: asyncio.Future | None = None, - ) -> asyncio.Task: - def task_done(_task: asyncio.Task) -> None: - if sys.version_info >= (3, 14) and self.cancel_scope._host_task is not None: - asyncio.future_discard_from_awaited_by( - _task, self.cancel_scope._host_task - ) - - task_state = _task_states[_task] - assert task_state.cancel_scope is not None - assert _task in task_state.cancel_scope._tasks - task_state.cancel_scope._tasks.remove(_task) - self._tasks.remove(task) - del _task_states[_task] - - if self._on_completed_fut is not None and not self._tasks: - try: - self._on_completed_fut.set_result(None) - except asyncio.InvalidStateError: - pass - - try: - exc = _task.exception() - except CancelledError as e: - while isinstance(e.__context__, CancelledError): - e = e.__context__ - - exc = e - - if exc is not None: - # The future can only be in the cancelled state if the host task was - # cancelled, so return immediately instead of adding one more - # CancelledError to the exceptions list - if task_status_future is not None and task_status_future.cancelled(): - return - - if task_status_future is None or task_status_future.done(): - if not isinstance(exc, CancelledError): - self._exceptions.append(exc) - - if not self.cancel_scope._effectively_cancelled: - self.cancel_scope.cancel() - else: - task_status_future.set_exception(exc) - elif task_status_future is not None and not task_status_future.done(): - task_status_future.set_exception( - RuntimeError("Child exited without calling task_status.started()") - ) - - if not self._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - kwargs = {} - if task_status_future: - parent_id = id(current_task()) - kwargs["task_status"] = _AsyncioTaskStatus( - task_status_future, id(self.cancel_scope._host_task) - ) - else: - parent_id = id(self.cancel_scope._host_task) - - coro = func(*args, **kwargs) - if not iscoroutine(coro): - prefix = f"{func.__module__}." if hasattr(func, "__module__") else "" - raise TypeError( - f"Expected {prefix}{func.__qualname__}() to return a coroutine, but " - f"the return value ({coro!r}) is not a coroutine object" - ) - - name = get_callable_name(func) if name is None else str(name) - loop = asyncio.get_running_loop() - if ( - (factory := loop.get_task_factory()) - and getattr(factory, "__code__", None) is _eager_task_factory_code - and (closure := getattr(factory, "__closure__", None)) - ): - custom_task_constructor = closure[0].cell_contents - task = custom_task_constructor(coro, loop=loop, name=name) - else: - task = create_task(coro, name=name) - - # Make the spawned task inherit the task group's cancel scope - _task_states[task] = TaskState( - parent_id=parent_id, cancel_scope=self.cancel_scope - ) - self.cancel_scope._tasks.add(task) - self._tasks.add(task) - if sys.version_info >= (3, 14) and self.cancel_scope._host_task is not None: - asyncio.future_add_to_awaited_by(task, self.cancel_scope._host_task) - - task.add_done_callback(task_done) - return task - - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> None: - self._spawn(func, args, name) - - async def start( - self, func: Callable[..., Awaitable[Any]], *args: object, name: object = None - ) -> Any: - future: asyncio.Future = asyncio.Future() - task = self._spawn(func, args, name, future) - - # If the task raises an exception after sending a start value without a switch - # point between, the task group is cancelled and this method never proceeds to - # process the completed future. That's why we have to have a shielded cancel - # scope here. - try: - return await future - except CancelledError: - # Cancel the task and wait for it to exit before returning - task.cancel() - with CancelScope(shield=True), suppress(CancelledError): - await task - - raise - - -# -# Threads -# - -_Retval_Queue_Type = tuple[T_Retval | None, BaseException | None] - - -class WorkerThread(Thread): - MAX_IDLE_TIME = 10 # seconds - - def __init__( - self, - root_task: asyncio.Task, - workers: set[WorkerThread], - idle_workers: deque[WorkerThread], - ): - super().__init__(name="AnyIO worker thread") - self.root_task = root_task - self.workers = workers - self.idle_workers = idle_workers - self.loop = root_task._loop - self.queue: Queue[ - tuple[Context, Callable, tuple, asyncio.Future, CancelScope] | None - ] = Queue(2) - self.idle_since = AsyncIOBackend.current_time() - self.stopping = False - - def _report_result( - self, future: asyncio.Future, result: Any, exc: BaseException | None - ) -> None: - self.idle_since = AsyncIOBackend.current_time() - if not self.stopping: - self.idle_workers.append(self) - - if not future.cancelled(): - if exc is not None: - if isinstance(exc, StopIteration): - new_exc = RuntimeError("coroutine raised StopIteration") - new_exc.__cause__ = exc - exc = new_exc - - future.set_exception(exc) - else: - future.set_result(result) - - def run(self) -> None: - with claim_worker_thread(AsyncIOBackend, self.loop): - while True: - item = self.queue.get() - if item is None: - # Shutdown command received - return - - context, func, args, future, cancel_scope = item - if not future.cancelled(): - result = None - exception: BaseException | None = None - threadlocals.current_cancel_scope = cancel_scope - try: - result = context.run(func, *args) - except BaseException as exc: - exception = exc - finally: - del threadlocals.current_cancel_scope - - if not self.loop.is_closed(): - self.loop.call_soon_threadsafe( - self._report_result, future, result, exception - ) - - del result, exception - - self.queue.task_done() - del item, context, func, args, future, cancel_scope - - def stop(self, f: asyncio.Task | None = None) -> None: - self.stopping = True - self.queue.put_nowait(None) - self.workers.discard(self) - try: - self.idle_workers.remove(self) - except ValueError: - pass - - -_threadpool_idle_workers: RunVar[deque[WorkerThread]] = RunVar( - "_threadpool_idle_workers" -) -_threadpool_workers: RunVar[set[WorkerThread]] = RunVar("_threadpool_workers") - - -# -# Subprocesses -# - - -@dataclass(eq=False) -class StreamReaderWrapper(abc.ByteReceiveStream): - _stream: asyncio.StreamReader - - async def receive(self, max_bytes: int = 65536) -> bytes: - data = await self._stream.read(max_bytes) - if data: - return data - else: - raise EndOfStream - - async def aclose(self) -> None: - self._stream.set_exception(ClosedResourceError()) - await AsyncIOBackend.checkpoint() - - -@dataclass(eq=False) -class StreamWriterWrapper(abc.ByteSendStream): - _stream: asyncio.StreamWriter - _closed: bool = field(init=False, default=False) - - async def send(self, item: bytes) -> None: - await AsyncIOBackend.checkpoint_if_cancelled() - stream_paused = self._stream._protocol._paused # type: ignore[attr-defined] - try: - self._stream.write(item) - await self._stream.drain() - except (ConnectionResetError, BrokenPipeError, RuntimeError) as exc: - # If closed by us and/or the peer: - # * on stdlib, drain() raises ConnectionResetError or BrokenPipeError - # * on uvloop and Winloop, write() eventually starts raising RuntimeError - if self._closed: - raise ClosedResourceError from exc - elif self._stream.is_closing(): - raise BrokenResourceError from exc - - raise - - if not stream_paused: - await AsyncIOBackend.cancel_shielded_checkpoint() - - async def aclose(self) -> None: - self._closed = True - self._stream.close() - await AsyncIOBackend.checkpoint() - - -@dataclass(eq=False) -class Process(abc.Process): - _process: asyncio.subprocess.Process - _stdin: StreamWriterWrapper | None - _stdout: StreamReaderWrapper | None - _stderr: StreamReaderWrapper | None - - async def aclose(self) -> None: - with CancelScope(shield=True) as scope: - if self._stdin: - await self._stdin.aclose() - if self._stdout: - await self._stdout.aclose() - if self._stderr: - await self._stderr.aclose() - - scope.shield = False - try: - await self.wait() - except BaseException: - scope.shield = True - self.kill() - await self.wait() - raise - - async def wait(self) -> int: - return await self._process.wait() - - def terminate(self) -> None: - self._process.terminate() - - def kill(self) -> None: - self._process.kill() - - def send_signal(self, signal: int) -> None: - self._process.send_signal(signal) - - @property - def pid(self) -> int: - return self._process.pid - - @property - def returncode(self) -> int | None: - return self._process.returncode - - @property - def stdin(self) -> abc.ByteSendStream | None: - return self._stdin - - @property - def stdout(self) -> abc.ByteReceiveStream | None: - return self._stdout - - @property - def stderr(self) -> abc.ByteReceiveStream | None: - return self._stderr - - -def _forcibly_shutdown_process_pool_on_exit( - workers: set[Process], _task: object -) -> None: - """ - Forcibly shuts down worker processes belonging to this event loop.""" - child_watcher: asyncio.AbstractChildWatcher | None = None # type: ignore[name-defined] - if sys.version_info < (3, 12): - try: - child_watcher = asyncio.get_event_loop_policy().get_child_watcher() - except NotImplementedError: - pass - - # Close as much as possible (w/o async/await) to avoid warnings - for process in workers.copy(): - if process.returncode is not None: - continue - - process._stdin._stream._transport.close() # type: ignore[union-attr] - process._stdout._stream._transport.close() # type: ignore[union-attr] - process._stderr._stream._transport.close() # type: ignore[union-attr] - process.kill() - if child_watcher: - child_watcher.remove_child_handler(process.pid) - - -async def _shutdown_process_pool_on_exit(workers: set[abc.Process]) -> None: - """ - Shuts down worker processes belonging to this event loop. - - NOTE: this only works when the event loop was started using asyncio.run() or - anyio.run(). - - """ - process: abc.Process - try: - await sleep(math.inf) - except asyncio.CancelledError: - workers = workers.copy() - for process in workers: - if process.returncode is None: - process.kill() - - for process in workers: - await process.aclose() - - -# -# Sockets and networking -# - - -class StreamProtocol(asyncio.Protocol): - read_queue: deque[bytes] - read_event: asyncio.Event - write_event: asyncio.Event - exception: Exception | None = None - is_at_eof: bool = False - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - self.read_queue = deque() - self.read_event = asyncio.Event() - self.write_event = asyncio.Event() - self.write_event.set() - cast(asyncio.Transport, transport).set_write_buffer_limits(0) - - def connection_lost(self, exc: Exception | None) -> None: - if exc: - self.exception = exc - - self.read_event.set() - self.write_event.set() - - def data_received(self, data: bytes) -> None: - # ProactorEventloop sometimes sends bytearray instead of bytes - self.read_queue.append(bytes(data)) - self.read_event.set() - - def eof_received(self) -> bool | None: - self.is_at_eof = True - self.read_event.set() - return True - - def pause_writing(self) -> None: - self.write_event = asyncio.Event() - - def resume_writing(self) -> None: - self.write_event.set() - - -class DatagramProtocol(asyncio.DatagramProtocol): - read_queue: deque[tuple[bytes, IPSockAddrType]] - read_event: asyncio.Event - write_event: asyncio.Event - exception: Exception | None = None - - def connection_made(self, transport: asyncio.BaseTransport) -> None: - self.read_queue = deque(maxlen=100) # arbitrary value - self.read_event = asyncio.Event() - self.write_event = asyncio.Event() - self.write_event.set() - - def connection_lost(self, exc: Exception | None) -> None: - self.read_event.set() - self.write_event.set() - - def datagram_received(self, data: bytes, addr: IPSockAddrType) -> None: - addr = convert_ipv6_sockaddr(addr) - self.read_queue.append((data, addr)) - self.read_event.set() - - def error_received(self, exc: Exception) -> None: - self.exception = exc - - def pause_writing(self) -> None: - self.write_event.clear() - - def resume_writing(self) -> None: - self.write_event.set() - - -class SocketStream(abc.SocketStream): - def __init__(self, transport: asyncio.Transport, protocol: StreamProtocol): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def receive(self, max_bytes: int = 65536) -> bytes: - with self._receive_guard: - if ( - not self._protocol.read_event.is_set() - and not self._transport.is_closing() - and not self._protocol.is_at_eof - ): - self._transport.resume_reading() - await self._protocol.read_event.wait() - self._transport.pause_reading() - else: - await AsyncIOBackend.checkpoint() - - try: - chunk = self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - elif self._protocol.exception: - raise BrokenResourceError from self._protocol.exception - else: - raise EndOfStream from None - - if len(chunk) > max_bytes: - # Split the oversized chunk - chunk, leftover = chunk[:max_bytes], chunk[max_bytes:] - self._protocol.read_queue.appendleft(leftover) - - # If the read queue is empty, clear the flag so that the next call will - # block until data is available - if not self._protocol.read_queue: - self._protocol.read_event.clear() - - return chunk - - async def send(self, item: bytes) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - - if self._closed: - raise ClosedResourceError - elif self._protocol.exception is not None: - raise BrokenResourceError from self._protocol.exception - - try: - self._transport.write(item) - except RuntimeError as exc: - if self._transport.is_closing(): - raise BrokenResourceError from exc - else: - raise - - await self._protocol.write_event.wait() - - async def send_eof(self) -> None: - try: - self._transport.write_eof() - except OSError: - pass - - async def aclose(self) -> None: - self._closed = True - if not self._transport.is_closing(): - try: - self._transport.write_eof() - except OSError: - pass - - self._transport.close() - await sleep(0) - self._transport.abort() - - -class _RawSocketMixin: - _receive_future: asyncio.Future | None = None - _send_future: asyncio.Future | None = None - _closing = False - - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - def _wait_until_readable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: - def callback(f: object) -> None: - del self._receive_future - loop.remove_reader(self.__raw_socket) - - f = self._receive_future = asyncio.Future() - loop.add_reader(self.__raw_socket, f.set_result, None) - f.add_done_callback(callback) - return f - - def _wait_until_writable(self, loop: asyncio.AbstractEventLoop) -> asyncio.Future: - def callback(f: object) -> None: - del self._send_future - loop.remove_writer(self.__raw_socket) - - f = self._send_future = asyncio.Future() - loop.add_writer(self.__raw_socket, f.set_result, None) - f.add_done_callback(callback) - return f - - async def aclose(self) -> None: - if not self._closing: - self._closing = True - if self.__raw_socket.fileno() != -1: - self.__raw_socket.close() - - if self._receive_future: - self._receive_future.set_result(None) - if self._send_future: - self._send_future.set_result(None) - - -class UNIXSocketStream(_RawSocketMixin, abc.UNIXSocketStream): - async def send_eof(self) -> None: - with self._send_guard: - self._raw_socket.shutdown(socket.SHUT_WR) - - async def receive(self, max_bytes: int = 65536) -> bytes: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recv(max_bytes) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - if not data: - raise EndOfStream - - return data - - async def send(self, item: bytes) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - view = memoryview(item) - while view: - try: - bytes_sent = self._raw_socket.send(view) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - view = view[bytes_sent:] - - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - if not isinstance(msglen, int) or msglen < 0: - raise ValueError("msglen must be a non-negative integer") - if not isinstance(maxfds, int) or maxfds < 1: - raise ValueError("maxfds must be a positive integer") - - loop = get_running_loop() - fds = array.array("i") - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - message, ancdata, flags, addr = self._raw_socket.recvmsg( - msglen, socket.CMSG_LEN(maxfds * fds.itemsize) - ) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - if not message and not ancdata: - raise EndOfStream - - break - - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: - raise RuntimeError( - f"Received unexpected ancillary data; message = {message!r}, " - f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" - ) - - fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - - return message, list(fds) - - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - if not message: - raise ValueError("message must not be empty") - if not fds: - raise ValueError("fds must not be empty") - - loop = get_running_loop() - filenos: list[int] = [] - for fd in fds: - if isinstance(fd, int): - filenos.append(fd) - elif isinstance(fd, IOBase): - filenos.append(fd.fileno()) - - fdarray = array.array("i", filenos) - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - # The ignore can be removed after mypy picks up - # https://github.com/python/typeshed/pull/5545 - self._raw_socket.sendmsg( - [message], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, fdarray)] - ) - break - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - - -class TCPSocketListener(abc.SocketListener): - _accept_scope: CancelScope | None = None - _closed = False - - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._loop = cast(asyncio.BaseEventLoop, get_running_loop()) - self._accept_guard = ResourceGuard("accepting connections from") - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - async def accept(self) -> abc.SocketStream: - if self._closed: - raise ClosedResourceError - - with self._accept_guard: - await AsyncIOBackend.checkpoint() - with CancelScope() as self._accept_scope: - try: - client_sock, _addr = await self._loop.sock_accept(self._raw_socket) - except asyncio.CancelledError: - # Workaround for https://bugs.python.org/issue41317 - try: - self._loop.remove_reader(self._raw_socket) - except (ValueError, NotImplementedError): - pass - - if self._closed: - raise ClosedResourceError from None - - raise - finally: - self._accept_scope = None - - client_sock.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - transport, protocol = await self._loop.connect_accepted_socket( - StreamProtocol, client_sock - ) - return SocketStream(transport, protocol) - - async def aclose(self) -> None: - if self._closed: - return - - self._closed = True - if self._accept_scope: - # Workaround for https://bugs.python.org/issue41317 - try: - self._loop.remove_reader(self._raw_socket) - except (ValueError, NotImplementedError): - pass - - self._accept_scope.cancel() - await sleep(0) - - self._raw_socket.close() - - -class UNIXSocketListener(abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - self.__raw_socket = raw_socket - self._loop = get_running_loop() - self._accept_guard = ResourceGuard("accepting connections from") - self._closed = False - - async def accept(self) -> abc.SocketStream: - await AsyncIOBackend.checkpoint() - with self._accept_guard: - while True: - try: - client_sock, _ = self.__raw_socket.accept() - client_sock.setblocking(False) - return UNIXSocketStream(client_sock) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - self._loop.add_reader(self.__raw_socket, f.set_result, None) - f.add_done_callback( - lambda _: self._loop.remove_reader(self.__raw_socket) - ) - await f - except OSError as exc: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - - async def aclose(self) -> None: - self._closed = True - self.__raw_socket.close() - - @property - def _raw_socket(self) -> socket.socket: - return self.__raw_socket - - -class UDPSocket(abc.UDPSocket): - def __init__( - self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol - ): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def aclose(self) -> None: - self._closed = True - if not self._transport.is_closing(): - self._transport.close() - - async def receive(self) -> tuple[bytes, IPSockAddrType]: - with self._receive_guard: - await AsyncIOBackend.checkpoint() - - # If the buffer is empty, ask for more data - if not self._protocol.read_queue and not self._transport.is_closing(): - self._protocol.read_event.clear() - await self._protocol.read_event.wait() - - try: - return self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from None - - async def send(self, item: UDPPacketType) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - await self._protocol.write_event.wait() - if self._closed: - raise ClosedResourceError - elif self._transport.is_closing(): - raise BrokenResourceError - else: - self._transport.sendto(*item) - - -class ConnectedUDPSocket(abc.ConnectedUDPSocket): - def __init__( - self, transport: asyncio.DatagramTransport, protocol: DatagramProtocol - ): - self._transport = transport - self._protocol = protocol - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - self._closed = False - - @property - def _raw_socket(self) -> socket.socket: - return self._transport.get_extra_info("socket") - - async def aclose(self) -> None: - self._closed = True - if not self._transport.is_closing(): - self._transport.close() - - async def receive(self) -> bytes: - with self._receive_guard: - await AsyncIOBackend.checkpoint() - - # If the buffer is empty, ask for more data - if not self._protocol.read_queue and not self._transport.is_closing(): - self._protocol.read_event.clear() - await self._protocol.read_event.wait() - - try: - packet = self._protocol.read_queue.popleft() - except IndexError: - if self._closed: - raise ClosedResourceError from None - else: - raise BrokenResourceError from None - - return packet[0] - - async def send(self, item: bytes) -> None: - with self._send_guard: - await AsyncIOBackend.checkpoint() - await self._protocol.write_event.wait() - if self._closed: - raise ClosedResourceError - elif self._transport.is_closing(): - raise BrokenResourceError - else: - self._transport.sendto(item) - - -class UNIXDatagramSocket(_RawSocketMixin, abc.UNIXDatagramSocket): - async def receive(self) -> UNIXDatagramPacketType: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recvfrom(65536) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return data - - async def send(self, item: UNIXDatagramPacketType) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - self._raw_socket.sendto(*item) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return - - -class ConnectedUNIXDatagramSocket(_RawSocketMixin, abc.ConnectedUNIXDatagramSocket): - async def receive(self) -> bytes: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._receive_guard: - while True: - try: - data = self._raw_socket.recv(65536) - except BlockingIOError: - await self._wait_until_readable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return data - - async def send(self, item: bytes) -> None: - loop = get_running_loop() - await AsyncIOBackend.checkpoint() - with self._send_guard: - while True: - try: - self._raw_socket.send(item) - except BlockingIOError: - await self._wait_until_writable(loop) - except OSError as exc: - if self._closing: - raise ClosedResourceError from None - else: - raise BrokenResourceError from exc - else: - return - - -_read_events: RunVar[dict[int, asyncio.Future[bool]]] = RunVar("read_events") -_write_events: RunVar[dict[int, asyncio.Future[bool]]] = RunVar("write_events") - - -# -# Synchronization -# - - -class Event(BaseEvent): - def __new__(cls) -> Event: - return object.__new__(cls) - - def __init__(self) -> None: - self._event = asyncio.Event() - - def set(self) -> None: - self._event.set() - - def is_set(self) -> bool: - return self._event.is_set() - - async def wait(self) -> None: - if self.is_set(): - await AsyncIOBackend.checkpoint() - else: - await self._event.wait() - - def statistics(self) -> EventStatistics: - return EventStatistics(len(self._event._waiters)) - - -class Lock(BaseLock): - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False) -> None: - self._fast_acquire = fast_acquire - self._owner_task: asyncio.Task | None = None - self._waiters: deque[tuple[asyncio.Task, asyncio.Future]] = deque() - - async def acquire(self) -> None: - task = cast(asyncio.Task, current_task()) - if self._owner_task is None and not self._waiters: - await AsyncIOBackend.checkpoint_if_cancelled() - self._owner_task = task - - # Unless on the "fast path", yield control of the event loop so that other - # tasks can run too - if not self._fast_acquire: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except CancelledError: - self.release() - raise - - return - - if self._owner_task == task: - raise RuntimeError("Attempted to acquire an already held Lock") - - fut: asyncio.Future[None] = asyncio.Future() - item = task, fut - self._waiters.append(item) - try: - await fut - except CancelledError: - self._waiters.remove(item) - if self._owner_task is task: - self.release() - - raise - - self._waiters.remove(item) - - def acquire_nowait(self) -> None: - task = cast(asyncio.Task, current_task()) - if self._owner_task is None and not self._waiters: - self._owner_task = task - return - - if self._owner_task is task: - raise RuntimeError("Attempted to acquire an already held Lock") - - raise WouldBlock - - def locked(self) -> bool: - return self._owner_task is not None - - def release(self) -> None: - if self._owner_task != current_task(): - raise RuntimeError("The current task is not holding this lock") - - for task, fut in self._waiters: - if not fut.cancelled(): - self._owner_task = task - fut.set_result(None) - return - - self._owner_task = None - - def statistics(self) -> LockStatistics: - task_info = AsyncIOTaskInfo(self._owner_task) if self._owner_task else None - return LockStatistics(self.locked(), task_info, len(self._waiters)) - - -class Semaphore(BaseSemaphore): - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ): - super().__init__(initial_value, max_value=max_value) - self._value = initial_value - self._max_value = max_value - self._fast_acquire = fast_acquire - self._waiters: deque[asyncio.Future[None]] = deque() - - async def acquire(self) -> None: - if self._value > 0 and not self._waiters: - await AsyncIOBackend.checkpoint_if_cancelled() - self._value -= 1 - - # Unless on the "fast path", yield control of the event loop so that other - # tasks can run too - if not self._fast_acquire: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except CancelledError: - self.release() - raise - - return - - fut: asyncio.Future[None] = asyncio.Future() - self._waiters.append(fut) - try: - await fut - except CancelledError: - try: - self._waiters.remove(fut) - except ValueError: - self.release() - - raise - - def acquire_nowait(self) -> None: - if self._value == 0: - raise WouldBlock - - self._value -= 1 - - def release(self) -> None: - if self._max_value is not None and self._value == self._max_value: - raise ValueError("semaphore released too many times") - - for fut in self._waiters: - if not fut.cancelled(): - fut.set_result(None) - self._waiters.remove(fut) - return - - self._value += 1 - - @property - def value(self) -> int: - return self._value - - @property - def max_value(self) -> int | None: - return self._max_value - - def statistics(self) -> SemaphoreStatistics: - return SemaphoreStatistics(len(self._waiters)) - - -class CapacityLimiter(BaseCapacityLimiter): - _total_tokens: float = 0 - - def __new__(cls, total_tokens: float) -> CapacityLimiter: - return object.__new__(cls) - - def __init__(self, total_tokens: float): - self._borrowers: set[Any] = set() - self._wait_queue: OrderedDict[Any, asyncio.Event] = OrderedDict() - self.total_tokens = total_tokens - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - @property - def total_tokens(self) -> float: - return self._total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - if not isinstance(value, int) and not math.isinf(value): - raise TypeError("total_tokens must be an int or math.inf") - - if value < 0: - raise ValueError("total_tokens must be >= 0") - - waiters_to_notify = max(value - self._total_tokens, 0) - self._total_tokens = value - - # Notify waiting tasks that they have acquired the limiter - while self._wait_queue and waiters_to_notify: - event = self._wait_queue.popitem(last=False)[1] - event.set() - waiters_to_notify -= 1 - - @property - def borrowed_tokens(self) -> int: - return len(self._borrowers) - - @property - def available_tokens(self) -> float: - return self._total_tokens - len(self._borrowers) - - def _notify_next_waiter(self) -> None: - """Notify the next task in line if this limiter has free capacity now.""" - if self._wait_queue and len(self._borrowers) < self._total_tokens: - event = self._wait_queue.popitem(last=False)[1] - event.set() - - def acquire_nowait(self) -> None: - self.acquire_on_behalf_of_nowait(current_task()) - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - if borrower in self._borrowers: - raise RuntimeError( - "this borrower is already holding one of this CapacityLimiter's tokens" - ) - - if self._wait_queue or len(self._borrowers) >= self._total_tokens: - raise WouldBlock - - self._borrowers.add(borrower) - - async def acquire(self) -> None: - return await self.acquire_on_behalf_of(current_task()) - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await AsyncIOBackend.checkpoint_if_cancelled() - try: - self.acquire_on_behalf_of_nowait(borrower) - except WouldBlock: - event = asyncio.Event() - self._wait_queue[borrower] = event - try: - await event.wait() - except BaseException: - self._wait_queue.pop(borrower, None) - if event.is_set(): - self._notify_next_waiter() - - raise - - self._borrowers.add(borrower) - else: - try: - await AsyncIOBackend.cancel_shielded_checkpoint() - except BaseException: - self.release() - raise - - def release(self) -> None: - self.release_on_behalf_of(current_task()) - - def release_on_behalf_of(self, borrower: object) -> None: - try: - self._borrowers.remove(borrower) - except KeyError: - raise RuntimeError( - "this borrower isn't holding any of this CapacityLimiter's tokens" - ) from None - - self._notify_next_waiter() - - def statistics(self) -> CapacityLimiterStatistics: - return CapacityLimiterStatistics( - self.borrowed_tokens, - self.total_tokens, - tuple(self._borrowers), - len(self._wait_queue), - ) - - -_default_thread_limiter: RunVar[CapacityLimiter] = RunVar("_default_thread_limiter") - - -# -# Operating system signals -# - - -class _SignalReceiver: - def __init__(self, signals: tuple[Signals, ...]): - self._signals = signals - self._loop = get_running_loop() - self._signal_queue: deque[Signals] = deque() - self._future: asyncio.Future = asyncio.Future() - self._handled_signals: set[Signals] = set() - - def _deliver(self, signum: Signals) -> None: - self._signal_queue.append(signum) - if not self._future.done(): - self._future.set_result(None) - - def __enter__(self) -> _SignalReceiver: - for sig in set(self._signals): - self._loop.add_signal_handler(sig, self._deliver, sig) - self._handled_signals.add(sig) - - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - for sig in self._handled_signals: - self._loop.remove_signal_handler(sig) - - def __aiter__(self) -> _SignalReceiver: - return self - - async def __anext__(self) -> Signals: - await AsyncIOBackend.checkpoint() - if not self._signal_queue: - self._future = asyncio.Future() - await self._future - - return self._signal_queue.popleft() - - -# -# Testing and debugging -# - - -class AsyncIOTaskInfo(TaskInfo): - def __init__(self, task: asyncio.Task): - task_state = _task_states.get(task) - if task_state is None: - parent_id = None - else: - parent_id = task_state.parent_id - - coro = task.get_coro() - assert coro is not None, "created TaskInfo from a completed Task" - super().__init__(id(task), parent_id, task.get_name(), coro) - self._task = weakref.ref(task) - - def has_pending_cancellation(self) -> bool: - if not (task := self._task()): - # If the task isn't around anymore, it won't have a pending cancellation - return False - - if task._must_cancel: # type: ignore[attr-defined] - return True - elif ( - isinstance(task._fut_waiter, asyncio.Future) # type: ignore[attr-defined] - and task._fut_waiter.cancelled() # type: ignore[attr-defined] - ): - return True - - if task_state := _task_states.get(task): - if cancel_scope := task_state.cancel_scope: - return cancel_scope._effectively_cancelled - - return False - - -class TestRunner(abc.TestRunner): - _send_stream: MemoryObjectSendStream[tuple[Awaitable[Any], asyncio.Future[Any]]] - - def __init__( - self, - *, - debug: bool | None = None, - use_uvloop: bool = False, - loop_factory: Callable[[], AbstractEventLoop] | None = None, - ) -> None: - if use_uvloop and loop_factory is None: - if sys.platform != "win32": - import uvloop - - loop_factory = uvloop.new_event_loop - else: - import winloop - - loop_factory = winloop.new_event_loop - - self._runner = Runner(debug=debug, loop_factory=loop_factory) - self._exceptions: list[BaseException] = [] - self._runner_task: asyncio.Task | None = None - - def __enter__(self) -> TestRunner: - self._runner.__enter__() - self.get_loop().set_exception_handler(self._exception_handler) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._runner.__exit__(exc_type, exc_val, exc_tb) - - def get_loop(self) -> AbstractEventLoop: - return self._runner.get_loop() - - def _exception_handler( - self, loop: asyncio.AbstractEventLoop, context: dict[str, Any] - ) -> None: - if isinstance(context.get("exception"), Exception): - self._exceptions.append(context["exception"]) - else: - loop.default_exception_handler(context) - - def _raise_async_exceptions(self) -> None: - # Re-raise any exceptions raised in asynchronous callbacks - if self._exceptions: - exceptions, self._exceptions = self._exceptions, [] - if len(exceptions) == 1: - raise exceptions[0] - elif exceptions: - raise BaseExceptionGroup( - "Multiple exceptions occurred in asynchronous callbacks", exceptions - ) - - async def _run_tests_and_fixtures( - self, - receive_stream: MemoryObjectReceiveStream[ - tuple[Awaitable[T_Retval], asyncio.Future[T_Retval]] - ], - ) -> None: - from _pytest.outcomes import OutcomeException - - with receive_stream, self._send_stream: - async for coro, future in receive_stream: - try: - retval = await coro - except CancelledError as exc: - if not future.cancelled(): - future.cancel(*exc.args) - - raise - except BaseException as exc: - if not future.cancelled(): - future.set_exception(exc) - - if not isinstance(exc, (Exception, OutcomeException)): - raise - else: - if not future.cancelled(): - future.set_result(retval) - - async def _call_in_runner_task( - self, - func: Callable[P, Awaitable[T_Retval]], - /, - *args: P.args, - **kwargs: P.kwargs, - ) -> T_Retval: - if not self._runner_task: - self._send_stream, receive_stream = create_memory_object_stream[ - tuple[Awaitable[Any], asyncio.Future] - ](1) - self._runner_task = self.get_loop().create_task( - self._run_tests_and_fixtures(receive_stream) - ) - - coro = func(*args, **kwargs) - future: asyncio.Future[T_Retval] = self.get_loop().create_future() - self._send_stream.send_nowait((coro, future)) - return await future - - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], - kwargs: dict[str, Any], - ) -> Iterable[T_Retval]: - asyncgen = fixture_func(**kwargs) - fixturevalue: T_Retval = self.get_loop().run_until_complete( - self._call_in_runner_task(asyncgen.asend, None) - ) - self._raise_async_exceptions() - - yield fixturevalue - - try: - self.get_loop().run_until_complete( - self._call_in_runner_task(asyncgen.asend, None) - ) - except StopAsyncIteration: - self._raise_async_exceptions() - else: - self.get_loop().run_until_complete(asyncgen.aclose()) - raise RuntimeError("Async generator fixture did not stop") - - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], - kwargs: dict[str, Any], - ) -> T_Retval: - retval = self.get_loop().run_until_complete( - self._call_in_runner_task(fixture_func, **kwargs) - ) - self._raise_async_exceptions() - return retval - - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - try: - self.get_loop().run_until_complete( - self._call_in_runner_task(test_func, **kwargs) - ) - except Exception as exc: - self._exceptions.append(exc) - - self._raise_async_exceptions() - - -class AsyncIOBackend(AsyncBackend): - @classmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - @wraps(func) - async def wrapper() -> T_Retval: - task = cast(asyncio.Task, current_task()) - task.set_name(get_callable_name(func)) - _task_states[task] = TaskState(None, None) - - try: - return await func(*args) - finally: - del _task_states[task] - - debug = options.get("debug", None) - loop_factory = options.get("loop_factory", None) - if loop_factory is None and options.get("use_uvloop", False): - if sys.platform != "win32": - import uvloop - - loop_factory = uvloop.new_event_loop - else: - import winloop - - loop_factory = winloop.new_event_loop - - with Runner(debug=debug, loop_factory=loop_factory) as runner: - return runner.run(wrapper()) - - @classmethod - def current_token(cls) -> object: - return get_running_loop() - - @classmethod - def current_time(cls) -> float: - return get_running_loop().time() - - @classmethod - def cancelled_exception_class(cls) -> type[BaseException]: - return CancelledError - - @classmethod - async def checkpoint(cls) -> None: - await sleep(0) - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - task = current_task() - if task is None: - return - - try: - cancel_scope = _task_states[task].cancel_scope - except KeyError: - return - - while cancel_scope: - if cancel_scope.cancel_called: - await sleep(0) - elif cancel_scope.shield: - break - else: - cancel_scope = cancel_scope._parent_scope - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - with CancelScope(shield=True): - await sleep(0) - - @classmethod - async def sleep(cls, delay: float) -> None: - await sleep(delay) - - @classmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return CancelScope(deadline=deadline, shield=shield) - - @classmethod - def current_effective_deadline(cls) -> float: - if (task := current_task()) is None: - return math.inf - - try: - cancel_scope = _task_states[task].cancel_scope - except KeyError: - return math.inf - - deadline = math.inf - while cancel_scope: - deadline = min(deadline, cancel_scope.deadline) - if cancel_scope._cancel_called: - deadline = -math.inf - break - elif cancel_scope.shield: - break - else: - cancel_scope = cancel_scope._parent_scope - - return deadline - - @classmethod - def create_task_group(cls) -> abc.TaskGroup: - return TaskGroup() - - @classmethod - def create_event(cls) -> abc.Event: - return Event() - - @classmethod - def create_lock(cls, *, fast_acquire: bool) -> abc.Lock: - return Lock(fast_acquire=fast_acquire) - - @classmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> abc.Semaphore: - return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) - - @classmethod - def create_capacity_limiter(cls, total_tokens: float) -> abc.CapacityLimiter: - return CapacityLimiter(total_tokens) - - @classmethod - async def run_sync_in_worker_thread( # type: ignore[return] - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: abc.CapacityLimiter | None = None, - ) -> T_Retval: - await cls.checkpoint() - - # If this is the first run in this event loop thread, set up the necessary - # variables - try: - idle_workers = _threadpool_idle_workers.get() - workers = _threadpool_workers.get() - except LookupError: - idle_workers = deque() - workers = set() - _threadpool_idle_workers.set(idle_workers) - _threadpool_workers.set(workers) - - async with limiter or cls.current_default_thread_limiter(): - with CancelScope(shield=not abandon_on_cancel) as scope: - future = asyncio.Future[T_Retval]() - root_task = find_root_task() - if not idle_workers: - worker = WorkerThread(root_task, workers, idle_workers) - worker.start() - workers.add(worker) - root_task.add_done_callback( - worker.stop, context=contextvars.Context() - ) - else: - worker = idle_workers.pop() - - # Prune any other workers that have been idle for MAX_IDLE_TIME - # seconds or longer - now = cls.current_time() - while idle_workers: - if ( - now - idle_workers[0].idle_since - < WorkerThread.MAX_IDLE_TIME - ): - break - - expired_worker = idle_workers.popleft() - expired_worker.root_task.remove_done_callback( - expired_worker.stop - ) - expired_worker.stop() - - context = copy_context() - context.run(set_current_async_library, None) - if abandon_on_cancel or scope._parent_scope is None: - worker_scope = scope - else: - worker_scope = scope._parent_scope - - worker.queue.put_nowait((context, func, args, future, worker_scope)) - return await future - - @classmethod - def check_cancelled(cls) -> None: - scope: CancelScope | None = threadlocals.current_cancel_scope - while scope is not None: - if scope.cancel_called: - raise CancelledError(f"Cancelled by cancel scope {id(scope):x}") - - if scope.shield: - return - - scope = scope._parent_scope - - @classmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - async def task_wrapper() -> T_Retval: - __tracebackhide__ = True - if scope is not None: - task = cast(asyncio.Task, current_task()) - _task_states[task] = TaskState(None, scope) - scope._tasks.add(task) - try: - return await func(*args) - except CancelledError as exc: - raise concurrent.futures.CancelledError(str(exc)) from None - finally: - if scope is not None: - scope._tasks.discard(task) - - loop = cast( - "AbstractEventLoop", token or threadlocals.current_token.native_token - ) - if loop.is_closed(): - raise RunFinishedError - - context = copy_context() - context.run(set_current_async_library, "asyncio") - scope = getattr(threadlocals, "current_cancel_scope", None) - f: concurrent.futures.Future[T_Retval] = context.run( - asyncio.run_coroutine_threadsafe, task_wrapper(), loop=loop - ) - return f.result() - - @classmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - @wraps(func) - def wrapper() -> None: - try: - set_current_async_library("asyncio") - f.set_result(func(*args)) - except BaseException as exc: - f.set_exception(exc) - if not isinstance(exc, Exception): - raise - - loop = cast( - "AbstractEventLoop", token or threadlocals.current_token.native_token - ) - if loop.is_closed(): - raise RunFinishedError - - f: concurrent.futures.Future[T_Retval] = Future() - loop.call_soon_threadsafe(wrapper) - return f.result() - - @classmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - await cls.checkpoint() - if isinstance(command, PathLike): - command = os.fspath(command) - - if isinstance(command, (str, bytes)): - process = await asyncio.create_subprocess_shell( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - else: - process = await asyncio.create_subprocess_exec( - *command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - **kwargs, - ) - - stdin_stream = StreamWriterWrapper(process.stdin) if process.stdin else None - stdout_stream = StreamReaderWrapper(process.stdout) if process.stdout else None - stderr_stream = StreamReaderWrapper(process.stderr) if process.stderr else None - return Process(process, stdin_stream, stdout_stream, stderr_stream) - - @classmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: - create_task( - _shutdown_process_pool_on_exit(workers), - name="AnyIO process pool shutdown task", - ) - find_root_task().add_done_callback( - partial(_forcibly_shutdown_process_pool_on_exit, workers) # type:ignore[arg-type] - ) - - @classmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> abc.SocketStream: - transport, protocol = cast( - tuple[asyncio.Transport, StreamProtocol], - await get_running_loop().create_connection( - StreamProtocol, host, port, local_addr=local_address - ), - ) - transport.pause_reading() - return SocketStream(transport, protocol) - - @classmethod - async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: - await cls.checkpoint() - loop = get_running_loop() - raw_socket = socket.socket(socket.AF_UNIX) - raw_socket.setblocking(False) - while True: - try: - raw_socket.connect(path) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - loop.add_writer(raw_socket, f.set_result, None) - f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) - await f - except BaseException: - raw_socket.close() - raise - else: - return UNIXSocketStream(raw_socket) - - @classmethod - def create_tcp_listener(cls, sock: socket.socket) -> SocketListener: - return TCPSocketListener(sock) - - @classmethod - def create_unix_listener(cls, sock: socket.socket) -> SocketListener: - return UNIXSocketListener(sock) - - @classmethod - async def create_udp_socket( - cls, - family: AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, - local_addr=local_address, - remote_addr=remote_address, - family=family, - reuse_port=reuse_port, - ) - if protocol.exception: - transport.close() - raise protocol.exception - - if not remote_address: - return UDPSocket(transport, protocol) - else: - return ConnectedUDPSocket(transport, protocol) - - @classmethod - async def create_unix_datagram_socket( # type: ignore[override] - cls, raw_socket: socket.socket, remote_path: str | bytes | None - ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: - await cls.checkpoint() - loop = get_running_loop() - - if remote_path: - while True: - try: - raw_socket.connect(remote_path) - except BlockingIOError: - f: asyncio.Future = asyncio.Future() - loop.add_writer(raw_socket, f.set_result, None) - f.add_done_callback(lambda _: loop.remove_writer(raw_socket)) - await f - except BaseException: - raw_socket.close() - raise - else: - return ConnectedUNIXDatagramSocket(raw_socket) - else: - return UNIXDatagramSocket(raw_socket) - - @classmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> Sequence[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] - ]: - return await get_running_loop().getaddrinfo( - host, port, family=family, type=type, proto=proto, flags=flags - ) - - @classmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - return await get_running_loop().getnameinfo(sockaddr, flags) - - @classmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - try: - read_events = _read_events.get() - except LookupError: - read_events = {} - _read_events.set(read_events) - - fd = obj if isinstance(obj, int) else obj.fileno() - if read_events.get(fd): - raise BusyResourceError("reading from") - - loop = get_running_loop() - fut: asyncio.Future[bool] = loop.create_future() - - def cb() -> None: - try: - del read_events[fd] - except KeyError: - pass - else: - remove_reader(fd) - - try: - fut.set_result(True) - except asyncio.InvalidStateError: - pass - - try: - loop.add_reader(fd, cb) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - selector = get_selector() - selector.add_reader(fd, cb) - remove_reader = selector.remove_reader - else: - remove_reader = loop.remove_reader - - read_events[fd] = fut - try: - success = await fut - finally: - try: - del read_events[fd] - except KeyError: - pass - else: - remove_reader(fd) - - if not success: - raise ClosedResourceError - - @classmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - try: - write_events = _write_events.get() - except LookupError: - write_events = {} - _write_events.set(write_events) - - fd = obj if isinstance(obj, int) else obj.fileno() - if write_events.get(fd): - raise BusyResourceError("writing to") - - loop = get_running_loop() - fut: asyncio.Future[bool] = loop.create_future() - - def cb() -> None: - try: - del write_events[fd] - except KeyError: - pass - else: - remove_writer(fd) - - try: - fut.set_result(True) - except asyncio.InvalidStateError: - pass - - try: - loop.add_writer(fd, cb) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - selector = get_selector() - selector.add_writer(fd, cb) - remove_writer = selector.remove_writer - else: - remove_writer = loop.remove_writer - - write_events[fd] = fut - try: - success = await fut - finally: - try: - del write_events[fd] - except KeyError: - pass - else: - remove_writer(fd) - - if not success: - raise ClosedResourceError - - @classmethod - def notify_closing(cls, obj: FileDescriptorLike) -> None: - fd = obj if isinstance(obj, int) else obj.fileno() - loop = get_running_loop() - - try: - write_events = _write_events.get() - except LookupError: - pass - else: - try: - fut = write_events.pop(fd) - except KeyError: - pass - else: - try: - fut.set_result(False) - except asyncio.InvalidStateError: - pass - - try: - loop.remove_writer(fd) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - get_selector().remove_writer(fd) - - try: - read_events = _read_events.get() - except LookupError: - pass - else: - try: - fut = read_events.pop(fd) - except KeyError: - pass - else: - try: - fut.set_result(False) - except asyncio.InvalidStateError: - pass - - try: - loop.remove_reader(fd) - except NotImplementedError: - from anyio._core._asyncio_selector_thread import get_selector - - get_selector().remove_reader(fd) - - @classmethod - async def wrap_listener_socket(cls, sock: socket.socket) -> SocketListener: - return TCPSocketListener(sock) - - @classmethod - async def wrap_stream_socket(cls, sock: socket.socket) -> SocketStream: - transport, protocol = await get_running_loop().create_connection( - StreamProtocol, sock=sock - ) - return SocketStream(transport, protocol) - - @classmethod - async def wrap_unix_stream_socket(cls, sock: socket.socket) -> UNIXSocketStream: - return UNIXSocketStream(sock) - - @classmethod - async def wrap_udp_socket(cls, sock: socket.socket) -> UDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, sock=sock - ) - return UDPSocket(transport, protocol) - - @classmethod - async def wrap_connected_udp_socket(cls, sock: socket.socket) -> ConnectedUDPSocket: - transport, protocol = await get_running_loop().create_datagram_endpoint( - DatagramProtocol, sock=sock - ) - return ConnectedUDPSocket(transport, protocol) - - @classmethod - async def wrap_unix_datagram_socket(cls, sock: socket.socket) -> UNIXDatagramSocket: - return UNIXDatagramSocket(sock) - - @classmethod - async def wrap_connected_unix_datagram_socket( - cls, sock: socket.socket - ) -> ConnectedUNIXDatagramSocket: - return ConnectedUNIXDatagramSocket(sock) - - @classmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - try: - return _default_thread_limiter.get() - except LookupError: - limiter = CapacityLimiter(40) - _default_thread_limiter.set(limiter) - return limiter - - @classmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - return _SignalReceiver(signals) - - @classmethod - def get_current_task(cls) -> TaskInfo: - return AsyncIOTaskInfo(current_task()) # type: ignore[arg-type] - - @classmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - return [AsyncIOTaskInfo(task) for task in all_tasks() if not task.done()] - - @classmethod - async def wait_all_tasks_blocked(cls) -> None: - await cls.checkpoint() - this_task = current_task() - while True: - for task in all_tasks(): - if task is this_task: - continue - - waiter = task._fut_waiter # type: ignore[attr-defined] - if waiter is None or waiter.done(): - await sleep(0.1) - break - else: - return - - @classmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - return TestRunner(**options) - - -backend_class = AsyncIOBackend diff --git a/functions/python/data-processor/anyio/_backends/_trio.py b/functions/python/data-processor/anyio/_backends/_trio.py deleted file mode 100644 index b85a10a13..000000000 --- a/functions/python/data-processor/anyio/_backends/_trio.py +++ /dev/null @@ -1,1343 +0,0 @@ -from __future__ import annotations - -import array -import math -import os -import socket -import sys -import types -import weakref -from collections.abc import ( - AsyncGenerator, - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Iterable, - Sequence, -) -from contextlib import AbstractContextManager -from dataclasses import dataclass -from io import IOBase -from os import PathLike -from signal import Signals -from socket import AddressFamily, SocketKind -from types import TracebackType -from typing import ( - IO, - TYPE_CHECKING, - Any, - Generic, - NoReturn, - ParamSpec, - TypeVar, - cast, - overload, -) - -import trio.from_thread -import trio.lowlevel -from outcome import Error, Outcome, Value -from trio.lowlevel import ( - current_root_task, - current_task, - notify_closing, - wait_readable, - wait_writable, -) -from trio.socket import SocketType as TrioSocketType -from trio.to_thread import run_sync - -from .. import ( - CapacityLimiterStatistics, - EventStatistics, - LockStatistics, - RunFinishedError, - TaskInfo, - WouldBlock, - abc, -) -from .._core._eventloop import claim_worker_thread -from .._core._exceptions import ( - BrokenResourceError, - BusyResourceError, - ClosedResourceError, - EndOfStream, -) -from .._core._sockets import convert_ipv6_sockaddr -from .._core._streams import create_memory_object_stream -from .._core._synchronization import ( - CapacityLimiter as BaseCapacityLimiter, -) -from .._core._synchronization import Event as BaseEvent -from .._core._synchronization import Lock as BaseLock -from .._core._synchronization import ( - ResourceGuard, - SemaphoreStatistics, -) -from .._core._synchronization import Semaphore as BaseSemaphore -from .._core._tasks import CancelScope as BaseCancelScope -from ..abc import IPSockAddrType, UDPPacketType, UNIXDatagramPacketType -from ..abc._eventloop import AsyncBackend, StrOrBytesPath -from ..streams.memory import MemoryObjectSendStream - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from exceptiongroup import BaseExceptionGroup - from typing_extensions import TypeVarTuple, Unpack - -T = TypeVar("T") -T_Retval = TypeVar("T_Retval") -T_SockAddr = TypeVar("T_SockAddr", str, IPSockAddrType) -PosArgsT = TypeVarTuple("PosArgsT") -P = ParamSpec("P") - - -# -# Event loop -# - -RunVar = trio.lowlevel.RunVar - - -# -# Timeouts and cancellation -# - - -class CancelScope(BaseCancelScope): - def __new__( - cls, original: trio.CancelScope | None = None, **kwargs: object - ) -> CancelScope: - return object.__new__(cls) - - def __init__(self, original: trio.CancelScope | None = None, **kwargs: Any) -> None: - self.__original = original or trio.CancelScope(**kwargs) - - def __enter__(self) -> CancelScope: - self.__original.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - return self.__original.__exit__(exc_type, exc_val, exc_tb) - - def cancel(self, reason: str | None = None) -> None: - self.__original.cancel(reason) - - @property - def deadline(self) -> float: - return self.__original.deadline - - @deadline.setter - def deadline(self, value: float) -> None: - self.__original.deadline = value - - @property - def cancel_called(self) -> bool: - return self.__original.cancel_called - - @property - def cancelled_caught(self) -> bool: - return self.__original.cancelled_caught - - @property - def shield(self) -> bool: - return self.__original.shield - - @shield.setter - def shield(self, value: bool) -> None: - self.__original.shield = value - - -# -# Task groups -# - - -class TaskGroup(abc.TaskGroup): - def __init__(self) -> None: - self._active = False - self._nursery_manager = trio.open_nursery(strict_exception_groups=True) - self.cancel_scope = None # type: ignore[assignment] - - async def __aenter__(self) -> TaskGroup: - self._active = True - self._nursery = await self._nursery_manager.__aenter__() - self.cancel_scope = CancelScope(self._nursery.cancel_scope) - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - try: - # trio.Nursery.__exit__ returns bool; .open_nursery has wrong type - return await self._nursery_manager.__aexit__(exc_type, exc_val, exc_tb) # type: ignore[return-value] - except BaseExceptionGroup as exc: - if not exc.split(trio.Cancelled)[1]: - raise trio.Cancelled._create() from exc - - raise - finally: - del exc_val, exc_tb - self._active = False - - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> None: - if not self._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - self._nursery.start_soon(func, *args, name=name) - - async def start( - self, func: Callable[..., Awaitable[Any]], *args: object, name: object = None - ) -> Any: - if not self._active: - raise RuntimeError( - "This task group is not active; no new tasks can be started." - ) - - return await self._nursery.start(func, *args, name=name) - - -# -# Subprocesses -# - - -@dataclass(eq=False) -class ReceiveStreamWrapper(abc.ByteReceiveStream): - _stream: trio.abc.ReceiveStream - - async def receive(self, max_bytes: int | None = None) -> bytes: - try: - data = await self._stream.receive_some(max_bytes) - except trio.ClosedResourceError as exc: - raise ClosedResourceError from exc.__cause__ - except trio.BrokenResourceError as exc: - raise BrokenResourceError from exc.__cause__ - - if data: - return bytes(data) - else: - raise EndOfStream - - async def aclose(self) -> None: - await self._stream.aclose() - - -@dataclass(eq=False) -class SendStreamWrapper(abc.ByteSendStream): - _stream: trio.abc.SendStream - - async def send(self, item: bytes) -> None: - try: - await self._stream.send_all(item) - except trio.ClosedResourceError as exc: - raise ClosedResourceError from exc.__cause__ - except trio.BrokenResourceError as exc: - raise BrokenResourceError from exc.__cause__ - - async def aclose(self) -> None: - await self._stream.aclose() - - -@dataclass(eq=False) -class Process(abc.Process): - _process: trio.Process - _stdin: abc.ByteSendStream | None - _stdout: abc.ByteReceiveStream | None - _stderr: abc.ByteReceiveStream | None - - async def aclose(self) -> None: - with CancelScope(shield=True): - if self._stdin: - await self._stdin.aclose() - if self._stdout: - await self._stdout.aclose() - if self._stderr: - await self._stderr.aclose() - - try: - await self.wait() - except BaseException: - self.kill() - with CancelScope(shield=True): - await self.wait() - raise - - async def wait(self) -> int: - return await self._process.wait() - - def terminate(self) -> None: - self._process.terminate() - - def kill(self) -> None: - self._process.kill() - - def send_signal(self, signal: Signals) -> None: - self._process.send_signal(signal) - - @property - def pid(self) -> int: - return self._process.pid - - @property - def returncode(self) -> int | None: - return self._process.returncode - - @property - def stdin(self) -> abc.ByteSendStream | None: - return self._stdin - - @property - def stdout(self) -> abc.ByteReceiveStream | None: - return self._stdout - - @property - def stderr(self) -> abc.ByteReceiveStream | None: - return self._stderr - - -class _ProcessPoolShutdownInstrument(trio.abc.Instrument): - def after_run(self) -> None: - super().after_run() - - -current_default_worker_process_limiter: trio.lowlevel.RunVar = RunVar( - "current_default_worker_process_limiter" -) - - -async def _shutdown_process_pool(workers: set[abc.Process]) -> None: - try: - await trio.sleep(math.inf) - except trio.Cancelled: - for process in workers: - if process.returncode is None: - process.kill() - - with CancelScope(shield=True): - for process in workers: - await process.aclose() - - -# -# Sockets and networking -# - - -class _TrioSocketMixin(Generic[T_SockAddr]): - def __init__(self, trio_socket: TrioSocketType) -> None: - self._trio_socket = trio_socket - self._closed = False - - def _check_closed(self) -> None: - if self._closed: - raise ClosedResourceError - if self._trio_socket.fileno() < 0: - raise BrokenResourceError - - @property - def _raw_socket(self) -> socket.socket: - return self._trio_socket._sock # type: ignore[attr-defined] - - async def aclose(self) -> None: - if self._trio_socket.fileno() >= 0: - self._closed = True - self._trio_socket.close() - - def _convert_socket_error(self, exc: BaseException) -> NoReturn: - if isinstance(exc, trio.ClosedResourceError): - raise ClosedResourceError from exc - elif self._trio_socket.fileno() < 0 and self._closed: - raise ClosedResourceError from None - elif isinstance(exc, OSError): - raise BrokenResourceError from exc - else: - raise exc - - -class SocketStream(_TrioSocketMixin, abc.SocketStream): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self, max_bytes: int = 65536) -> bytes: - with self._receive_guard: - try: - data = await self._trio_socket.recv(max_bytes) - except BaseException as exc: - self._convert_socket_error(exc) - - if data: - return data - else: - raise EndOfStream - - async def send(self, item: bytes) -> None: - with self._send_guard: - view = memoryview(item) - while view: - try: - bytes_sent = await self._trio_socket.send(view) - except BaseException as exc: - self._convert_socket_error(exc) - - view = view[bytes_sent:] - - async def send_eof(self) -> None: - self._trio_socket.shutdown(socket.SHUT_WR) - - -class UNIXSocketStream(SocketStream, abc.UNIXSocketStream): - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - if not isinstance(msglen, int) or msglen < 0: - raise ValueError("msglen must be a non-negative integer") - if not isinstance(maxfds, int) or maxfds < 1: - raise ValueError("maxfds must be a positive integer") - - fds = array.array("i") - await trio.lowlevel.checkpoint() - with self._receive_guard: - while True: - try: - message, ancdata, flags, addr = await self._trio_socket.recvmsg( - msglen, socket.CMSG_LEN(maxfds * fds.itemsize) - ) - except BaseException as exc: - self._convert_socket_error(exc) - else: - if not message and not ancdata: - raise EndOfStream - - break - - for cmsg_level, cmsg_type, cmsg_data in ancdata: - if cmsg_level != socket.SOL_SOCKET or cmsg_type != socket.SCM_RIGHTS: - raise RuntimeError( - f"Received unexpected ancillary data; message = {message!r}, " - f"cmsg_level = {cmsg_level}, cmsg_type = {cmsg_type}" - ) - - fds.frombytes(cmsg_data[: len(cmsg_data) - (len(cmsg_data) % fds.itemsize)]) - - return message, list(fds) - - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - if not message: - raise ValueError("message must not be empty") - if not fds: - raise ValueError("fds must not be empty") - - filenos: list[int] = [] - for fd in fds: - if isinstance(fd, int): - filenos.append(fd) - elif isinstance(fd, IOBase): - filenos.append(fd.fileno()) - - fdarray = array.array("i", filenos) - await trio.lowlevel.checkpoint() - with self._send_guard: - while True: - try: - await self._trio_socket.sendmsg( - [message], - [ - ( - socket.SOL_SOCKET, - socket.SCM_RIGHTS, - fdarray, - ) - ], - ) - break - except BaseException as exc: - self._convert_socket_error(exc) - - -class TCPSocketListener(_TrioSocketMixin, abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - super().__init__(trio.socket.from_stdlib_socket(raw_socket)) - self._accept_guard = ResourceGuard("accepting connections from") - - async def accept(self) -> SocketStream: - with self._accept_guard: - try: - trio_socket, _addr = await self._trio_socket.accept() - except BaseException as exc: - self._convert_socket_error(exc) - - trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - return SocketStream(trio_socket) - - -class UNIXSocketListener(_TrioSocketMixin, abc.SocketListener): - def __init__(self, raw_socket: socket.socket): - super().__init__(trio.socket.from_stdlib_socket(raw_socket)) - self._accept_guard = ResourceGuard("accepting connections from") - - async def accept(self) -> UNIXSocketStream: - with self._accept_guard: - try: - trio_socket, _addr = await self._trio_socket.accept() - except BaseException as exc: - self._convert_socket_error(exc) - - return UNIXSocketStream(trio_socket) - - -class UDPSocket(_TrioSocketMixin[IPSockAddrType], abc.UDPSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> tuple[bytes, IPSockAddrType]: - with self._receive_guard: - try: - data, addr = await self._trio_socket.recvfrom(65536) - return data, convert_ipv6_sockaddr(addr) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: UDPPacketType) -> None: - with self._send_guard: - try: - await self._trio_socket.sendto(*item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class ConnectedUDPSocket(_TrioSocketMixin[IPSockAddrType], abc.ConnectedUDPSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> bytes: - with self._receive_guard: - try: - return await self._trio_socket.recv(65536) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: bytes) -> None: - with self._send_guard: - try: - await self._trio_socket.send(item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class UNIXDatagramSocket(_TrioSocketMixin[str], abc.UNIXDatagramSocket): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> UNIXDatagramPacketType: - with self._receive_guard: - try: - data, addr = await self._trio_socket.recvfrom(65536) - return data, addr - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: UNIXDatagramPacketType) -> None: - with self._send_guard: - try: - await self._trio_socket.sendto(*item) - except BaseException as exc: - self._convert_socket_error(exc) - - -class ConnectedUNIXDatagramSocket( - _TrioSocketMixin[str], abc.ConnectedUNIXDatagramSocket -): - def __init__(self, trio_socket: TrioSocketType) -> None: - super().__init__(trio_socket) - self._receive_guard = ResourceGuard("reading from") - self._send_guard = ResourceGuard("writing to") - - async def receive(self) -> bytes: - with self._receive_guard: - try: - return await self._trio_socket.recv(65536) - except BaseException as exc: - self._convert_socket_error(exc) - - async def send(self, item: bytes) -> None: - with self._send_guard: - try: - await self._trio_socket.send(item) - except BaseException as exc: - self._convert_socket_error(exc) - - -# -# Synchronization -# - - -class Event(BaseEvent): - def __new__(cls) -> Event: - return object.__new__(cls) - - def __init__(self) -> None: - self.__original = trio.Event() - - def is_set(self) -> bool: - return self.__original.is_set() - - async def wait(self) -> None: - return await self.__original.wait() - - def statistics(self) -> EventStatistics: - orig_statistics = self.__original.statistics() - return EventStatistics(tasks_waiting=orig_statistics.tasks_waiting) - - def set(self) -> None: - self.__original.set() - - -class Lock(BaseLock): - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False) -> None: - self._fast_acquire = fast_acquire - self.__original = trio.Lock() - - @staticmethod - def _convert_runtime_error_msg(exc: RuntimeError) -> None: - if exc.args == ("attempt to re-acquire an already held Lock",): - exc.args = ("Attempted to acquire an already held Lock",) - - async def acquire(self) -> None: - if not self._fast_acquire: - try: - await self.__original.acquire() - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - return - - # This is the "fast path" where we don't let other tasks run - await trio.lowlevel.checkpoint_if_cancelled() - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - await self.__original._lot.park() - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - except RuntimeError as exc: - self._convert_runtime_error_msg(exc) - raise - - def locked(self) -> bool: - return self.__original.locked() - - def release(self) -> None: - self.__original.release() - - def statistics(self) -> LockStatistics: - orig_statistics = self.__original.statistics() - owner = TrioTaskInfo(orig_statistics.owner) if orig_statistics.owner else None - return LockStatistics( - orig_statistics.locked, owner, orig_statistics.tasks_waiting - ) - - -class Semaphore(BaseSemaphore): - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> None: - super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) - self.__original = trio.Semaphore(initial_value, max_value=max_value) - - async def acquire(self) -> None: - if not self._fast_acquire: - await self.__original.acquire() - return - - # This is the "fast path" where we don't let other tasks run - await trio.lowlevel.checkpoint_if_cancelled() - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - await self.__original._lot.park() - - def acquire_nowait(self) -> None: - try: - self.__original.acquire_nowait() - except trio.WouldBlock: - raise WouldBlock from None - - @property - def max_value(self) -> int | None: - return self.__original.max_value - - @property - def value(self) -> int: - return self.__original.value - - def release(self) -> None: - self.__original.release() - - def statistics(self) -> SemaphoreStatistics: - orig_statistics = self.__original.statistics() - return SemaphoreStatistics(orig_statistics.tasks_waiting) - - -class CapacityLimiter(BaseCapacityLimiter): - def __new__( - cls, - total_tokens: float | None = None, - *, - original: trio.CapacityLimiter | None = None, - ) -> CapacityLimiter: - return object.__new__(cls) - - def __init__( - self, - total_tokens: float | None = None, - *, - original: trio.CapacityLimiter | None = None, - ) -> None: - if original is not None: - self.__original = original - else: - assert total_tokens is not None - self.__original = trio.CapacityLimiter(total_tokens) - - async def __aenter__(self) -> None: - return await self.__original.__aenter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - await self.__original.__aexit__(exc_type, exc_val, exc_tb) - - @property - def total_tokens(self) -> float: - return self.__original.total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - self.__original.total_tokens = value - - @property - def borrowed_tokens(self) -> int: - return self.__original.borrowed_tokens - - @property - def available_tokens(self) -> float: - return self.__original.available_tokens - - def acquire_nowait(self) -> None: - self.__original.acquire_nowait() - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - self.__original.acquire_on_behalf_of_nowait(borrower) - - async def acquire(self) -> None: - await self.__original.acquire() - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await self.__original.acquire_on_behalf_of(borrower) - - def release(self) -> None: - return self.__original.release() - - def release_on_behalf_of(self, borrower: object) -> None: - return self.__original.release_on_behalf_of(borrower) - - def statistics(self) -> CapacityLimiterStatistics: - orig = self.__original.statistics() - return CapacityLimiterStatistics( - borrowed_tokens=orig.borrowed_tokens, - total_tokens=orig.total_tokens, - borrowers=tuple(orig.borrowers), - tasks_waiting=orig.tasks_waiting, - ) - - -_capacity_limiter_wrapper: trio.lowlevel.RunVar = RunVar("_capacity_limiter_wrapper") - - -# -# Signal handling -# - - -class _SignalReceiver: - _iterator: AsyncIterator[int] - - def __init__(self, signals: tuple[Signals, ...]): - self._signals = signals - - def __enter__(self) -> _SignalReceiver: - self._cm = trio.open_signal_receiver(*self._signals) - self._iterator = self._cm.__enter__() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool | None: - return self._cm.__exit__(exc_type, exc_val, exc_tb) - - def __aiter__(self) -> _SignalReceiver: - return self - - async def __anext__(self) -> Signals: - signum = await self._iterator.__anext__() - return Signals(signum) - - -# -# Testing and debugging -# - - -class TestRunner(abc.TestRunner): - def __init__(self, **options: Any) -> None: - from queue import Queue - - self._call_queue: Queue[Callable[[], object]] = Queue() - self._send_stream: MemoryObjectSendStream | None = None - self._options = options - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: types.TracebackType | None, - ) -> None: - if self._send_stream: - self._send_stream.close() - while self._send_stream is not None: - self._call_queue.get()() - - async def _run_tests_and_fixtures(self) -> None: - self._send_stream, receive_stream = create_memory_object_stream(1) - with receive_stream: - async for coro, outcome_holder in receive_stream: - try: - retval = await coro - except BaseException as exc: - outcome_holder.append(Error(exc)) - else: - outcome_holder.append(Value(retval)) - - def _main_task_finished(self, outcome: object) -> None: - self._send_stream = None - - def _call_in_runner_task( - self, - func: Callable[P, Awaitable[T_Retval]], - /, - *args: P.args, - **kwargs: P.kwargs, - ) -> T_Retval: - if self._send_stream is None: - trio.lowlevel.start_guest_run( - self._run_tests_and_fixtures, - run_sync_soon_threadsafe=self._call_queue.put, - done_callback=self._main_task_finished, - **self._options, - ) - while self._send_stream is None: - self._call_queue.get()() - - outcome_holder: list[Outcome] = [] - self._send_stream.send_nowait((func(*args, **kwargs), outcome_holder)) - while not outcome_holder: - self._call_queue.get()() - - return outcome_holder[0].unwrap() - - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[T_Retval, Any]], - kwargs: dict[str, Any], - ) -> Iterable[T_Retval]: - asyncgen = fixture_func(**kwargs) - fixturevalue: T_Retval = self._call_in_runner_task(asyncgen.asend, None) - - yield fixturevalue - - try: - self._call_in_runner_task(asyncgen.asend, None) - except StopAsyncIteration: - pass - else: - self._call_in_runner_task(asyncgen.aclose) - raise RuntimeError("Async generator fixture did not stop") - - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, T_Retval]], - kwargs: dict[str, Any], - ) -> T_Retval: - return self._call_in_runner_task(fixture_func, **kwargs) - - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - self._call_in_runner_task(test_func, **kwargs) - - -class TrioTaskInfo(TaskInfo): - def __init__(self, task: trio.lowlevel.Task): - parent_id = None - if task.parent_nursery and task.parent_nursery.parent_task: - parent_id = id(task.parent_nursery.parent_task) - - super().__init__(id(task), parent_id, task.name, task.coro) - self._task = weakref.proxy(task) - - def has_pending_cancellation(self) -> bool: - try: - return self._task._cancel_status.effectively_cancelled - except ReferenceError: - # If the task is no longer around, it surely doesn't have a cancellation - # pending - return False - - -class TrioBackend(AsyncBackend): - @classmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - return trio.run(func, *args) - - @classmethod - def current_token(cls) -> object: - return trio.lowlevel.current_trio_token() - - @classmethod - def current_time(cls) -> float: - return trio.current_time() - - @classmethod - def cancelled_exception_class(cls) -> type[BaseException]: - return trio.Cancelled - - @classmethod - async def checkpoint(cls) -> None: - await trio.lowlevel.checkpoint() - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - await trio.lowlevel.checkpoint_if_cancelled() - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - await trio.lowlevel.cancel_shielded_checkpoint() - - @classmethod - async def sleep(cls, delay: float) -> None: - await trio.sleep(delay) - - @classmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> abc.CancelScope: - return CancelScope(deadline=deadline, shield=shield) - - @classmethod - def current_effective_deadline(cls) -> float: - return trio.current_effective_deadline() - - @classmethod - def create_task_group(cls) -> abc.TaskGroup: - return TaskGroup() - - @classmethod - def create_event(cls) -> abc.Event: - return Event() - - @classmethod - def create_lock(cls, *, fast_acquire: bool) -> Lock: - return Lock(fast_acquire=fast_acquire) - - @classmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> abc.Semaphore: - return Semaphore(initial_value, max_value=max_value, fast_acquire=fast_acquire) - - @classmethod - def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: - return CapacityLimiter(total_tokens) - - @classmethod - async def run_sync_in_worker_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: abc.CapacityLimiter | None = None, - ) -> T_Retval: - def wrapper() -> T_Retval: - with claim_worker_thread(TrioBackend, token): - return func(*args) - - token = TrioBackend.current_token() - return await run_sync( - wrapper, - abandon_on_cancel=abandon_on_cancel, - limiter=cast(trio.CapacityLimiter, limiter), - ) - - @classmethod - def check_cancelled(cls) -> None: - trio.from_thread.check_cancelled() - - @classmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - trio_token = cast("trio.lowlevel.TrioToken | None", token) - try: - return trio.from_thread.run(func, *args, trio_token=trio_token) - except trio.RunFinishedError: - raise RunFinishedError from None - - @classmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - trio_token = cast("trio.lowlevel.TrioToken | None", token) - try: - return trio.from_thread.run_sync(func, *args, trio_token=trio_token) - except trio.RunFinishedError: - raise RunFinishedError from None - - @classmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - def convert_item(item: StrOrBytesPath) -> str: - str_or_bytes = os.fspath(item) - if isinstance(str_or_bytes, str): - return str_or_bytes - else: - return os.fsdecode(str_or_bytes) - - if isinstance(command, (str, bytes, PathLike)): - process = await trio.lowlevel.open_process( - convert_item(command), - stdin=stdin, - stdout=stdout, - stderr=stderr, - shell=True, - **kwargs, - ) - else: - process = await trio.lowlevel.open_process( - [convert_item(item) for item in command], - stdin=stdin, - stdout=stdout, - stderr=stderr, - shell=False, - **kwargs, - ) - - stdin_stream = SendStreamWrapper(process.stdin) if process.stdin else None - stdout_stream = ReceiveStreamWrapper(process.stdout) if process.stdout else None - stderr_stream = ReceiveStreamWrapper(process.stderr) if process.stderr else None - return Process(process, stdin_stream, stdout_stream, stderr_stream) - - @classmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[abc.Process]) -> None: - trio.lowlevel.spawn_system_task(_shutdown_process_pool, workers) - - @classmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> SocketStream: - family = socket.AF_INET6 if ":" in host else socket.AF_INET - trio_socket = trio.socket.socket(family) - trio_socket.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - if local_address: - await trio_socket.bind(local_address) - - try: - await trio_socket.connect((host, port)) - except BaseException: - trio_socket.close() - raise - - return SocketStream(trio_socket) - - @classmethod - async def connect_unix(cls, path: str | bytes) -> abc.UNIXSocketStream: - trio_socket = trio.socket.socket(socket.AF_UNIX) - try: - await trio_socket.connect(path) - except BaseException: - trio_socket.close() - raise - - return UNIXSocketStream(trio_socket) - - @classmethod - def create_tcp_listener(cls, sock: socket.socket) -> abc.SocketListener: - return TCPSocketListener(sock) - - @classmethod - def create_unix_listener(cls, sock: socket.socket) -> abc.SocketListener: - return UNIXSocketListener(sock) - - @classmethod - async def create_udp_socket( - cls, - family: socket.AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - trio_socket = trio.socket.socket(family=family, type=socket.SOCK_DGRAM) - - if reuse_port: - trio_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - - if local_address: - await trio_socket.bind(local_address) - - if remote_address: - await trio_socket.connect(remote_address) - return ConnectedUDPSocket(trio_socket) - else: - return UDPSocket(trio_socket) - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: None - ) -> abc.UNIXDatagramSocket: ... - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: str | bytes - ) -> abc.ConnectedUNIXDatagramSocket: ... - - @classmethod - async def create_unix_datagram_socket( - cls, raw_socket: socket.socket, remote_path: str | bytes | None - ) -> abc.UNIXDatagramSocket | abc.ConnectedUNIXDatagramSocket: - trio_socket = trio.socket.from_stdlib_socket(raw_socket) - - if remote_path: - await trio_socket.connect(remote_path) - return ConnectedUNIXDatagramSocket(trio_socket) - else: - return UNIXDatagramSocket(trio_socket) - - @classmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> Sequence[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] - ]: - return await trio.socket.getaddrinfo(host, port, family, type, proto, flags) - - @classmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - return await trio.socket.getnameinfo(sockaddr, flags) - - @classmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - try: - await wait_readable(obj) - except trio.ClosedResourceError as exc: - raise ClosedResourceError().with_traceback(exc.__traceback__) from None - except trio.BusyResourceError: - raise BusyResourceError("reading from") from None - - @classmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - try: - await wait_writable(obj) - except trio.ClosedResourceError as exc: - raise ClosedResourceError().with_traceback(exc.__traceback__) from None - except trio.BusyResourceError: - raise BusyResourceError("writing to") from None - - @classmethod - def notify_closing(cls, obj: FileDescriptorLike) -> None: - notify_closing(obj) - - @classmethod - async def wrap_listener_socket(cls, sock: socket.socket) -> abc.SocketListener: - return TCPSocketListener(sock) - - @classmethod - async def wrap_stream_socket(cls, sock: socket.socket) -> SocketStream: - trio_sock = trio.socket.from_stdlib_socket(sock) - return SocketStream(trio_sock) - - @classmethod - async def wrap_unix_stream_socket(cls, sock: socket.socket) -> UNIXSocketStream: - trio_sock = trio.socket.from_stdlib_socket(sock) - return UNIXSocketStream(trio_sock) - - @classmethod - async def wrap_udp_socket(cls, sock: socket.socket) -> UDPSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return UDPSocket(trio_sock) - - @classmethod - async def wrap_connected_udp_socket(cls, sock: socket.socket) -> ConnectedUDPSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return ConnectedUDPSocket(trio_sock) - - @classmethod - async def wrap_unix_datagram_socket(cls, sock: socket.socket) -> UNIXDatagramSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return UNIXDatagramSocket(trio_sock) - - @classmethod - async def wrap_connected_unix_datagram_socket( - cls, sock: socket.socket - ) -> ConnectedUNIXDatagramSocket: - trio_sock = trio.socket.from_stdlib_socket(sock) - return ConnectedUNIXDatagramSocket(trio_sock) - - @classmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - try: - return _capacity_limiter_wrapper.get() - except LookupError: - limiter = CapacityLimiter( - original=trio.to_thread.current_default_thread_limiter() - ) - _capacity_limiter_wrapper.set(limiter) - return limiter - - @classmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - return _SignalReceiver(signals) - - @classmethod - def get_current_task(cls) -> TaskInfo: - task = current_task() - return TrioTaskInfo(task) - - @classmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - root_task = current_root_task() - assert root_task - task_infos = [TrioTaskInfo(root_task)] - nurseries = root_task.child_nurseries - while nurseries: - new_nurseries: list[trio.Nursery] = [] - for nursery in nurseries: - for task in nursery.child_tasks: - task_infos.append(TrioTaskInfo(task)) - new_nurseries.extend(task.child_nurseries) - - nurseries = new_nurseries - - return task_infos - - @classmethod - async def wait_all_tasks_blocked(cls) -> None: - from trio.testing import wait_all_tasks_blocked - - await wait_all_tasks_blocked() - - @classmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - return TestRunner(**options) - - -backend_class = TrioBackend diff --git a/functions/python/data-processor/anyio/_core/__init__.py b/functions/python/data-processor/anyio/_core/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/anyio/_core/_asyncio_selector_thread.py b/functions/python/data-processor/anyio/_core/_asyncio_selector_thread.py deleted file mode 100644 index 9f35bae56..000000000 --- a/functions/python/data-processor/anyio/_core/_asyncio_selector_thread.py +++ /dev/null @@ -1,167 +0,0 @@ -from __future__ import annotations - -import asyncio -import socket -import threading -from collections.abc import Callable -from selectors import EVENT_READ, EVENT_WRITE, DefaultSelector -from typing import TYPE_CHECKING, Any - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - -_selector_lock = threading.Lock() -_selector: Selector | None = None - - -class Selector: - def __init__(self) -> None: - self._thread = threading.Thread(target=self.run, name="AnyIO socket selector") - self._selector = DefaultSelector() - self._send, self._receive = socket.socketpair() - self._send.setblocking(False) - self._receive.setblocking(False) - # This somewhat reduces the amount of memory wasted queueing up data - # for wakeups. With these settings, maximum number of 1-byte sends - # before getting BlockingIOError: - # Linux 4.8: 6 - # macOS (darwin 15.5): 1 - # Windows 10: 525347 - # Windows you're weird. (And on Windows setting SNDBUF to 0 makes send - # blocking, even on non-blocking sockets, so don't do that.) - self._receive.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, 1) - self._send.setsockopt(socket.SOL_SOCKET, socket.SO_SNDBUF, 1) - # On Windows this is a TCP socket so this might matter. On other - # platforms this fails b/c AF_UNIX sockets aren't actually TCP. - try: - self._send.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) - except OSError: - pass - - self._selector.register(self._receive, EVENT_READ) - self._closed = False - - def start(self) -> None: - self._thread.start() - threading._register_atexit(self._stop) # type: ignore[attr-defined] - - def _stop(self) -> None: - global _selector - self._closed = True - self._notify_self() - self._send.close() - self._thread.join() - self._selector.unregister(self._receive) - self._receive.close() - self._selector.close() - _selector = None - assert not self._selector.get_map(), ( - "selector still has registered file descriptors after shutdown" - ) - - def _notify_self(self) -> None: - try: - self._send.send(b"\x00") - except BlockingIOError: - pass - - def add_reader(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: - loop = asyncio.get_running_loop() - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, EVENT_READ, {EVENT_READ: (loop, callback)}) - else: - if EVENT_READ in key.data: - raise ValueError( - "this file descriptor is already registered for reading" - ) - - key.data[EVENT_READ] = loop, callback - self._selector.modify(fd, key.events | EVENT_READ, key.data) - - self._notify_self() - - def add_writer(self, fd: FileDescriptorLike, callback: Callable[[], Any]) -> None: - loop = asyncio.get_running_loop() - try: - key = self._selector.get_key(fd) - except KeyError: - self._selector.register(fd, EVENT_WRITE, {EVENT_WRITE: (loop, callback)}) - else: - if EVENT_WRITE in key.data: - raise ValueError( - "this file descriptor is already registered for writing" - ) - - key.data[EVENT_WRITE] = loop, callback - self._selector.modify(fd, key.events | EVENT_WRITE, key.data) - - self._notify_self() - - def remove_reader(self, fd: FileDescriptorLike) -> bool: - try: - key = self._selector.get_key(fd) - except KeyError: - return False - - if new_events := key.events ^ EVENT_READ: - del key.data[EVENT_READ] - self._selector.modify(fd, new_events, key.data) - else: - self._selector.unregister(fd) - - return True - - def remove_writer(self, fd: FileDescriptorLike) -> bool: - try: - key = self._selector.get_key(fd) - except KeyError: - return False - - if new_events := key.events ^ EVENT_WRITE: - del key.data[EVENT_WRITE] - self._selector.modify(fd, new_events, key.data) - else: - self._selector.unregister(fd) - - return True - - def run(self) -> None: - while not self._closed: - for key, events in self._selector.select(): - if key.fileobj is self._receive: - try: - while self._receive.recv(4096): - pass - except BlockingIOError: - pass - - continue - - if events & EVENT_READ: - loop, callback = key.data[EVENT_READ] - self.remove_reader(key.fd) - try: - loop.call_soon_threadsafe(callback) - except RuntimeError: - pass # the loop was already closed - - if events & EVENT_WRITE: - loop, callback = key.data[EVENT_WRITE] - self.remove_writer(key.fd) - try: - loop.call_soon_threadsafe(callback) - except RuntimeError: - pass # the loop was already closed - - -def get_selector() -> Selector: - global _selector - - with _selector_lock: - if _selector is None: - _selector = Selector() - _selector.start() - - return _selector diff --git a/functions/python/data-processor/anyio/_core/_contextmanagers.py b/functions/python/data-processor/anyio/_core/_contextmanagers.py deleted file mode 100644 index 302f32b0c..000000000 --- a/functions/python/data-processor/anyio/_core/_contextmanagers.py +++ /dev/null @@ -1,200 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from contextlib import AbstractAsyncContextManager, AbstractContextManager -from inspect import isasyncgen, iscoroutine, isgenerator -from types import TracebackType -from typing import Protocol, TypeVar, cast, final - -_T_co = TypeVar("_T_co", covariant=True) -_ExitT_co = TypeVar("_ExitT_co", covariant=True, bound="bool | None") - - -class _SupportsCtxMgr(Protocol[_T_co, _ExitT_co]): - def __contextmanager__(self) -> AbstractContextManager[_T_co, _ExitT_co]: ... - - -class _SupportsAsyncCtxMgr(Protocol[_T_co, _ExitT_co]): - def __asynccontextmanager__( - self, - ) -> AbstractAsyncContextManager[_T_co, _ExitT_co]: ... - - -class ContextManagerMixin: - """ - Mixin class providing context manager functionality via a generator-based - implementation. - - This class allows you to implement a context manager via :meth:`__contextmanager__` - which should return a generator. The mechanics are meant to mirror those of - :func:`@contextmanager `. - - .. note:: Classes using this mix-in are not reentrant as context managers, meaning - that once you enter it, you can't re-enter before first exiting it. - - .. seealso:: :doc:`contextmanagers` - """ - - __cm: AbstractContextManager[object, bool | None] | None = None - - @final - def __enter__(self: _SupportsCtxMgr[_T_co, bool | None]) -> _T_co: - # Needed for mypy to assume self still has the __cm member - assert isinstance(self, ContextManagerMixin) - if self.__cm is not None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has already been entered" - ) - - cm = self.__contextmanager__() - if not isinstance(cm, AbstractContextManager): - if isgenerator(cm): - raise TypeError( - "__contextmanager__() returned a generator object instead of " - "a context manager. Did you forget to add the @contextmanager " - "decorator?" - ) - - raise TypeError( - f"__contextmanager__() did not return a context manager object, " - f"but {cm.__class__!r}" - ) - - if cm is self: - raise TypeError( - f"{self.__class__.__qualname__}.__contextmanager__() returned " - f"self. Did you forget to add the @contextmanager decorator and a " - f"'yield' statement?" - ) - - value = cm.__enter__() - self.__cm = cm - return value - - @final - def __exit__( - self: _SupportsCtxMgr[object, _ExitT_co], - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> _ExitT_co: - # Needed for mypy to assume self still has the __cm member - assert isinstance(self, ContextManagerMixin) - if self.__cm is None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has not been entered yet" - ) - - # Prevent circular references - cm = self.__cm - del self.__cm - - return cast(_ExitT_co, cm.__exit__(exc_type, exc_val, exc_tb)) - - @abstractmethod - def __contextmanager__(self) -> AbstractContextManager[object, bool | None]: - """ - Implement your context manager logic here. - - This method **must** be decorated with - :func:`@contextmanager `. - - .. note:: Remember that the ``yield`` will raise any exception raised in the - enclosed context block, so use a ``finally:`` block to clean up resources! - - :return: a context manager object - """ - - -class AsyncContextManagerMixin: - """ - Mixin class providing async context manager functionality via a generator-based - implementation. - - This class allows you to implement a context manager via - :meth:`__asynccontextmanager__`. The mechanics are meant to mirror those of - :func:`@asynccontextmanager `. - - .. note:: Classes using this mix-in are not reentrant as context managers, meaning - that once you enter it, you can't re-enter before first exiting it. - - .. seealso:: :doc:`contextmanagers` - """ - - __cm: AbstractAsyncContextManager[object, bool | None] | None = None - - @final - async def __aenter__(self: _SupportsAsyncCtxMgr[_T_co, bool | None]) -> _T_co: - # Needed for mypy to assume self still has the __cm member - assert isinstance(self, AsyncContextManagerMixin) - if self.__cm is not None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has already been entered" - ) - - cm = self.__asynccontextmanager__() - if not isinstance(cm, AbstractAsyncContextManager): - if isasyncgen(cm): - raise TypeError( - "__asynccontextmanager__() returned an async generator instead of " - "an async context manager. Did you forget to add the " - "@asynccontextmanager decorator?" - ) - elif iscoroutine(cm): - cm.close() - raise TypeError( - "__asynccontextmanager__() returned a coroutine object instead of " - "an async context manager. Did you forget to add the " - "@asynccontextmanager decorator and a 'yield' statement?" - ) - - raise TypeError( - f"__asynccontextmanager__() did not return an async context manager, " - f"but {cm.__class__!r}" - ) - - if cm is self: - raise TypeError( - f"{self.__class__.__qualname__}.__asynccontextmanager__() returned " - f"self. Did you forget to add the @asynccontextmanager decorator and a " - f"'yield' statement?" - ) - - value = await cm.__aenter__() - self.__cm = cm - return value - - @final - async def __aexit__( - self: _SupportsAsyncCtxMgr[object, _ExitT_co], - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> _ExitT_co: - assert isinstance(self, AsyncContextManagerMixin) - if self.__cm is None: - raise RuntimeError( - f"this {self.__class__.__qualname__} has not been entered yet" - ) - - # Prevent circular references - cm = self.__cm - del self.__cm - - return cast(_ExitT_co, await cm.__aexit__(exc_type, exc_val, exc_tb)) - - @abstractmethod - def __asynccontextmanager__( - self, - ) -> AbstractAsyncContextManager[object, bool | None]: - """ - Implement your async context manager logic here. - - This method **must** be decorated with - :func:`@asynccontextmanager `. - - .. note:: Remember that the ``yield`` will raise any exception raised in the - enclosed context block, so use a ``finally:`` block to clean up resources! - - :return: an async context manager object - """ diff --git a/functions/python/data-processor/anyio/_core/_eventloop.py b/functions/python/data-processor/anyio/_core/_eventloop.py deleted file mode 100644 index 59a69ccdf..000000000 --- a/functions/python/data-processor/anyio/_core/_eventloop.py +++ /dev/null @@ -1,234 +0,0 @@ -from __future__ import annotations - -import math -import sys -import threading -from collections.abc import Awaitable, Callable, Generator -from contextlib import contextmanager -from contextvars import Token -from importlib import import_module -from typing import TYPE_CHECKING, Any, TypeVar - -from ._exceptions import NoEventLoopError - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -sniffio: Any -try: - import sniffio -except ModuleNotFoundError: - sniffio = None - -if TYPE_CHECKING: - from ..abc import AsyncBackend - -# This must be updated when new backends are introduced -BACKENDS = "asyncio", "trio" - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -threadlocals = threading.local() -loaded_backends: dict[str, type[AsyncBackend]] = {} - - -def run( - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - backend: str = "asyncio", - backend_options: dict[str, Any] | None = None, -) -> T_Retval: - """ - Run the given coroutine function in an asynchronous event loop. - - The current thread must not be already running an event loop. - - :param func: a coroutine function - :param args: positional arguments to ``func`` - :param backend: name of the asynchronous event loop implementation – currently - either ``asyncio`` or ``trio`` - :param backend_options: keyword arguments to call the backend ``run()`` - implementation with (documented :ref:`here `) - :return: the return value of the coroutine function - :raises RuntimeError: if an asynchronous event loop is already running in this - thread - :raises LookupError: if the named backend is not found - - """ - if asynclib_name := current_async_library(): - raise RuntimeError(f"Already running {asynclib_name} in this thread") - - try: - async_backend = get_async_backend(backend) - except ImportError as exc: - raise LookupError(f"No such backend: {backend}") from exc - - token = None - if asynclib_name is None: - # Since we're in control of the event loop, we can cache the name of the async - # library - token = set_current_async_library(backend) - - try: - backend_options = backend_options or {} - return async_backend.run(func, args, {}, backend_options) - finally: - reset_current_async_library(token) - - -async def sleep(delay: float) -> None: - """ - Pause the current task for the specified duration. - - :param delay: the duration, in seconds - - """ - return await get_async_backend().sleep(delay) - - -async def sleep_forever() -> None: - """ - Pause the current task until it's cancelled. - - This is a shortcut for ``sleep(math.inf)``. - - .. versionadded:: 3.1 - - """ - await sleep(math.inf) - - -async def sleep_until(deadline: float) -> None: - """ - Pause the current task until the given time. - - :param deadline: the absolute time to wake up at (according to the internal - monotonic clock of the event loop) - - .. versionadded:: 3.1 - - """ - now = current_time() - await sleep(max(deadline - now, 0)) - - -def current_time() -> float: - """ - Return the current value of the event loop's internal clock. - - :return: the clock value (seconds) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().current_time() - - -def get_all_backends() -> tuple[str, ...]: - """Return a tuple of the names of all built-in backends.""" - return BACKENDS - - -def get_available_backends() -> tuple[str, ...]: - """ - Test for the availability of built-in backends. - - :return a tuple of the built-in backend names that were successfully imported - - .. versionadded:: 4.12 - - """ - available_backends: list[str] = [] - for backend_name in get_all_backends(): - try: - get_async_backend(backend_name) - except ImportError: - continue - - available_backends.append(backend_name) - - return tuple(available_backends) - - -def get_cancelled_exc_class() -> type[BaseException]: - """ - Return the current async library's cancellation exception class. - - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().cancelled_exception_class() - - -# -# Private API -# - - -@contextmanager -def claim_worker_thread( - backend_class: type[AsyncBackend], token: object -) -> Generator[Any, None, None]: - from ..lowlevel import EventLoopToken - - threadlocals.current_token = EventLoopToken(backend_class, token) - try: - yield - finally: - del threadlocals.current_token - - -def get_async_backend(asynclib_name: str | None = None) -> type[AsyncBackend]: - if asynclib_name is None: - asynclib_name = current_async_library() - if not asynclib_name: - raise NoEventLoopError( - f"Not currently running on any asynchronous event loop. " - f"Available async backends: {', '.join(get_all_backends())}" - ) - - # We use our own dict instead of sys.modules to get the already imported back-end - # class because the appropriate modules in sys.modules could potentially be only - # partially initialized - try: - return loaded_backends[asynclib_name] - except KeyError: - module = import_module(f"anyio._backends._{asynclib_name}") - loaded_backends[asynclib_name] = module.backend_class - return module.backend_class - - -def current_async_library() -> str | None: - if sniffio is None: - # If sniffio is not installed, we assume we're either running asyncio or nothing - import asyncio - - try: - asyncio.get_running_loop() - return "asyncio" - except RuntimeError: - pass - else: - try: - return sniffio.current_async_library() - except sniffio.AsyncLibraryNotFoundError: - pass - - return None - - -def set_current_async_library(asynclib_name: str | None) -> Token | None: - # no-op if sniffio is not installed - if sniffio is None: - return None - - return sniffio.current_async_library_cvar.set(asynclib_name) - - -def reset_current_async_library(token: Token | None) -> None: - if token is not None: - sniffio.current_async_library_cvar.reset(token) diff --git a/functions/python/data-processor/anyio/_core/_exceptions.py b/functions/python/data-processor/anyio/_core/_exceptions.py deleted file mode 100644 index 3776bedcd..000000000 --- a/functions/python/data-processor/anyio/_core/_exceptions.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Generator -from textwrap import dedent -from typing import Any - -if sys.version_info < (3, 11): - from exceptiongroup import BaseExceptionGroup - - -class BrokenResourceError(Exception): - """ - Raised when trying to use a resource that has been rendered unusable due to external - causes (e.g. a send stream whose peer has disconnected). - """ - - -class BrokenWorkerProcess(Exception): - """ - Raised by :meth:`~anyio.to_process.run_sync` if the worker process terminates abruptly or - otherwise misbehaves. - """ - - -class BrokenWorkerInterpreter(Exception): - """ - Raised by :meth:`~anyio.to_interpreter.run_sync` if an unexpected exception is - raised in the subinterpreter. - """ - - def __init__(self, excinfo: Any): - # This was adapted from concurrent.futures.interpreter.ExecutionFailed - msg = excinfo.formatted - if not msg: - if excinfo.type and excinfo.msg: - msg = f"{excinfo.type.__name__}: {excinfo.msg}" - else: - msg = excinfo.type.__name__ or excinfo.msg - - super().__init__(msg) - self.excinfo = excinfo - - def __str__(self) -> str: - try: - formatted = self.excinfo.errdisplay - except Exception: - return super().__str__() - else: - return dedent( - f""" - {super().__str__()} - - Uncaught in the interpreter: - - {formatted} - """.strip() - ) - - -class BusyResourceError(Exception): - """ - Raised when two tasks are trying to read from or write to the same resource - concurrently. - """ - - def __init__(self, action: str): - super().__init__(f"Another task is already {action} this resource") - - -class ClosedResourceError(Exception): - """Raised when trying to use a resource that has been closed.""" - - -class ConnectionFailed(OSError): - """ - Raised when a connection attempt fails. - - .. note:: This class inherits from :exc:`OSError` for backwards compatibility. - """ - - -def iterate_exceptions( - exception: BaseException, -) -> Generator[BaseException, None, None]: - if isinstance(exception, BaseExceptionGroup): - for exc in exception.exceptions: - yield from iterate_exceptions(exc) - else: - yield exception - - -class DelimiterNotFound(Exception): - """ - Raised during - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the - maximum number of bytes has been read without the delimiter being found. - """ - - def __init__(self, max_bytes: int) -> None: - super().__init__( - f"The delimiter was not found among the first {max_bytes} bytes" - ) - - -class EndOfStream(Exception): - """ - Raised when trying to read from a stream that has been closed from the other end. - """ - - -class IncompleteRead(Exception): - """ - Raised during - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_exactly` or - :meth:`~anyio.streams.buffered.BufferedByteReceiveStream.receive_until` if the - connection is closed before the requested amount of bytes has been read. - """ - - def __init__(self) -> None: - super().__init__( - "The stream was closed before the read operation could be completed" - ) - - -class TypedAttributeLookupError(LookupError): - """ - Raised by :meth:`~anyio.TypedAttributeProvider.extra` when the given typed attribute - is not found and no default value has been given. - """ - - -class WouldBlock(Exception): - """Raised by ``X_nowait`` functions if ``X()`` would block.""" - - -class NoEventLoopError(RuntimeError): - """ - Raised by several functions that require an event loop to be running in the current - thread when there is no running event loop. - - This is also raised by :func:`.from_thread.run` and :func:`.from_thread.run_sync` - if not calling from an AnyIO worker thread, and no ``token`` was passed. - """ - - -class RunFinishedError(RuntimeError): - """ - Raised by :func:`.from_thread.run` and :func:`.from_thread.run_sync` if the event - loop associated with the explicitly passed token has already finished. - """ - - def __init__(self) -> None: - super().__init__( - "The event loop associated with the given token has already finished" - ) diff --git a/functions/python/data-processor/anyio/_core/_fileio.py b/functions/python/data-processor/anyio/_core/_fileio.py deleted file mode 100644 index 3bb8c8456..000000000 --- a/functions/python/data-processor/anyio/_core/_fileio.py +++ /dev/null @@ -1,799 +0,0 @@ -from __future__ import annotations - -import os -import pathlib -import sys -from collections.abc import ( - AsyncIterator, - Callable, - Iterable, - Iterator, - Sequence, -) -from dataclasses import dataclass -from functools import partial -from os import PathLike -from typing import ( - IO, - TYPE_CHECKING, - Any, - AnyStr, - ClassVar, - Final, - Generic, - overload, -) - -from .. import to_thread -from ..abc import AsyncResource - -if TYPE_CHECKING: - from types import ModuleType - - from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer -else: - ReadableBuffer = OpenBinaryMode = OpenTextMode = WriteableBuffer = object - - -class AsyncFile(AsyncResource, Generic[AnyStr]): - """ - An asynchronous file object. - - This class wraps a standard file object and provides async friendly versions of the - following blocking methods (where available on the original file object): - - * read - * read1 - * readline - * readlines - * readinto - * readinto1 - * write - * writelines - * truncate - * seek - * tell - * flush - - All other methods are directly passed through. - - This class supports the asynchronous context manager protocol which closes the - underlying file at the end of the context block. - - This class also supports asynchronous iteration:: - - async with await open_file(...) as f: - async for line in f: - print(line) - """ - - def __init__(self, fp: IO[AnyStr]) -> None: - self._fp: Any = fp - - def __getattr__(self, name: str) -> object: - return getattr(self._fp, name) - - @property - def wrapped(self) -> IO[AnyStr]: - """The wrapped file object.""" - return self._fp - - async def __aiter__(self) -> AsyncIterator[AnyStr]: - while True: - line = await self.readline() - if line: - yield line - else: - break - - async def aclose(self) -> None: - return await to_thread.run_sync(self._fp.close) - - async def read(self, size: int = -1) -> AnyStr: - return await to_thread.run_sync(self._fp.read, size) - - async def read1(self: AsyncFile[bytes], size: int = -1) -> bytes: - return await to_thread.run_sync(self._fp.read1, size) - - async def readline(self) -> AnyStr: - return await to_thread.run_sync(self._fp.readline) - - async def readlines(self) -> list[AnyStr]: - return await to_thread.run_sync(self._fp.readlines) - - async def readinto(self: AsyncFile[bytes], b: WriteableBuffer) -> int: - return await to_thread.run_sync(self._fp.readinto, b) - - async def readinto1(self: AsyncFile[bytes], b: WriteableBuffer) -> int: - return await to_thread.run_sync(self._fp.readinto1, b) - - @overload - async def write(self: AsyncFile[bytes], b: ReadableBuffer) -> int: ... - - @overload - async def write(self: AsyncFile[str], b: str) -> int: ... - - async def write(self, b: ReadableBuffer | str) -> int: - return await to_thread.run_sync(self._fp.write, b) - - @overload - async def writelines( - self: AsyncFile[bytes], lines: Iterable[ReadableBuffer] - ) -> None: ... - - @overload - async def writelines(self: AsyncFile[str], lines: Iterable[str]) -> None: ... - - async def writelines(self, lines: Iterable[ReadableBuffer] | Iterable[str]) -> None: - return await to_thread.run_sync(self._fp.writelines, lines) - - async def truncate(self, size: int | None = None) -> int: - return await to_thread.run_sync(self._fp.truncate, size) - - async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: - return await to_thread.run_sync(self._fp.seek, offset, whence) - - async def tell(self) -> int: - return await to_thread.run_sync(self._fp.tell) - - async def flush(self) -> None: - return await to_thread.run_sync(self._fp.flush) - - -@overload -async def open_file( - file: str | PathLike[str] | int, - mode: OpenBinaryMode, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: Callable[[str, int], int] | None = ..., -) -> AsyncFile[bytes]: ... - - -@overload -async def open_file( - file: str | PathLike[str] | int, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - closefd: bool = ..., - opener: Callable[[str, int], int] | None = ..., -) -> AsyncFile[str]: ... - - -async def open_file( - file: str | PathLike[str] | int, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - closefd: bool = True, - opener: Callable[[str, int], int] | None = None, -) -> AsyncFile[Any]: - """ - Open a file asynchronously. - - The arguments are exactly the same as for the builtin :func:`open`. - - :return: an asynchronous file object - - """ - fp = await to_thread.run_sync( - open, file, mode, buffering, encoding, errors, newline, closefd, opener - ) - return AsyncFile(fp) - - -def wrap_file(file: IO[AnyStr]) -> AsyncFile[AnyStr]: - """ - Wrap an existing file as an asynchronous file. - - :param file: an existing file-like object - :return: an asynchronous file object - - """ - return AsyncFile(file) - - -@dataclass(eq=False) -class _PathIterator(AsyncIterator["Path"]): - iterator: Iterator[PathLike[str]] - - async def __anext__(self) -> Path: - nextval = await to_thread.run_sync( - next, self.iterator, None, abandon_on_cancel=True - ) - if nextval is None: - raise StopAsyncIteration from None - - return Path(nextval) - - -class Path: - """ - An asynchronous version of :class:`pathlib.Path`. - - This class cannot be substituted for :class:`pathlib.Path` or - :class:`pathlib.PurePath`, but it is compatible with the :class:`os.PathLike` - interface. - - It implements the Python 3.10 version of :class:`pathlib.Path` interface, except for - the deprecated :meth:`~pathlib.Path.link_to` method. - - Some methods may be unavailable or have limited functionality, based on the Python - version: - - * :meth:`~pathlib.Path.copy` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.copy_into` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.from_uri` (available on Python 3.13 or later) - * :meth:`~pathlib.PurePath.full_match` (available on Python 3.13 or later) - * :attr:`~pathlib.Path.info` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.is_junction` (available on Python 3.12 or later) - * :meth:`~pathlib.PurePath.match` (the ``case_sensitive`` parameter is only - available on Python 3.13 or later) - * :meth:`~pathlib.Path.move` (available on Python 3.14 or later) - * :meth:`~pathlib.Path.move_into` (available on Python 3.14 or later) - * :meth:`~pathlib.PurePath.relative_to` (the ``walk_up`` parameter is only available - on Python 3.12 or later) - * :meth:`~pathlib.Path.walk` (available on Python 3.12 or later) - - Any methods that do disk I/O need to be awaited on. These methods are: - - * :meth:`~pathlib.Path.absolute` - * :meth:`~pathlib.Path.chmod` - * :meth:`~pathlib.Path.cwd` - * :meth:`~pathlib.Path.exists` - * :meth:`~pathlib.Path.expanduser` - * :meth:`~pathlib.Path.group` - * :meth:`~pathlib.Path.hardlink_to` - * :meth:`~pathlib.Path.home` - * :meth:`~pathlib.Path.is_block_device` - * :meth:`~pathlib.Path.is_char_device` - * :meth:`~pathlib.Path.is_dir` - * :meth:`~pathlib.Path.is_fifo` - * :meth:`~pathlib.Path.is_file` - * :meth:`~pathlib.Path.is_junction` - * :meth:`~pathlib.Path.is_mount` - * :meth:`~pathlib.Path.is_socket` - * :meth:`~pathlib.Path.is_symlink` - * :meth:`~pathlib.Path.lchmod` - * :meth:`~pathlib.Path.lstat` - * :meth:`~pathlib.Path.mkdir` - * :meth:`~pathlib.Path.open` - * :meth:`~pathlib.Path.owner` - * :meth:`~pathlib.Path.read_bytes` - * :meth:`~pathlib.Path.read_text` - * :meth:`~pathlib.Path.readlink` - * :meth:`~pathlib.Path.rename` - * :meth:`~pathlib.Path.replace` - * :meth:`~pathlib.Path.resolve` - * :meth:`~pathlib.Path.rmdir` - * :meth:`~pathlib.Path.samefile` - * :meth:`~pathlib.Path.stat` - * :meth:`~pathlib.Path.symlink_to` - * :meth:`~pathlib.Path.touch` - * :meth:`~pathlib.Path.unlink` - * :meth:`~pathlib.Path.walk` - * :meth:`~pathlib.Path.write_bytes` - * :meth:`~pathlib.Path.write_text` - - Additionally, the following methods return an async iterator yielding - :class:`~.Path` objects: - - * :meth:`~pathlib.Path.glob` - * :meth:`~pathlib.Path.iterdir` - * :meth:`~pathlib.Path.rglob` - """ - - __slots__ = "_path", "__weakref__" - - __weakref__: Any - - def __init__(self, *args: str | PathLike[str]) -> None: - self._path: Final[pathlib.Path] = pathlib.Path(*args) - - def __fspath__(self) -> str: - return self._path.__fspath__() - - if sys.version_info >= (3, 15): - - def __vfspath__(self) -> str: - return self._path.__vfspath__() - - def __str__(self) -> str: - return self._path.__str__() - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.as_posix()!r})" - - def __bytes__(self) -> bytes: - return self._path.__bytes__() - - def __hash__(self) -> int: - return self._path.__hash__() - - def __eq__(self, other: object) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__eq__(target) - - def __lt__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__lt__(target) - - def __le__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__le__(target) - - def __gt__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__gt__(target) - - def __ge__(self, other: pathlib.PurePath | Path) -> bool: - target = other._path if isinstance(other, Path) else other - return self._path.__ge__(target) - - def __truediv__(self, other: str | PathLike[str]) -> Path: - return Path(self._path / other) - - def __rtruediv__(self, other: str | PathLike[str]) -> Path: - return Path(other) / self - - @property - def parts(self) -> tuple[str, ...]: - return self._path.parts - - @property - def drive(self) -> str: - return self._path.drive - - @property - def root(self) -> str: - return self._path.root - - @property - def anchor(self) -> str: - return self._path.anchor - - @property - def parents(self) -> Sequence[Path]: - return tuple(Path(p) for p in self._path.parents) - - @property - def parent(self) -> Path: - return Path(self._path.parent) - - @property - def name(self) -> str: - return self._path.name - - @property - def suffix(self) -> str: - return self._path.suffix - - @property - def suffixes(self) -> list[str]: - return self._path.suffixes - - @property - def stem(self) -> str: - return self._path.stem - - async def absolute(self) -> Path: - path = await to_thread.run_sync(self._path.absolute) - return Path(path) - - def as_posix(self) -> str: - return self._path.as_posix() - - def as_uri(self) -> str: - return self._path.as_uri() - - if sys.version_info >= (3, 13): - parser: ClassVar[ModuleType] = pathlib.Path.parser - - @classmethod - def from_uri(cls, uri: str) -> Path: - return Path(pathlib.Path.from_uri(uri)) - - def full_match( - self, path_pattern: str, *, case_sensitive: bool | None = None - ) -> bool: - return self._path.full_match(path_pattern, case_sensitive=case_sensitive) - - def match( - self, path_pattern: str, *, case_sensitive: bool | None = None - ) -> bool: - return self._path.match(path_pattern, case_sensitive=case_sensitive) - else: - - def match(self, path_pattern: str) -> bool: - return self._path.match(path_pattern) - - if sys.version_info >= (3, 14): - - @property - def info(self) -> Any: # TODO: add return type annotation when Typeshed gets it - return self._path.info - - async def copy( - self, - target: str | os.PathLike[str], - *, - follow_symlinks: bool = True, - preserve_metadata: bool = False, - ) -> Path: - func = partial( - self._path.copy, - follow_symlinks=follow_symlinks, - preserve_metadata=preserve_metadata, - ) - return Path(await to_thread.run_sync(func, pathlib.Path(target))) - - async def copy_into( - self, - target_dir: str | os.PathLike[str], - *, - follow_symlinks: bool = True, - preserve_metadata: bool = False, - ) -> Path: - func = partial( - self._path.copy_into, - follow_symlinks=follow_symlinks, - preserve_metadata=preserve_metadata, - ) - return Path(await to_thread.run_sync(func, pathlib.Path(target_dir))) - - async def move(self, target: str | os.PathLike[str]) -> Path: - # Upstream does not handle anyio.Path properly as a PathLike - target = pathlib.Path(target) - return Path(await to_thread.run_sync(self._path.move, target)) - - async def move_into( - self, - target_dir: str | os.PathLike[str], - ) -> Path: - return Path(await to_thread.run_sync(self._path.move_into, target_dir)) - - def is_relative_to(self, other: str | PathLike[str]) -> bool: - try: - self.relative_to(other) - return True - except ValueError: - return False - - async def chmod(self, mode: int, *, follow_symlinks: bool = True) -> None: - func = partial(os.chmod, follow_symlinks=follow_symlinks) - return await to_thread.run_sync(func, self._path, mode) - - @classmethod - async def cwd(cls) -> Path: - path = await to_thread.run_sync(pathlib.Path.cwd) - return cls(path) - - async def exists(self) -> bool: - return await to_thread.run_sync(self._path.exists, abandon_on_cancel=True) - - async def expanduser(self) -> Path: - return Path( - await to_thread.run_sync(self._path.expanduser, abandon_on_cancel=True) - ) - - if sys.version_info < (3, 12): - # Python 3.11 and earlier - def glob(self, pattern: str) -> AsyncIterator[Path]: - gen = self._path.glob(pattern) - return _PathIterator(gen) - elif (3, 12) <= sys.version_info < (3, 13): - # changed in Python 3.12: - # - The case_sensitive parameter was added. - def glob( - self, - pattern: str, - *, - case_sensitive: bool | None = None, - ) -> AsyncIterator[Path]: - gen = self._path.glob(pattern, case_sensitive=case_sensitive) - return _PathIterator(gen) - elif sys.version_info >= (3, 13): - # Changed in Python 3.13: - # - The recurse_symlinks parameter was added. - # - The pattern parameter accepts a path-like object. - def glob( # type: ignore[misc] # mypy doesn't allow for differing signatures in a conditional block - self, - pattern: str | PathLike[str], - *, - case_sensitive: bool | None = None, - recurse_symlinks: bool = False, - ) -> AsyncIterator[Path]: - gen = self._path.glob( - pattern, # type: ignore[arg-type] - case_sensitive=case_sensitive, - recurse_symlinks=recurse_symlinks, - ) - return _PathIterator(gen) - - async def group(self) -> str: - return await to_thread.run_sync(self._path.group, abandon_on_cancel=True) - - async def hardlink_to( - self, target: str | bytes | PathLike[str] | PathLike[bytes] - ) -> None: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(os.link, target, self) - - @classmethod - async def home(cls) -> Path: - home_path = await to_thread.run_sync(pathlib.Path.home) - return cls(home_path) - - def is_absolute(self) -> bool: - return self._path.is_absolute() - - async def is_block_device(self) -> bool: - return await to_thread.run_sync( - self._path.is_block_device, abandon_on_cancel=True - ) - - async def is_char_device(self) -> bool: - return await to_thread.run_sync( - self._path.is_char_device, abandon_on_cancel=True - ) - - async def is_dir(self) -> bool: - return await to_thread.run_sync(self._path.is_dir, abandon_on_cancel=True) - - async def is_fifo(self) -> bool: - return await to_thread.run_sync(self._path.is_fifo, abandon_on_cancel=True) - - async def is_file(self) -> bool: - return await to_thread.run_sync(self._path.is_file, abandon_on_cancel=True) - - if sys.version_info >= (3, 12): - - async def is_junction(self) -> bool: - return await to_thread.run_sync(self._path.is_junction) - - async def is_mount(self) -> bool: - return await to_thread.run_sync( - os.path.ismount, self._path, abandon_on_cancel=True - ) - - if sys.version_info < (3, 15): - - def is_reserved(self) -> bool: - return self._path.is_reserved() - - async def is_socket(self) -> bool: - return await to_thread.run_sync(self._path.is_socket, abandon_on_cancel=True) - - async def is_symlink(self) -> bool: - return await to_thread.run_sync(self._path.is_symlink, abandon_on_cancel=True) - - async def iterdir(self) -> AsyncIterator[Path]: - gen = ( - self._path.iterdir() - if sys.version_info < (3, 13) - else await to_thread.run_sync(self._path.iterdir, abandon_on_cancel=True) - ) - async for path in _PathIterator(gen): - yield path - - def joinpath(self, *args: str | PathLike[str]) -> Path: - return Path(self._path.joinpath(*args)) - - async def lchmod(self, mode: int) -> None: - await to_thread.run_sync(self._path.lchmod, mode) - - async def lstat(self) -> os.stat_result: - return await to_thread.run_sync(self._path.lstat, abandon_on_cancel=True) - - async def mkdir( - self, mode: int = 0o777, parents: bool = False, exist_ok: bool = False - ) -> None: - await to_thread.run_sync(self._path.mkdir, mode, parents, exist_ok) - - @overload - async def open( - self, - mode: OpenBinaryMode, - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> AsyncFile[bytes]: ... - - @overload - async def open( - self, - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - errors: str | None = ..., - newline: str | None = ..., - ) -> AsyncFile[str]: ... - - async def open( - self, - mode: str = "r", - buffering: int = -1, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> AsyncFile[Any]: - fp = await to_thread.run_sync( - self._path.open, mode, buffering, encoding, errors, newline - ) - return AsyncFile(fp) - - async def owner(self) -> str: - return await to_thread.run_sync(self._path.owner, abandon_on_cancel=True) - - async def read_bytes(self) -> bytes: - return await to_thread.run_sync(self._path.read_bytes) - - async def read_text( - self, encoding: str | None = None, errors: str | None = None - ) -> str: - return await to_thread.run_sync(self._path.read_text, encoding, errors) - - if sys.version_info >= (3, 12): - - def relative_to( - self, *other: str | PathLike[str], walk_up: bool = False - ) -> Path: - # relative_to() should work with any PathLike but it doesn't - others = [pathlib.Path(other) for other in other] - return Path(self._path.relative_to(*others, walk_up=walk_up)) - - else: - - def relative_to(self, *other: str | PathLike[str]) -> Path: - return Path(self._path.relative_to(*other)) - - async def readlink(self) -> Path: - target = await to_thread.run_sync(os.readlink, self._path) - return Path(target) - - async def rename(self, target: str | pathlib.PurePath | Path) -> Path: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.rename, target) - return Path(target) - - async def replace(self, target: str | pathlib.PurePath | Path) -> Path: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.replace, target) - return Path(target) - - async def resolve(self, strict: bool = False) -> Path: - func = partial(self._path.resolve, strict=strict) - return Path(await to_thread.run_sync(func, abandon_on_cancel=True)) - - if sys.version_info < (3, 12): - # Pre Python 3.12 - def rglob(self, pattern: str) -> AsyncIterator[Path]: - gen = self._path.rglob(pattern) - return _PathIterator(gen) - elif (3, 12) <= sys.version_info < (3, 13): - # Changed in Python 3.12: - # - The case_sensitive parameter was added. - def rglob( - self, pattern: str, *, case_sensitive: bool | None = None - ) -> AsyncIterator[Path]: - gen = self._path.rglob(pattern, case_sensitive=case_sensitive) - return _PathIterator(gen) - elif sys.version_info >= (3, 13): - # Changed in Python 3.13: - # - The recurse_symlinks parameter was added. - # - The pattern parameter accepts a path-like object. - def rglob( # type: ignore[misc] # mypy doesn't allow for differing signatures in a conditional block - self, - pattern: str | PathLike[str], - *, - case_sensitive: bool | None = None, - recurse_symlinks: bool = False, - ) -> AsyncIterator[Path]: - gen = self._path.rglob( - pattern, # type: ignore[arg-type] - case_sensitive=case_sensitive, - recurse_symlinks=recurse_symlinks, - ) - return _PathIterator(gen) - - async def rmdir(self) -> None: - await to_thread.run_sync(self._path.rmdir) - - async def samefile(self, other_path: str | PathLike[str]) -> bool: - if isinstance(other_path, Path): - other_path = other_path._path - - return await to_thread.run_sync( - self._path.samefile, other_path, abandon_on_cancel=True - ) - - async def stat(self, *, follow_symlinks: bool = True) -> os.stat_result: - func = partial(os.stat, follow_symlinks=follow_symlinks) - return await to_thread.run_sync(func, self._path, abandon_on_cancel=True) - - async def symlink_to( - self, - target: str | bytes | PathLike[str] | PathLike[bytes], - target_is_directory: bool = False, - ) -> None: - if isinstance(target, Path): - target = target._path - - await to_thread.run_sync(self._path.symlink_to, target, target_is_directory) - - async def touch(self, mode: int = 0o666, exist_ok: bool = True) -> None: - await to_thread.run_sync(self._path.touch, mode, exist_ok) - - async def unlink(self, missing_ok: bool = False) -> None: - try: - await to_thread.run_sync(self._path.unlink) - except FileNotFoundError: - if not missing_ok: - raise - - if sys.version_info >= (3, 12): - - async def walk( - self, - top_down: bool = True, - on_error: Callable[[OSError], object] | None = None, - follow_symlinks: bool = False, - ) -> AsyncIterator[tuple[Path, list[str], list[str]]]: - def get_next_value() -> tuple[pathlib.Path, list[str], list[str]] | None: - try: - return next(gen) - except StopIteration: - return None - - gen = self._path.walk(top_down, on_error, follow_symlinks) - while True: - value = await to_thread.run_sync(get_next_value) - if value is None: - return - - root, dirs, paths = value - yield Path(root), dirs, paths - - def with_name(self, name: str) -> Path: - return Path(self._path.with_name(name)) - - def with_stem(self, stem: str) -> Path: - return Path(self._path.with_name(stem + self._path.suffix)) - - def with_suffix(self, suffix: str) -> Path: - return Path(self._path.with_suffix(suffix)) - - def with_segments(self, *pathsegments: str | PathLike[str]) -> Path: - return Path(*pathsegments) - - async def write_bytes(self, data: bytes) -> int: - return await to_thread.run_sync(self._path.write_bytes, data) - - async def write_text( - self, - data: str, - encoding: str | None = None, - errors: str | None = None, - newline: str | None = None, - ) -> int: - return await to_thread.run_sync( - self._path.write_text, data, encoding, errors, newline - ) - - -PathLike.register(Path) diff --git a/functions/python/data-processor/anyio/_core/_resources.py b/functions/python/data-processor/anyio/_core/_resources.py deleted file mode 100644 index b9a5344ae..000000000 --- a/functions/python/data-processor/anyio/_core/_resources.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from ..abc import AsyncResource -from ._tasks import CancelScope - - -async def aclose_forcefully(resource: AsyncResource) -> None: - """ - Close an asynchronous resource in a cancelled scope. - - Doing this closes the resource without waiting on anything. - - :param resource: the resource to close - - """ - with CancelScope() as scope: - scope.cancel() - await resource.aclose() diff --git a/functions/python/data-processor/anyio/_core/_signals.py b/functions/python/data-processor/anyio/_core/_signals.py deleted file mode 100644 index e24c79e10..000000000 --- a/functions/python/data-processor/anyio/_core/_signals.py +++ /dev/null @@ -1,29 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncIterator -from contextlib import AbstractContextManager -from signal import Signals - -from ._eventloop import get_async_backend - - -def open_signal_receiver( - *signals: Signals, -) -> AbstractContextManager[AsyncIterator[Signals]]: - """ - Start receiving operating system signals. - - :param signals: signals to receive (e.g. ``signal.SIGINT``) - :return: an asynchronous context manager for an asynchronous iterator which yields - signal numbers - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. warning:: Windows does not support signals natively so it is best to avoid - relying on this in cross-platform applications. - - .. warning:: On asyncio, this permanently replaces any previous signal handler for - the given signals, as set via :meth:`~asyncio.loop.add_signal_handler`. - - """ - return get_async_backend().open_signal_receiver(*signals) diff --git a/functions/python/data-processor/anyio/_core/_sockets.py b/functions/python/data-processor/anyio/_core/_sockets.py deleted file mode 100644 index 6c99b3a1c..000000000 --- a/functions/python/data-processor/anyio/_core/_sockets.py +++ /dev/null @@ -1,1003 +0,0 @@ -from __future__ import annotations - -import errno -import os -import socket -import ssl -import stat -import sys -from collections.abc import Awaitable -from dataclasses import dataclass -from ipaddress import IPv4Address, IPv6Address, ip_address -from os import PathLike, chmod -from socket import AddressFamily, SocketKind -from typing import TYPE_CHECKING, Any, Literal, cast, overload - -from .. import ConnectionFailed, to_thread -from ..abc import ( - ByteStreamConnectable, - ConnectedUDPSocket, - ConnectedUNIXDatagramSocket, - IPAddressType, - IPSockAddrType, - SocketListener, - SocketStream, - UDPSocket, - UNIXDatagramSocket, - UNIXSocketStream, -) -from ..streams.stapled import MultiListener -from ..streams.tls import TLSConnectable, TLSStream -from ._eventloop import get_async_backend -from ._resources import aclose_forcefully -from ._synchronization import Event -from ._tasks import create_task_group, move_on_after - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike -else: - FileDescriptorLike = object - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - -if sys.version_info < (3, 13): - from typing_extensions import deprecated -else: - from warnings import deprecated - -IPPROTO_IPV6 = getattr(socket, "IPPROTO_IPV6", 41) # https://bugs.python.org/issue29515 - -AnyIPAddressFamily = Literal[ - AddressFamily.AF_UNSPEC, AddressFamily.AF_INET, AddressFamily.AF_INET6 -] -IPAddressFamily = Literal[AddressFamily.AF_INET, AddressFamily.AF_INET6] - - -# tls_hostname given -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str, - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# ssl_context given -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - ssl_context: ssl.SSLContext, - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# tls=True -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - tls: Literal[True], - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> TLSStream: ... - - -# tls=False -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - tls: Literal[False], - ssl_context: ssl.SSLContext | None = ..., - tls_standard_compatible: bool = ..., - tls_hostname: str | None = ..., - happy_eyeballs_delay: float = ..., -) -> SocketStream: ... - - -# No TLS arguments -@overload -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = ..., - happy_eyeballs_delay: float = ..., -) -> SocketStream: ... - - -async def connect_tcp( - remote_host: IPAddressType, - remote_port: int, - *, - local_host: IPAddressType | None = None, - tls: bool = False, - ssl_context: ssl.SSLContext | None = None, - tls_standard_compatible: bool = True, - tls_hostname: str | None = None, - happy_eyeballs_delay: float = 0.25, -) -> SocketStream | TLSStream: - """ - Connect to a host using the TCP protocol. - - This function implements the stateless version of the Happy Eyeballs algorithm (RFC - 6555). If ``remote_host`` is a host name that resolves to multiple IP addresses, - each one is tried until one connection attempt succeeds. If the first attempt does - not connected within 250 milliseconds, a second attempt is started using the next - address in the list, and so on. On IPv6 enabled systems, an IPv6 address (if - available) is tried first. - - When the connection has been established, a TLS handshake will be done if either - ``ssl_context`` or ``tls_hostname`` is not ``None``, or if ``tls`` is ``True``. - - :param remote_host: the IP address or host name to connect to - :param remote_port: port on the target host to connect to - :param local_host: the interface address or name to bind the socket to before - connecting - :param tls: ``True`` to do a TLS handshake with the connected stream and return a - :class:`~anyio.streams.tls.TLSStream` instead - :param ssl_context: the SSL context object to use (if omitted, a default context is - created) - :param tls_standard_compatible: If ``True``, performs the TLS shutdown handshake - before closing the stream and requires that the server does this as well. - Otherwise, :exc:`~ssl.SSLEOFError` may be raised during reads from the stream. - Some protocols, such as HTTP, require this option to be ``False``. - See :meth:`~ssl.SSLContext.wrap_socket` for details. - :param tls_hostname: host name to check the server certificate against (defaults to - the value of ``remote_host``) - :param happy_eyeballs_delay: delay (in seconds) before starting the next connection - attempt - :return: a socket stream object if no TLS handshake was done, otherwise a TLS stream - :raises ConnectionFailed: if the connection fails - - """ - # Placed here due to https://github.com/python/mypy/issues/7057 - connected_stream: SocketStream | None = None - - async def try_connect(remote_host: str, event: Event) -> None: - nonlocal connected_stream - try: - stream = await asynclib.connect_tcp(remote_host, remote_port, local_address) - except OSError as exc: - oserrors.append(exc) - return - else: - if connected_stream is None: - connected_stream = stream - tg.cancel_scope.cancel() - else: - await stream.aclose() - finally: - event.set() - - asynclib = get_async_backend() - local_address: IPSockAddrType | None = None - family = socket.AF_UNSPEC - if local_host: - gai_res = await getaddrinfo(str(local_host), None) - family, *_, local_address = gai_res[0] - - target_host = str(remote_host) - try: - addr_obj = ip_address(remote_host) - except ValueError: - addr_obj = None - - if addr_obj is not None: - if isinstance(addr_obj, IPv6Address): - target_addrs = [(socket.AF_INET6, addr_obj.compressed)] - else: - target_addrs = [(socket.AF_INET, addr_obj.compressed)] - else: - # getaddrinfo() will raise an exception if name resolution fails - gai_res = await getaddrinfo( - target_host, remote_port, family=family, type=socket.SOCK_STREAM - ) - - # Organize the list so that the first address is an IPv6 address (if available) - # and the second one is an IPv4 addresses. The rest can be in whatever order. - v6_found = v4_found = False - target_addrs = [] - for af, *_, sa in gai_res: - if af == socket.AF_INET6 and not v6_found: - v6_found = True - target_addrs.insert(0, (af, sa[0])) - elif af == socket.AF_INET and not v4_found and v6_found: - v4_found = True - target_addrs.insert(1, (af, sa[0])) - else: - target_addrs.append((af, sa[0])) - - oserrors: list[OSError] = [] - try: - async with create_task_group() as tg: - for _af, addr in target_addrs: - event = Event() - tg.start_soon(try_connect, addr, event) - with move_on_after(happy_eyeballs_delay): - await event.wait() - - if connected_stream is None: - cause = ( - oserrors[0] - if len(oserrors) == 1 - else ExceptionGroup("multiple connection attempts failed", oserrors) - ) - raise OSError("All connection attempts failed") from cause - finally: - oserrors.clear() - - if tls or tls_hostname or ssl_context: - try: - return await TLSStream.wrap( - connected_stream, - server_side=False, - hostname=tls_hostname or str(remote_host), - ssl_context=ssl_context, - standard_compatible=tls_standard_compatible, - ) - except BaseException: - await aclose_forcefully(connected_stream) - raise - - return connected_stream - - -async def connect_unix(path: str | bytes | PathLike[Any]) -> UNIXSocketStream: - """ - Connect to the given UNIX socket. - - Not available on Windows. - - :param path: path to the socket - :return: a socket stream object - :raises ConnectionFailed: if the connection fails - - """ - path = os.fspath(path) - return await get_async_backend().connect_unix(path) - - -async def create_tcp_listener( - *, - local_host: IPAddressType | None = None, - local_port: int = 0, - family: AnyIPAddressFamily = socket.AddressFamily.AF_UNSPEC, - backlog: int = 65536, - reuse_port: bool = False, -) -> MultiListener[SocketStream]: - """ - Create a TCP socket listener. - - :param local_port: port number to listen on - :param local_host: IP address of the interface to listen on. If omitted, listen on - all IPv4 and IPv6 interfaces. To listen on all interfaces on a specific address - family, use ``0.0.0.0`` for IPv4 or ``::`` for IPv6. - :param family: address family (used if ``local_host`` was omitted) - :param backlog: maximum number of queued incoming connections (up to a maximum of - 2**16, or 65536) - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a multi-listener object containing one or more socket listeners - :raises OSError: if there's an error creating a socket, or binding to one or more - interfaces failed - - """ - asynclib = get_async_backend() - backlog = min(backlog, 65536) - local_host = str(local_host) if local_host is not None else None - - def setup_raw_socket( - fam: AddressFamily, - bind_addr: tuple[str, int] | tuple[str, int, int, int], - *, - v6only: bool = True, - ) -> socket.socket: - sock = socket.socket(fam) - try: - sock.setblocking(False) - - if fam == AddressFamily.AF_INET6: - sock.setsockopt(IPPROTO_IPV6, socket.IPV6_V6ONLY, v6only) - - # For Windows, enable exclusive address use. For others, enable address - # reuse. - if sys.platform == "win32": - sock.setsockopt(socket.SOL_SOCKET, socket.SO_EXCLUSIVEADDRUSE, 1) - else: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - - if reuse_port: - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEPORT, 1) - - # Workaround for #554 - if fam == socket.AF_INET6 and "%" in bind_addr[0]: - addr, scope_id = bind_addr[0].split("%", 1) - bind_addr = (addr, bind_addr[1], 0, int(scope_id)) - - sock.bind(bind_addr) - sock.listen(backlog) - except BaseException: - sock.close() - raise - - return sock - - # We passing type=0 on non-Windows platforms as a workaround for a uvloop bug - # where we don't get the correct scope ID for IPv6 link-local addresses when passing - # type=socket.SOCK_STREAM to getaddrinfo(): - # https://github.com/MagicStack/uvloop/issues/539 - gai_res = await getaddrinfo( - local_host, - local_port, - family=family, - type=socket.SOCK_STREAM if sys.platform == "win32" else 0, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - - # The set comprehension is here to work around a glibc bug: - # https://sourceware.org/bugzilla/show_bug.cgi?id=14969 - sockaddrs = sorted({res for res in gai_res if res[1] == SocketKind.SOCK_STREAM}) - - # Special case for dual-stack binding on the "any" interface - if ( - local_host is None - and family == AddressFamily.AF_UNSPEC - and socket.has_dualstack_ipv6() - and any(fam == AddressFamily.AF_INET6 for fam, *_ in gai_res) - ): - raw_socket = setup_raw_socket( - AddressFamily.AF_INET6, ("::", local_port), v6only=False - ) - listener = asynclib.create_tcp_listener(raw_socket) - return MultiListener([listener]) - - errors: list[OSError] = [] - try: - for _ in range(len(sockaddrs)): - listeners: list[SocketListener] = [] - bound_ephemeral_port = local_port - try: - for fam, *_, sockaddr in sockaddrs: - sockaddr = sockaddr[0], bound_ephemeral_port, *sockaddr[2:] - raw_socket = setup_raw_socket(fam, sockaddr) - - # Store the assigned port if an ephemeral port was requested, so - # we'll bind to the same port on all interfaces - if local_port == 0 and len(gai_res) > 1: - bound_ephemeral_port = raw_socket.getsockname()[1] - - listeners.append(asynclib.create_tcp_listener(raw_socket)) - except BaseException as exc: - for listener in listeners: - await listener.aclose() - - # If an ephemeral port was requested but binding the assigned port - # failed for another interface, rotate the address list and try again - if ( - isinstance(exc, OSError) - and exc.errno == errno.EADDRINUSE - and local_port == 0 - and bound_ephemeral_port - ): - errors.append(exc) - sockaddrs.append(sockaddrs.pop(0)) - continue - - raise - - return MultiListener(listeners) - - raise OSError( - f"Could not create {len(sockaddrs)} listeners with a consistent port" - ) from ExceptionGroup("Several bind attempts failed", errors) - finally: - del errors # Prevent reference cycles - - -async def create_unix_listener( - path: str | bytes | PathLike[Any], - *, - mode: int | None = None, - backlog: int = 65536, -) -> SocketListener: - """ - Create a UNIX socket listener. - - Not available on Windows. - - :param path: path of the socket - :param mode: permissions to set on the socket - :param backlog: maximum number of queued incoming connections (up to a maximum of - 2**16, or 65536) - :return: a listener object - - .. versionchanged:: 3.0 - If a socket already exists on the file system in the given path, it will be - removed first. - - """ - backlog = min(backlog, 65536) - raw_socket = await setup_unix_local_socket(path, mode, socket.SOCK_STREAM) - try: - raw_socket.listen(backlog) - return get_async_backend().create_unix_listener(raw_socket) - except BaseException: - raw_socket.close() - raise - - -async def create_udp_socket( - family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, - *, - local_host: IPAddressType | None = None, - local_port: int = 0, - reuse_port: bool = False, -) -> UDPSocket: - """ - Create a UDP socket. - - If ``port`` has been given, the socket will be bound to this port on the local - machine, making this socket suitable for providing UDP based services. - - :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically - determined from ``local_host`` if omitted - :param local_host: IP address or host name of the local interface to bind to - :param local_port: local port to bind to - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a UDP socket - - """ - if family is AddressFamily.AF_UNSPEC and not local_host: - raise ValueError('Either "family" or "local_host" must be given') - - if local_host: - gai_res = await getaddrinfo( - str(local_host), - local_port, - family=family, - type=socket.SOCK_DGRAM, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - local_address = gai_res[0][-1] - elif family is AddressFamily.AF_INET6: - local_address = ("::", 0) - else: - local_address = ("0.0.0.0", 0) - - sock = await get_async_backend().create_udp_socket( - family, local_address, None, reuse_port - ) - return cast(UDPSocket, sock) - - -async def create_connected_udp_socket( - remote_host: IPAddressType, - remote_port: int, - *, - family: AnyIPAddressFamily = AddressFamily.AF_UNSPEC, - local_host: IPAddressType | None = None, - local_port: int = 0, - reuse_port: bool = False, -) -> ConnectedUDPSocket: - """ - Create a connected UDP socket. - - Connected UDP sockets can only communicate with the specified remote host/port, an - any packets sent from other sources are dropped. - - :param remote_host: remote host to set as the default target - :param remote_port: port on the remote host to set as the default target - :param family: address family (``AF_INET`` or ``AF_INET6``) – automatically - determined from ``local_host`` or ``remote_host`` if omitted - :param local_host: IP address or host name of the local interface to bind to - :param local_port: local port to bind to - :param reuse_port: ``True`` to allow multiple sockets to bind to the same - address/port (not supported on Windows) - :return: a connected UDP socket - - """ - local_address = None - if local_host: - gai_res = await getaddrinfo( - str(local_host), - local_port, - family=family, - type=socket.SOCK_DGRAM, - flags=socket.AI_PASSIVE | socket.AI_ADDRCONFIG, - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - local_address = gai_res[0][-1] - - gai_res = await getaddrinfo( - str(remote_host), remote_port, family=family, type=socket.SOCK_DGRAM - ) - family = cast(AnyIPAddressFamily, gai_res[0][0]) - remote_address = gai_res[0][-1] - - sock = await get_async_backend().create_udp_socket( - family, local_address, remote_address, reuse_port - ) - return cast(ConnectedUDPSocket, sock) - - -async def create_unix_datagram_socket( - *, - local_path: None | str | bytes | PathLike[Any] = None, - local_mode: int | None = None, -) -> UNIXDatagramSocket: - """ - Create a UNIX datagram socket. - - Not available on Windows. - - If ``local_path`` has been given, the socket will be bound to this path, making this - socket suitable for receiving datagrams from other processes. Other processes can - send datagrams to this socket only if ``local_path`` is set. - - If a socket already exists on the file system in the ``local_path``, it will be - removed first. - - :param local_path: the path on which to bind to - :param local_mode: permissions to set on the local socket - :return: a UNIX datagram socket - - """ - raw_socket = await setup_unix_local_socket( - local_path, local_mode, socket.SOCK_DGRAM - ) - return await get_async_backend().create_unix_datagram_socket(raw_socket, None) - - -async def create_connected_unix_datagram_socket( - remote_path: str | bytes | PathLike[Any], - *, - local_path: None | str | bytes | PathLike[Any] = None, - local_mode: int | None = None, -) -> ConnectedUNIXDatagramSocket: - """ - Create a connected UNIX datagram socket. - - Connected datagram sockets can only communicate with the specified remote path. - - If ``local_path`` has been given, the socket will be bound to this path, making - this socket suitable for receiving datagrams from other processes. Other processes - can send datagrams to this socket only if ``local_path`` is set. - - If a socket already exists on the file system in the ``local_path``, it will be - removed first. - - :param remote_path: the path to set as the default target - :param local_path: the path on which to bind to - :param local_mode: permissions to set on the local socket - :return: a connected UNIX datagram socket - - """ - remote_path = os.fspath(remote_path) - raw_socket = await setup_unix_local_socket( - local_path, local_mode, socket.SOCK_DGRAM - ) - return await get_async_backend().create_unix_datagram_socket( - raw_socket, remote_path - ) - - -async def getaddrinfo( - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, -) -> list[tuple[AddressFamily, SocketKind, int, str, tuple[str, int]]]: - """ - Look up a numeric IP address given a host name. - - Internationalized domain names are translated according to the (non-transitional) - IDNA 2008 standard. - - .. note:: 4-tuple IPv6 socket addresses are automatically converted to 2-tuples of - (host, port), unlike what :func:`socket.getaddrinfo` does. - - :param host: host name - :param port: port number - :param family: socket family (`'AF_INET``, ...) - :param type: socket type (``SOCK_STREAM``, ...) - :param proto: protocol number - :param flags: flags to pass to upstream ``getaddrinfo()`` - :return: list of tuples containing (family, type, proto, canonname, sockaddr) - - .. seealso:: :func:`socket.getaddrinfo` - - """ - # Handle unicode hostnames - if isinstance(host, str): - try: - encoded_host: bytes | None = host.encode("ascii") - except UnicodeEncodeError: - import idna - - encoded_host = idna.encode(host, uts46=True) - else: - encoded_host = host - - gai_res = await get_async_backend().getaddrinfo( - encoded_host, port, family=family, type=type, proto=proto, flags=flags - ) - return [ - (family, type, proto, canonname, convert_ipv6_sockaddr(sockaddr)) - for family, type, proto, canonname, sockaddr in gai_res - # filter out IPv6 results when IPv6 is disabled - if not isinstance(sockaddr[0], int) - ] - - -def getnameinfo(sockaddr: IPSockAddrType, flags: int = 0) -> Awaitable[tuple[str, str]]: - """ - Look up the host name of an IP address. - - :param sockaddr: socket address (e.g. (ipaddress, port) for IPv4) - :param flags: flags to pass to upstream ``getnameinfo()`` - :return: a tuple of (host name, service name) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. seealso:: :func:`socket.getnameinfo` - - """ - return get_async_backend().getnameinfo(sockaddr, flags) - - -@deprecated("This function is deprecated; use `wait_readable` instead") -def wait_socket_readable(sock: socket.socket) -> Awaitable[None]: - """ - .. deprecated:: 4.7.0 - Use :func:`wait_readable` instead. - - Wait until the given socket has data to be read. - - .. warning:: Only use this on raw sockets that have not been wrapped by any higher - level constructs like socket streams! - - :param sock: a socket object - :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the - socket to become readable - :raises ~anyio.BusyResourceError: if another task is already waiting for the socket - to become readable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().wait_readable(sock.fileno()) - - -@deprecated("This function is deprecated; use `wait_writable` instead") -def wait_socket_writable(sock: socket.socket) -> Awaitable[None]: - """ - .. deprecated:: 4.7.0 - Use :func:`wait_writable` instead. - - Wait until the given socket can be written to. - - This does **NOT** work on Windows when using the asyncio backend with a proactor - event loop (default on py3.8+). - - .. warning:: Only use this on raw sockets that have not been wrapped by any higher - level constructs like socket streams! - - :param sock: a socket object - :raises ~anyio.ClosedResourceError: if the socket was closed while waiting for the - socket to become writable - :raises ~anyio.BusyResourceError: if another task is already waiting for the socket - to become writable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().wait_writable(sock.fileno()) - - -def wait_readable(obj: FileDescriptorLike) -> Awaitable[None]: - """ - Wait until the given object has data to be read. - - On Unix systems, ``obj`` must either be an integer file descriptor, or else an - object with a ``.fileno()`` method which returns an integer file descriptor. Any - kind of file descriptor can be passed, though the exact semantics will depend on - your kernel. For example, this probably won't do anything useful for on-disk files. - - On Windows systems, ``obj`` must either be an integer ``SOCKET`` handle, or else an - object with a ``.fileno()`` method which returns an integer ``SOCKET`` handle. File - descriptors aren't supported, and neither are handles that refer to anything besides - a ``SOCKET``. - - On backends where this functionality is not natively provided (asyncio - ``ProactorEventLoop`` on Windows), it is provided using a separate selector thread - which is set to shut down when the interpreter shuts down. - - .. warning:: Don't use this on raw sockets that have been wrapped by any higher - level constructs like socket streams! - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the - object to become readable - :raises ~anyio.BusyResourceError: if another task is already waiting for the object - to become readable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().wait_readable(obj) - - -def wait_writable(obj: FileDescriptorLike) -> Awaitable[None]: - """ - Wait until the given object can be written to. - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises ~anyio.ClosedResourceError: if the object was closed while waiting for the - object to become writable - :raises ~anyio.BusyResourceError: if another task is already waiting for the object - to become writable - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. seealso:: See the documentation of :func:`wait_readable` for the definition of - ``obj`` and notes on backend compatibility. - - .. warning:: Don't use this on raw sockets that have been wrapped by any higher - level constructs like socket streams! - - """ - return get_async_backend().wait_writable(obj) - - -def notify_closing(obj: FileDescriptorLike) -> None: - """ - Call this before closing a file descriptor (on Unix) or socket (on - Windows). This will cause any `wait_readable` or `wait_writable` - calls on the given object to immediately wake up and raise - `~anyio.ClosedResourceError`. - - This doesn't actually close the object – you still have to do that - yourself afterwards. Also, you want to be careful to make sure no - new tasks start waiting on the object in between when you call this - and when it's actually closed. So to close something properly, you - usually want to do these steps in order: - - 1. Explicitly mark the object as closed, so that any new attempts - to use it will abort before they start. - 2. Call `notify_closing` to wake up any already-existing users. - 3. Actually close the object. - - It's also possible to do them in a different order if that's more - convenient, *but only if* you make sure not to have any checkpoints in - between the steps. This way they all happen in a single atomic - step, so other tasks won't be able to tell what order they happened - in anyway. - - :param obj: an object with a ``.fileno()`` method or an integer handle - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - get_async_backend().notify_closing(obj) - - -# -# Private API -# - - -def convert_ipv6_sockaddr( - sockaddr: tuple[str, int, int, int] | tuple[str, int], -) -> tuple[str, int]: - """ - Convert a 4-tuple IPv6 socket address to a 2-tuple (address, port) format. - - If the scope ID is nonzero, it is added to the address, separated with ``%``. - Otherwise the flow id and scope id are simply cut off from the tuple. - Any other kinds of socket addresses are returned as-is. - - :param sockaddr: the result of :meth:`~socket.socket.getsockname` - :return: the converted socket address - - """ - # This is more complicated than it should be because of MyPy - if isinstance(sockaddr, tuple) and len(sockaddr) == 4: - host, port, flowinfo, scope_id = sockaddr - if scope_id: - # PyPy (as of v7.3.11) leaves the interface name in the result, so - # we discard it and only get the scope ID from the end - # (https://foss.heptapod.net/pypy/pypy/-/issues/3938) - host = host.split("%")[0] - - # Add scope_id to the address - return f"{host}%{scope_id}", port - else: - return host, port - else: - return sockaddr - - -async def setup_unix_local_socket( - path: None | str | bytes | PathLike[Any], - mode: int | None, - socktype: int, -) -> socket.socket: - """ - Create a UNIX local socket object, deleting the socket at the given path if it - exists. - - Not available on Windows. - - :param path: path of the socket - :param mode: permissions to set on the socket - :param socktype: socket.SOCK_STREAM or socket.SOCK_DGRAM - - """ - path_str: str | None - if path is not None: - path_str = os.fsdecode(path) - - # Linux abstract namespace sockets aren't backed by a concrete file so skip stat call - if not path_str.startswith("\0"): - # Copied from pathlib... - try: - stat_result = os.stat(path) - except OSError as e: - if e.errno not in ( - errno.ENOENT, - errno.ENOTDIR, - errno.EBADF, - errno.ELOOP, - ): - raise - else: - if stat.S_ISSOCK(stat_result.st_mode): - os.unlink(path) - else: - path_str = None - - raw_socket = socket.socket(socket.AF_UNIX, socktype) - raw_socket.setblocking(False) - - if path_str is not None: - try: - await to_thread.run_sync(raw_socket.bind, path_str, abandon_on_cancel=True) - if mode is not None: - await to_thread.run_sync(chmod, path_str, mode, abandon_on_cancel=True) - except BaseException: - raw_socket.close() - raise - - return raw_socket - - -@dataclass -class TCPConnectable(ByteStreamConnectable): - """ - Connects to a TCP server at the given host and port. - - :param host: host name or IP address of the server - :param port: TCP port number of the server - """ - - host: str | IPv4Address | IPv6Address - port: int - - def __post_init__(self) -> None: - if self.port < 1 or self.port > 65535: - raise ValueError("TCP port number out of range") - - @override - async def connect(self) -> SocketStream: - try: - return await connect_tcp(self.host, self.port) - except OSError as exc: - raise ConnectionFailed( - f"error connecting to {self.host}:{self.port}: {exc}" - ) from exc - - -@dataclass -class UNIXConnectable(ByteStreamConnectable): - """ - Connects to a UNIX domain socket at the given path. - - :param path: the file system path of the socket - """ - - path: str | bytes | PathLike[str] | PathLike[bytes] - - @override - async def connect(self) -> UNIXSocketStream: - try: - return await connect_unix(self.path) - except OSError as exc: - raise ConnectionFailed(f"error connecting to {self.path!r}: {exc}") from exc - - -def as_connectable( - remote: ByteStreamConnectable - | tuple[str | IPv4Address | IPv6Address, int] - | str - | bytes - | PathLike[str], - /, - *, - tls: bool = False, - ssl_context: ssl.SSLContext | None = None, - tls_hostname: str | None = None, - tls_standard_compatible: bool = True, -) -> ByteStreamConnectable: - """ - Return a byte stream connectable from the given object. - - If a bytestream connectable is given, it is returned unchanged. - If a tuple of (host, port) is given, a TCP connectable is returned. - If a string or bytes path is given, a UNIX connectable is returned. - - If ``tls=True``, the connectable will be wrapped in a - :class:`~.streams.tls.TLSConnectable`. - - :param remote: a connectable, a tuple of (host, port) or a path to a UNIX socket - :param tls: if ``True``, wrap the plaintext connectable in a - :class:`~.streams.tls.TLSConnectable`, using the provided TLS settings) - :param ssl_context: if ``tls=True``, the SSLContext object to use (if not provided, - a secure default will be created) - :param tls_hostname: if ``tls=True``, host name of the server to use for checking - the server certificate (defaults to the host portion of the address for TCP - connectables) - :param tls_standard_compatible: if ``False`` and ``tls=True``, makes the TLS stream - skip the closing handshake when closing the connection, so it won't raise an - exception if the server does the same - - """ - connectable: TCPConnectable | UNIXConnectable | TLSConnectable - if isinstance(remote, ByteStreamConnectable): - return remote - elif isinstance(remote, tuple) and len(remote) == 2: - connectable = TCPConnectable(*remote) - elif isinstance(remote, (str, bytes, PathLike)): - connectable = UNIXConnectable(remote) - else: - raise TypeError(f"cannot convert {remote!r} to a connectable") - - if tls: - if not tls_hostname and isinstance(connectable, TCPConnectable): - tls_hostname = str(connectable.host) - - connectable = TLSConnectable( - connectable, - ssl_context=ssl_context, - hostname=tls_hostname, - standard_compatible=tls_standard_compatible, - ) - - return connectable diff --git a/functions/python/data-processor/anyio/_core/_streams.py b/functions/python/data-processor/anyio/_core/_streams.py deleted file mode 100644 index 2b9c7df20..000000000 --- a/functions/python/data-processor/anyio/_core/_streams.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import math -from typing import TypeVar -from warnings import warn - -from ..streams.memory import ( - MemoryObjectReceiveStream, - MemoryObjectSendStream, - _MemoryObjectStreamState, -) - -T_Item = TypeVar("T_Item") - - -class create_memory_object_stream( - tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]], -): - """ - Create a memory object stream. - - The stream's item type can be annotated like - :func:`create_memory_object_stream[T_Item]`. - - :param max_buffer_size: number of items held in the buffer until ``send()`` starts - blocking - :param item_type: old way of marking the streams with the right generic type for - static typing (does nothing on AnyIO 4) - - .. deprecated:: 4.0 - Use ``create_memory_object_stream[YourItemType](...)`` instead. - :return: a tuple of (send stream, receive stream) - - """ - - def __new__( # type: ignore[misc] - cls, max_buffer_size: float = 0, item_type: object = None - ) -> tuple[MemoryObjectSendStream[T_Item], MemoryObjectReceiveStream[T_Item]]: - if max_buffer_size != math.inf and not isinstance(max_buffer_size, int): - raise ValueError("max_buffer_size must be either an integer or math.inf") - if max_buffer_size < 0: - raise ValueError("max_buffer_size cannot be negative") - if item_type is not None: - warn( - "The item_type argument has been deprecated in AnyIO 4.0. " - "Use create_memory_object_stream[YourItemType](...) instead.", - DeprecationWarning, - stacklevel=2, - ) - - state = _MemoryObjectStreamState[T_Item](max_buffer_size) - return (MemoryObjectSendStream(state), MemoryObjectReceiveStream(state)) diff --git a/functions/python/data-processor/anyio/_core/_subprocesses.py b/functions/python/data-processor/anyio/_core/_subprocesses.py deleted file mode 100644 index 9796f8bb9..000000000 --- a/functions/python/data-processor/anyio/_core/_subprocesses.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncIterable, Iterable, Mapping, Sequence -from io import BytesIO -from os import PathLike -from subprocess import PIPE, CalledProcessError, CompletedProcess -from typing import IO, Any, TypeAlias, cast - -from ..abc import Process -from ._eventloop import get_async_backend -from ._tasks import create_task_group - -StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] - - -async def run_process( - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - input: bytes | None = None, - stdin: int | IO[Any] | None = None, - stdout: int | IO[Any] | None = PIPE, - stderr: int | IO[Any] | None = PIPE, - check: bool = True, - cwd: StrOrBytesPath | None = None, - env: Mapping[str, str] | None = None, - startupinfo: Any = None, - creationflags: int = 0, - start_new_session: bool = False, - pass_fds: Sequence[int] = (), - user: str | int | None = None, - group: str | int | None = None, - extra_groups: Iterable[str | int] | None = None, - umask: int = -1, -) -> CompletedProcess[bytes]: - """ - Run an external command in a subprocess and wait until it completes. - - .. seealso:: :func:`subprocess.run` - - :param command: either a string to pass to the shell, or an iterable of strings - containing the executable name or path and its arguments - :param input: bytes passed to the standard input of the subprocess - :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or `None`; ``input`` overrides this - :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or `None` - :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - :data:`subprocess.STDOUT`, a file-like object, or `None` - :param check: if ``True``, raise :exc:`~subprocess.CalledProcessError` if the - process terminates with a return code other than 0 - :param cwd: If not ``None``, change the working directory to this before running the - command - :param env: if not ``None``, this mapping replaces the inherited environment - variables from the parent process - :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used - to specify process startup parameters (Windows only) - :param creationflags: flags that can be used to control the creation of the - subprocess (see :class:`subprocess.Popen` for the specifics) - :param start_new_session: if ``true`` the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - :param pass_fds: sequence of file descriptors to keep open between the parent and - child processes. (POSIX only) - :param user: effective user to run the process as (Python >= 3.9, POSIX only) - :param group: effective group to run the process as (Python >= 3.9, POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (Python >= 3.9, - POSIX only) - :param umask: if not negative, this umask is applied in the child process before - running the given command (Python >= 3.9, POSIX only) - :return: an object representing the completed process - :raises ~subprocess.CalledProcessError: if ``check`` is ``True`` and the process - exits with a nonzero return code - - """ - - async def drain_stream(stream: AsyncIterable[bytes], index: int) -> None: - buffer = BytesIO() - async for chunk in stream: - buffer.write(chunk) - - stream_contents[index] = buffer.getvalue() - - if stdin is not None and input is not None: - raise ValueError("only one of stdin and input is allowed") - - async with await open_process( - command, - stdin=PIPE if input else stdin, - stdout=stdout, - stderr=stderr, - cwd=cwd, - env=env, - startupinfo=startupinfo, - creationflags=creationflags, - start_new_session=start_new_session, - pass_fds=pass_fds, - user=user, - group=group, - extra_groups=extra_groups, - umask=umask, - ) as process: - stream_contents: list[bytes | None] = [None, None] - async with create_task_group() as tg: - if process.stdout: - tg.start_soon(drain_stream, process.stdout, 0) - - if process.stderr: - tg.start_soon(drain_stream, process.stderr, 1) - - if process.stdin and input: - await process.stdin.send(input) - await process.stdin.aclose() - - await process.wait() - - output, errors = stream_contents - if check and process.returncode != 0: - raise CalledProcessError(cast(int, process.returncode), command, output, errors) - - return CompletedProcess(command, cast(int, process.returncode), output, errors) - - -async def open_process( - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None = PIPE, - stdout: int | IO[Any] | None = PIPE, - stderr: int | IO[Any] | None = PIPE, - cwd: StrOrBytesPath | None = None, - env: Mapping[str, str] | None = None, - startupinfo: Any = None, - creationflags: int = 0, - start_new_session: bool = False, - pass_fds: Sequence[int] = (), - user: str | int | None = None, - group: str | int | None = None, - extra_groups: Iterable[str | int] | None = None, - umask: int = -1, -) -> Process: - """ - Start an external command in a subprocess. - - .. seealso:: :class:`subprocess.Popen` - - :param command: either a string to pass to the shell, or an iterable of strings - containing the executable name or path and its arguments - :param stdin: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, a - file-like object, or ``None`` - :param stdout: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - a file-like object, or ``None`` - :param stderr: one of :data:`subprocess.PIPE`, :data:`subprocess.DEVNULL`, - :data:`subprocess.STDOUT`, a file-like object, or ``None`` - :param cwd: If not ``None``, the working directory is changed before executing - :param env: If env is not ``None``, it must be a mapping that defines the - environment variables for the new process - :param creationflags: flags that can be used to control the creation of the - subprocess (see :class:`subprocess.Popen` for the specifics) - :param startupinfo: an instance of :class:`subprocess.STARTUPINFO` that can be used - to specify process startup parameters (Windows only) - :param start_new_session: if ``true`` the setsid() system call will be made in the - child process prior to the execution of the subprocess. (POSIX only) - :param pass_fds: sequence of file descriptors to keep open between the parent and - child processes. (POSIX only) - :param user: effective user to run the process as (POSIX only) - :param group: effective group to run the process as (POSIX only) - :param extra_groups: supplementary groups to set in the subprocess (POSIX only) - :param umask: if not negative, this umask is applied in the child process before - running the given command (POSIX only) - :return: an asynchronous process object - - """ - kwargs: dict[str, Any] = {} - if user is not None: - kwargs["user"] = user - - if group is not None: - kwargs["group"] = group - - if extra_groups is not None: - kwargs["extra_groups"] = group - - if umask >= 0: - kwargs["umask"] = umask - - return await get_async_backend().open_process( - command, - stdin=stdin, - stdout=stdout, - stderr=stderr, - cwd=cwd, - env=env, - startupinfo=startupinfo, - creationflags=creationflags, - start_new_session=start_new_session, - pass_fds=pass_fds, - **kwargs, - ) diff --git a/functions/python/data-processor/anyio/_core/_synchronization.py b/functions/python/data-processor/anyio/_core/_synchronization.py deleted file mode 100644 index 9c6f9a072..000000000 --- a/functions/python/data-processor/anyio/_core/_synchronization.py +++ /dev/null @@ -1,757 +0,0 @@ -from __future__ import annotations - -import math -from collections import deque -from collections.abc import Callable -from dataclasses import dataclass -from types import TracebackType -from typing import TypeVar - -from ..lowlevel import checkpoint_if_cancelled -from ._eventloop import get_async_backend -from ._exceptions import BusyResourceError, NoEventLoopError -from ._tasks import CancelScope -from ._testing import TaskInfo, get_current_task - -T = TypeVar("T") - - -@dataclass(frozen=True) -class EventStatistics: - """ - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Event.wait` - """ - - tasks_waiting: int - - -@dataclass(frozen=True) -class CapacityLimiterStatistics: - """ - :ivar int borrowed_tokens: number of tokens currently borrowed by tasks - :ivar float total_tokens: total number of available tokens - :ivar tuple borrowers: tasks or other objects currently holding tokens borrowed from - this limiter - :ivar int tasks_waiting: number of tasks waiting on - :meth:`~.CapacityLimiter.acquire` or - :meth:`~.CapacityLimiter.acquire_on_behalf_of` - """ - - borrowed_tokens: int - total_tokens: float - borrowers: tuple[object, ...] - tasks_waiting: int - - -@dataclass(frozen=True) -class LockStatistics: - """ - :ivar bool locked: flag indicating if this lock is locked or not - :ivar ~anyio.TaskInfo owner: task currently holding the lock (or ``None`` if the - lock is not held by any task) - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Lock.acquire` - """ - - locked: bool - owner: TaskInfo | None - tasks_waiting: int - - -@dataclass(frozen=True) -class ConditionStatistics: - """ - :ivar int tasks_waiting: number of tasks blocked on :meth:`~.Condition.wait` - :ivar ~anyio.LockStatistics lock_statistics: statistics of the underlying - :class:`~.Lock` - """ - - tasks_waiting: int - lock_statistics: LockStatistics - - -@dataclass(frozen=True) -class SemaphoreStatistics: - """ - :ivar int tasks_waiting: number of tasks waiting on :meth:`~.Semaphore.acquire` - - """ - - tasks_waiting: int - - -class Event: - def __new__(cls) -> Event: - try: - return get_async_backend().create_event() - except NoEventLoopError: - return EventAdapter() - - def set(self) -> None: - """Set the flag, notifying all listeners.""" - raise NotImplementedError - - def is_set(self) -> bool: - """Return ``True`` if the flag is set, ``False`` if not.""" - raise NotImplementedError - - async def wait(self) -> None: - """ - Wait until the flag has been set. - - If the flag has already been set when this method is called, it returns - immediately. - - """ - raise NotImplementedError - - def statistics(self) -> EventStatistics: - """Return statistics about the current state of this event.""" - raise NotImplementedError - - -class EventAdapter(Event): - _internal_event: Event | None = None - _is_set: bool = False - - def __new__(cls) -> EventAdapter: - return object.__new__(cls) - - @property - def _event(self) -> Event: - if self._internal_event is None: - self._internal_event = get_async_backend().create_event() - if self._is_set: - self._internal_event.set() - - return self._internal_event - - def set(self) -> None: - if self._internal_event is None: - self._is_set = True - else: - self._event.set() - - def is_set(self) -> bool: - if self._internal_event is None: - return self._is_set - - return self._internal_event.is_set() - - async def wait(self) -> None: - await self._event.wait() - - def statistics(self) -> EventStatistics: - if self._internal_event is None: - return EventStatistics(tasks_waiting=0) - - return self._internal_event.statistics() - - -class Lock: - def __new__(cls, *, fast_acquire: bool = False) -> Lock: - try: - return get_async_backend().create_lock(fast_acquire=fast_acquire) - except NoEventLoopError: - return LockAdapter(fast_acquire=fast_acquire) - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - async def acquire(self) -> None: - """Acquire the lock.""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire the lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - raise NotImplementedError - - def release(self) -> None: - """Release the lock.""" - raise NotImplementedError - - def locked(self) -> bool: - """Return True if the lock is currently held.""" - raise NotImplementedError - - def statistics(self) -> LockStatistics: - """ - Return statistics about the current state of this lock. - - .. versionadded:: 3.0 - """ - raise NotImplementedError - - -class LockAdapter(Lock): - _internal_lock: Lock | None = None - - def __new__(cls, *, fast_acquire: bool = False) -> LockAdapter: - return object.__new__(cls) - - def __init__(self, *, fast_acquire: bool = False): - self._fast_acquire = fast_acquire - - @property - def _lock(self) -> Lock: - if self._internal_lock is None: - self._internal_lock = get_async_backend().create_lock( - fast_acquire=self._fast_acquire - ) - - return self._internal_lock - - async def __aenter__(self) -> None: - await self._lock.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - if self._internal_lock is not None: - self._internal_lock.release() - - async def acquire(self) -> None: - """Acquire the lock.""" - await self._lock.acquire() - - def acquire_nowait(self) -> None: - """ - Acquire the lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - self._lock.acquire_nowait() - - def release(self) -> None: - """Release the lock.""" - self._lock.release() - - def locked(self) -> bool: - """Return True if the lock is currently held.""" - return self._lock.locked() - - def statistics(self) -> LockStatistics: - """ - Return statistics about the current state of this lock. - - .. versionadded:: 3.0 - - """ - if self._internal_lock is None: - return LockStatistics(False, None, 0) - - return self._internal_lock.statistics() - - -class Condition: - _owner_task: TaskInfo | None = None - - def __init__(self, lock: Lock | None = None): - self._lock = lock or Lock() - self._waiters: deque[Event] = deque() - - async def __aenter__(self) -> None: - await self.acquire() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - def _check_acquired(self) -> None: - if self._owner_task != get_current_task(): - raise RuntimeError("The current task is not holding the underlying lock") - - async def acquire(self) -> None: - """Acquire the underlying lock.""" - await self._lock.acquire() - self._owner_task = get_current_task() - - def acquire_nowait(self) -> None: - """ - Acquire the underlying lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - self._lock.acquire_nowait() - self._owner_task = get_current_task() - - def release(self) -> None: - """Release the underlying lock.""" - self._lock.release() - - def locked(self) -> bool: - """Return True if the lock is set.""" - return self._lock.locked() - - def notify(self, n: int = 1) -> None: - """Notify exactly n listeners.""" - self._check_acquired() - for _ in range(n): - try: - event = self._waiters.popleft() - except IndexError: - break - - event.set() - - def notify_all(self) -> None: - """Notify all the listeners.""" - self._check_acquired() - for event in self._waiters: - event.set() - - self._waiters.clear() - - async def wait(self) -> None: - """Wait for a notification.""" - await checkpoint_if_cancelled() - self._check_acquired() - event = Event() - self._waiters.append(event) - self.release() - try: - await event.wait() - except BaseException: - if not event.is_set(): - self._waiters.remove(event) - elif self._waiters: - # This task was notified by could not act on it, so pass - # it on to the next task - self._waiters.popleft().set() - - raise - finally: - with CancelScope(shield=True): - await self.acquire() - - async def wait_for(self, predicate: Callable[[], T]) -> T: - """ - Wait until a predicate becomes true. - - :param predicate: a callable that returns a truthy value when the condition is - met - :return: the result of the predicate - - .. versionadded:: 4.11.0 - - """ - while not (result := predicate()): - await self.wait() - - return result - - def statistics(self) -> ConditionStatistics: - """ - Return statistics about the current state of this condition. - - .. versionadded:: 3.0 - """ - return ConditionStatistics(len(self._waiters), self._lock.statistics()) - - -class Semaphore: - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - try: - return get_async_backend().create_semaphore( - initial_value, max_value=max_value, fast_acquire=fast_acquire - ) - except NoEventLoopError: - return SemaphoreAdapter(initial_value, max_value=max_value) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ): - if not isinstance(initial_value, int): - raise TypeError("initial_value must be an integer") - if initial_value < 0: - raise ValueError("initial_value must be >= 0") - if max_value is not None: - if not isinstance(max_value, int): - raise TypeError("max_value must be an integer or None") - if max_value < initial_value: - raise ValueError( - "max_value must be equal to or higher than initial_value" - ) - - self._fast_acquire = fast_acquire - - async def __aenter__(self) -> Semaphore: - await self.acquire() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.release() - - async def acquire(self) -> None: - """Decrement the semaphore value, blocking if necessary.""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire the underlying lock, without blocking. - - :raises ~anyio.WouldBlock: if the operation would block - - """ - raise NotImplementedError - - def release(self) -> None: - """Increment the semaphore value.""" - raise NotImplementedError - - @property - def value(self) -> int: - """The current value of the semaphore.""" - raise NotImplementedError - - @property - def max_value(self) -> int | None: - """The maximum value of the semaphore.""" - raise NotImplementedError - - def statistics(self) -> SemaphoreStatistics: - """ - Return statistics about the current state of this semaphore. - - .. versionadded:: 3.0 - """ - raise NotImplementedError - - -class SemaphoreAdapter(Semaphore): - _internal_semaphore: Semaphore | None = None - - def __new__( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> SemaphoreAdapter: - return object.__new__(cls) - - def __init__( - self, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> None: - super().__init__(initial_value, max_value=max_value, fast_acquire=fast_acquire) - self._initial_value = initial_value - self._max_value = max_value - - @property - def _semaphore(self) -> Semaphore: - if self._internal_semaphore is None: - self._internal_semaphore = get_async_backend().create_semaphore( - self._initial_value, max_value=self._max_value - ) - - return self._internal_semaphore - - async def acquire(self) -> None: - await self._semaphore.acquire() - - def acquire_nowait(self) -> None: - self._semaphore.acquire_nowait() - - def release(self) -> None: - self._semaphore.release() - - @property - def value(self) -> int: - if self._internal_semaphore is None: - return self._initial_value - - return self._semaphore.value - - @property - def max_value(self) -> int | None: - return self._max_value - - def statistics(self) -> SemaphoreStatistics: - if self._internal_semaphore is None: - return SemaphoreStatistics(tasks_waiting=0) - - return self._semaphore.statistics() - - -class CapacityLimiter: - def __new__(cls, total_tokens: float) -> CapacityLimiter: - try: - return get_async_backend().create_capacity_limiter(total_tokens) - except NoEventLoopError: - return CapacityLimiterAdapter(total_tokens) - - async def __aenter__(self) -> None: - raise NotImplementedError - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - raise NotImplementedError - - @property - def total_tokens(self) -> float: - """ - The total number of tokens available for borrowing. - - This is a read-write property. If the total number of tokens is increased, the - proportionate number of tasks waiting on this limiter will be granted their - tokens. - - .. versionchanged:: 3.0 - The property is now writable. - .. versionchanged:: 4.12 - The value can now be set to 0. - - """ - raise NotImplementedError - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - raise NotImplementedError - - @property - def borrowed_tokens(self) -> int: - """The number of tokens that have currently been borrowed.""" - raise NotImplementedError - - @property - def available_tokens(self) -> float: - """The number of tokens currently available to be borrowed""" - raise NotImplementedError - - def acquire_nowait(self) -> None: - """ - Acquire a token for the current task without waiting for one to become - available. - - :raises ~anyio.WouldBlock: if there are no tokens available for borrowing - - """ - raise NotImplementedError - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - """ - Acquire a token without waiting for one to become available. - - :param borrower: the entity borrowing a token - :raises ~anyio.WouldBlock: if there are no tokens available for borrowing - - """ - raise NotImplementedError - - async def acquire(self) -> None: - """ - Acquire a token for the current task, waiting if necessary for one to become - available. - - """ - raise NotImplementedError - - async def acquire_on_behalf_of(self, borrower: object) -> None: - """ - Acquire a token, waiting if necessary for one to become available. - - :param borrower: the entity borrowing a token - - """ - raise NotImplementedError - - def release(self) -> None: - """ - Release the token held by the current task. - - :raises RuntimeError: if the current task has not borrowed a token from this - limiter. - - """ - raise NotImplementedError - - def release_on_behalf_of(self, borrower: object) -> None: - """ - Release the token held by the given borrower. - - :raises RuntimeError: if the borrower has not borrowed a token from this - limiter. - - """ - raise NotImplementedError - - def statistics(self) -> CapacityLimiterStatistics: - """ - Return statistics about the current state of this limiter. - - .. versionadded:: 3.0 - - """ - raise NotImplementedError - - -class CapacityLimiterAdapter(CapacityLimiter): - _internal_limiter: CapacityLimiter | None = None - - def __new__(cls, total_tokens: float) -> CapacityLimiterAdapter: - return object.__new__(cls) - - def __init__(self, total_tokens: float) -> None: - self.total_tokens = total_tokens - - @property - def _limiter(self) -> CapacityLimiter: - if self._internal_limiter is None: - self._internal_limiter = get_async_backend().create_capacity_limiter( - self._total_tokens - ) - - return self._internal_limiter - - async def __aenter__(self) -> None: - await self._limiter.__aenter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - return await self._limiter.__aexit__(exc_type, exc_val, exc_tb) - - @property - def total_tokens(self) -> float: - if self._internal_limiter is None: - return self._total_tokens - - return self._internal_limiter.total_tokens - - @total_tokens.setter - def total_tokens(self, value: float) -> None: - if not isinstance(value, int) and value is not math.inf: - raise TypeError("total_tokens must be an int or math.inf") - elif value < 1: - raise ValueError("total_tokens must be >= 1") - - if self._internal_limiter is None: - self._total_tokens = value - return - - self._limiter.total_tokens = value - - @property - def borrowed_tokens(self) -> int: - if self._internal_limiter is None: - return 0 - - return self._internal_limiter.borrowed_tokens - - @property - def available_tokens(self) -> float: - if self._internal_limiter is None: - return self._total_tokens - - return self._internal_limiter.available_tokens - - def acquire_nowait(self) -> None: - self._limiter.acquire_nowait() - - def acquire_on_behalf_of_nowait(self, borrower: object) -> None: - self._limiter.acquire_on_behalf_of_nowait(borrower) - - async def acquire(self) -> None: - await self._limiter.acquire() - - async def acquire_on_behalf_of(self, borrower: object) -> None: - await self._limiter.acquire_on_behalf_of(borrower) - - def release(self) -> None: - self._limiter.release() - - def release_on_behalf_of(self, borrower: object) -> None: - self._limiter.release_on_behalf_of(borrower) - - def statistics(self) -> CapacityLimiterStatistics: - if self._internal_limiter is None: - return CapacityLimiterStatistics( - borrowed_tokens=0, - total_tokens=self.total_tokens, - borrowers=(), - tasks_waiting=0, - ) - - return self._internal_limiter.statistics() - - -class ResourceGuard: - """ - A context manager for ensuring that a resource is only used by a single task at a - time. - - Entering this context manager while the previous has not exited it yet will trigger - :exc:`BusyResourceError`. - - :param action: the action to guard against (visible in the :exc:`BusyResourceError` - when triggered, e.g. "Another task is already {action} this resource") - - .. versionadded:: 4.1 - """ - - __slots__ = "action", "_guarded" - - def __init__(self, action: str = "using"): - self.action: str = action - self._guarded = False - - def __enter__(self) -> None: - if self._guarded: - raise BusyResourceError(self.action) - - self._guarded = True - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._guarded = False diff --git a/functions/python/data-processor/anyio/_core/_tasks.py b/functions/python/data-processor/anyio/_core/_tasks.py deleted file mode 100644 index 0688bfe96..000000000 --- a/functions/python/data-processor/anyio/_core/_tasks.py +++ /dev/null @@ -1,173 +0,0 @@ -from __future__ import annotations - -import math -from collections.abc import Generator -from contextlib import contextmanager -from types import TracebackType - -from ..abc._tasks import TaskGroup, TaskStatus -from ._eventloop import get_async_backend - - -class _IgnoredTaskStatus(TaskStatus[object]): - def started(self, value: object = None) -> None: - pass - - -TASK_STATUS_IGNORED = _IgnoredTaskStatus() - - -class CancelScope: - """ - Wraps a unit of work that can be made separately cancellable. - - :param deadline: The time (clock value) when this scope is cancelled automatically - :param shield: ``True`` to shield the cancel scope from external cancellation - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - """ - - def __new__( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - return get_async_backend().create_cancel_scope(shield=shield, deadline=deadline) - - def cancel(self, reason: str | None = None) -> None: - """ - Cancel this scope immediately. - - :param reason: a message describing the reason for the cancellation - - """ - raise NotImplementedError - - @property - def deadline(self) -> float: - """ - The time (clock value) when this scope is cancelled automatically. - - Will be ``float('inf')`` if no timeout has been set. - - """ - raise NotImplementedError - - @deadline.setter - def deadline(self, value: float) -> None: - raise NotImplementedError - - @property - def cancel_called(self) -> bool: - """``True`` if :meth:`cancel` has been called.""" - raise NotImplementedError - - @property - def cancelled_caught(self) -> bool: - """ - ``True`` if this scope suppressed a cancellation exception it itself raised. - - This is typically used to check if any work was interrupted, or to see if the - scope was cancelled due to its deadline being reached. The value will, however, - only be ``True`` if the cancellation was triggered by the scope itself (and not - an outer scope). - - """ - raise NotImplementedError - - @property - def shield(self) -> bool: - """ - ``True`` if this scope is shielded from external cancellation. - - While a scope is shielded, it will not receive cancellations from outside. - - """ - raise NotImplementedError - - @shield.setter - def shield(self, value: bool) -> None: - raise NotImplementedError - - def __enter__(self) -> CancelScope: - raise NotImplementedError - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - raise NotImplementedError - - -@contextmanager -def fail_after( - delay: float | None, shield: bool = False -) -> Generator[CancelScope, None, None]: - """ - Create a context manager which raises a :class:`TimeoutError` if does not finish in - time. - - :param delay: maximum allowed time (in seconds) before raising the exception, or - ``None`` to disable the timeout - :param shield: ``True`` to shield the cancel scope from external cancellation - :return: a context manager that yields a cancel scope - :rtype: :class:`~typing.ContextManager`\\[:class:`~anyio.CancelScope`\\] - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - current_time = get_async_backend().current_time - deadline = (current_time() + delay) if delay is not None else math.inf - with get_async_backend().create_cancel_scope( - deadline=deadline, shield=shield - ) as cancel_scope: - yield cancel_scope - - if cancel_scope.cancelled_caught and current_time() >= cancel_scope.deadline: - raise TimeoutError - - -def move_on_after(delay: float | None, shield: bool = False) -> CancelScope: - """ - Create a cancel scope with a deadline that expires after the given delay. - - :param delay: maximum allowed time (in seconds) before exiting the context block, or - ``None`` to disable the timeout - :param shield: ``True`` to shield the cancel scope from external cancellation - :return: a cancel scope - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - deadline = ( - (get_async_backend().current_time() + delay) if delay is not None else math.inf - ) - return get_async_backend().create_cancel_scope(deadline=deadline, shield=shield) - - -def current_effective_deadline() -> float: - """ - Return the nearest deadline among all the cancel scopes effective for the current - task. - - :return: a clock value from the event loop's internal clock (or ``float('inf')`` if - there is no deadline in effect, or ``float('-inf')`` if the current scope has - been cancelled) - :rtype: float - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().current_effective_deadline() - - -def create_task_group() -> TaskGroup: - """ - Create a task group. - - :return: a task group - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().create_task_group() diff --git a/functions/python/data-processor/anyio/_core/_tempfile.py b/functions/python/data-processor/anyio/_core/_tempfile.py deleted file mode 100644 index 75a09f793..000000000 --- a/functions/python/data-processor/anyio/_core/_tempfile.py +++ /dev/null @@ -1,613 +0,0 @@ -from __future__ import annotations - -import os -import sys -import tempfile -from collections.abc import Iterable -from io import BytesIO, TextIOWrapper -from types import TracebackType -from typing import ( - TYPE_CHECKING, - Any, - AnyStr, - Generic, - overload, -) - -from .. import to_thread -from .._core._fileio import AsyncFile -from ..lowlevel import checkpoint_if_cancelled - -if TYPE_CHECKING: - from _typeshed import OpenBinaryMode, OpenTextMode, ReadableBuffer, WriteableBuffer - - -class TemporaryFile(Generic[AnyStr]): - """ - An asynchronous temporary file that is automatically created and cleaned up. - - This class provides an asynchronous context manager interface to a temporary file. - The file is created using Python's standard `tempfile.TemporaryFile` function in a - background thread, and is wrapped as an asynchronous file using `AsyncFile`. - - :param mode: The mode in which the file is opened. Defaults to "w+b". - :param buffering: The buffering policy (-1 means the default buffering). - :param encoding: The encoding used to decode or encode the file. Only applicable in - text mode. - :param newline: Controls how universal newlines mode works (only applicable in text - mode). - :param suffix: The suffix for the temporary file name. - :param prefix: The prefix for the temporary file name. - :param dir: The directory in which the temporary file is created. - :param errors: The error handling scheme used for encoding/decoding errors. - """ - - _async_file: AsyncFile[AnyStr] - - @overload - def __init__( - self: TemporaryFile[bytes], - mode: OpenBinaryMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - @overload - def __init__( - self: TemporaryFile[str], - mode: OpenTextMode, - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - - def __init__( - self, - mode: OpenTextMode | OpenBinaryMode = "w+b", - buffering: int = -1, - encoding: str | None = None, - newline: str | None = None, - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - *, - errors: str | None = None, - ) -> None: - self.mode = mode - self.buffering = buffering - self.encoding = encoding - self.newline = newline - self.suffix: str | None = suffix - self.prefix: str | None = prefix - self.dir: str | None = dir - self.errors = errors - - async def __aenter__(self) -> AsyncFile[AnyStr]: - fp = await to_thread.run_sync( - lambda: tempfile.TemporaryFile( - self.mode, - self.buffering, - self.encoding, - self.newline, - self.suffix, - self.prefix, - self.dir, - errors=self.errors, - ) - ) - self._async_file = AsyncFile(fp) - return self._async_file - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self._async_file.aclose() - - -class NamedTemporaryFile(Generic[AnyStr]): - """ - An asynchronous named temporary file that is automatically created and cleaned up. - - This class provides an asynchronous context manager for a temporary file with a - visible name in the file system. It uses Python's standard - :func:`~tempfile.NamedTemporaryFile` function and wraps the file object with - :class:`AsyncFile` for asynchronous operations. - - :param mode: The mode in which the file is opened. Defaults to "w+b". - :param buffering: The buffering policy (-1 means the default buffering). - :param encoding: The encoding used to decode or encode the file. Only applicable in - text mode. - :param newline: Controls how universal newlines mode works (only applicable in text - mode). - :param suffix: The suffix for the temporary file name. - :param prefix: The prefix for the temporary file name. - :param dir: The directory in which the temporary file is created. - :param delete: Whether to delete the file when it is closed. - :param errors: The error handling scheme used for encoding/decoding errors. - :param delete_on_close: (Python 3.12+) Whether to delete the file on close. - """ - - _async_file: AsyncFile[AnyStr] - - @overload - def __init__( - self: NamedTemporaryFile[bytes], - mode: OpenBinaryMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - delete: bool = ..., - *, - errors: str | None = ..., - delete_on_close: bool = ..., - ): ... - @overload - def __init__( - self: NamedTemporaryFile[str], - mode: OpenTextMode, - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - delete: bool = ..., - *, - errors: str | None = ..., - delete_on_close: bool = ..., - ): ... - - def __init__( - self, - mode: OpenBinaryMode | OpenTextMode = "w+b", - buffering: int = -1, - encoding: str | None = None, - newline: str | None = None, - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - delete: bool = True, - *, - errors: str | None = None, - delete_on_close: bool = True, - ) -> None: - self._params: dict[str, Any] = { - "mode": mode, - "buffering": buffering, - "encoding": encoding, - "newline": newline, - "suffix": suffix, - "prefix": prefix, - "dir": dir, - "delete": delete, - "errors": errors, - } - if sys.version_info >= (3, 12): - self._params["delete_on_close"] = delete_on_close - - async def __aenter__(self) -> AsyncFile[AnyStr]: - fp = await to_thread.run_sync( - lambda: tempfile.NamedTemporaryFile(**self._params) - ) - self._async_file = AsyncFile(fp) - return self._async_file - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - await self._async_file.aclose() - - -class SpooledTemporaryFile(AsyncFile[AnyStr]): - """ - An asynchronous spooled temporary file that starts in memory and is spooled to disk. - - This class provides an asynchronous interface to a spooled temporary file, much like - Python's standard :class:`~tempfile.SpooledTemporaryFile`. It supports asynchronous - write operations and provides a method to force a rollover to disk. - - :param max_size: Maximum size in bytes before the file is rolled over to disk. - :param mode: The mode in which the file is opened. Defaults to "w+b". - :param buffering: The buffering policy (-1 means the default buffering). - :param encoding: The encoding used to decode or encode the file (text mode only). - :param newline: Controls how universal newlines mode works (text mode only). - :param suffix: The suffix for the temporary file name. - :param prefix: The prefix for the temporary file name. - :param dir: The directory in which the temporary file is created. - :param errors: The error handling scheme used for encoding/decoding errors. - """ - - _rolled: bool = False - - @overload - def __init__( - self: SpooledTemporaryFile[bytes], - max_size: int = ..., - mode: OpenBinaryMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - @overload - def __init__( - self: SpooledTemporaryFile[str], - max_size: int = ..., - mode: OpenTextMode = ..., - buffering: int = ..., - encoding: str | None = ..., - newline: str | None = ..., - suffix: str | None = ..., - prefix: str | None = ..., - dir: str | None = ..., - *, - errors: str | None = ..., - ): ... - - def __init__( - self, - max_size: int = 0, - mode: OpenBinaryMode | OpenTextMode = "w+b", - buffering: int = -1, - encoding: str | None = None, - newline: str | None = None, - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - *, - errors: str | None = None, - ) -> None: - self._tempfile_params: dict[str, Any] = { - "mode": mode, - "buffering": buffering, - "encoding": encoding, - "newline": newline, - "suffix": suffix, - "prefix": prefix, - "dir": dir, - "errors": errors, - } - self._max_size = max_size - if "b" in mode: - super().__init__(BytesIO()) # type: ignore[arg-type] - else: - super().__init__( - TextIOWrapper( # type: ignore[arg-type] - BytesIO(), - encoding=encoding, - errors=errors, - newline=newline, - write_through=True, - ) - ) - - async def aclose(self) -> None: - if not self._rolled: - self._fp.close() - return - - await super().aclose() - - async def _check(self) -> None: - if self._rolled or self._fp.tell() <= self._max_size: - return - - await self.rollover() - - async def rollover(self) -> None: - if self._rolled: - return - - self._rolled = True - buffer = self._fp - buffer.seek(0) - self._fp = await to_thread.run_sync( - lambda: tempfile.TemporaryFile(**self._tempfile_params) - ) - await self.write(buffer.read()) - buffer.close() - - @property - def closed(self) -> bool: - return self._fp.closed - - async def read(self, size: int = -1) -> AnyStr: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.read(size) - - return await super().read(size) # type: ignore[return-value] - - async def read1(self: SpooledTemporaryFile[bytes], size: int = -1) -> bytes: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.read1(size) - - return await super().read1(size) - - async def readline(self) -> AnyStr: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.readline() - - return await super().readline() # type: ignore[return-value] - - async def readlines(self) -> list[AnyStr]: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.readlines() - - return await super().readlines() # type: ignore[return-value] - - async def readinto(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - self._fp.readinto(b) - - return await super().readinto(b) - - async def readinto1(self: SpooledTemporaryFile[bytes], b: WriteableBuffer) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - self._fp.readinto(b) - - return await super().readinto1(b) - - async def seek(self, offset: int, whence: int | None = os.SEEK_SET) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.seek(offset, whence) - - return await super().seek(offset, whence) - - async def tell(self) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.tell() - - return await super().tell() - - async def truncate(self, size: int | None = None) -> int: - if not self._rolled: - await checkpoint_if_cancelled() - return self._fp.truncate(size) - - return await super().truncate(size) - - @overload - async def write(self: SpooledTemporaryFile[bytes], b: ReadableBuffer) -> int: ... - @overload - async def write(self: SpooledTemporaryFile[str], b: str) -> int: ... - - async def write(self, b: ReadableBuffer | str) -> int: - """ - Asynchronously write data to the spooled temporary file. - - If the file has not yet been rolled over, the data is written synchronously, - and a rollover is triggered if the size exceeds the maximum size. - - :param s: The data to write. - :return: The number of bytes written. - :raises RuntimeError: If the underlying file is not initialized. - - """ - if not self._rolled: - await checkpoint_if_cancelled() - result = self._fp.write(b) - await self._check() - return result - - return await super().write(b) # type: ignore[misc] - - @overload - async def writelines( - self: SpooledTemporaryFile[bytes], lines: Iterable[ReadableBuffer] - ) -> None: ... - @overload - async def writelines( - self: SpooledTemporaryFile[str], lines: Iterable[str] - ) -> None: ... - - async def writelines(self, lines: Iterable[str] | Iterable[ReadableBuffer]) -> None: - """ - Asynchronously write a list of lines to the spooled temporary file. - - If the file has not yet been rolled over, the lines are written synchronously, - and a rollover is triggered if the size exceeds the maximum size. - - :param lines: An iterable of lines to write. - :raises RuntimeError: If the underlying file is not initialized. - - """ - if not self._rolled: - await checkpoint_if_cancelled() - result = self._fp.writelines(lines) - await self._check() - return result - - return await super().writelines(lines) # type: ignore[misc] - - -class TemporaryDirectory(Generic[AnyStr]): - """ - An asynchronous temporary directory that is created and cleaned up automatically. - - This class provides an asynchronous context manager for creating a temporary - directory. It wraps Python's standard :class:`~tempfile.TemporaryDirectory` to - perform directory creation and cleanup operations in a background thread. - - :param suffix: Suffix to be added to the temporary directory name. - :param prefix: Prefix to be added to the temporary directory name. - :param dir: The parent directory where the temporary directory is created. - :param ignore_cleanup_errors: Whether to ignore errors during cleanup - :param delete: Whether to delete the directory upon closing (Python 3.12+). - """ - - def __init__( - self, - suffix: AnyStr | None = None, - prefix: AnyStr | None = None, - dir: AnyStr | None = None, - *, - ignore_cleanup_errors: bool = False, - delete: bool = True, - ) -> None: - self.suffix: AnyStr | None = suffix - self.prefix: AnyStr | None = prefix - self.dir: AnyStr | None = dir - self.ignore_cleanup_errors = ignore_cleanup_errors - self.delete = delete - - self._tempdir: tempfile.TemporaryDirectory | None = None - - async def __aenter__(self) -> str: - params: dict[str, Any] = { - "suffix": self.suffix, - "prefix": self.prefix, - "dir": self.dir, - "ignore_cleanup_errors": self.ignore_cleanup_errors, - } - if sys.version_info >= (3, 12): - params["delete"] = self.delete - - self._tempdir = await to_thread.run_sync( - lambda: tempfile.TemporaryDirectory(**params) - ) - return await to_thread.run_sync(self._tempdir.__enter__) - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - if self._tempdir is not None: - await to_thread.run_sync( - self._tempdir.__exit__, exc_type, exc_value, traceback - ) - - async def cleanup(self) -> None: - if self._tempdir is not None: - await to_thread.run_sync(self._tempdir.cleanup) - - -@overload -async def mkstemp( - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, - text: bool = False, -) -> tuple[int, str]: ... - - -@overload -async def mkstemp( - suffix: bytes | None = None, - prefix: bytes | None = None, - dir: bytes | None = None, - text: bool = False, -) -> tuple[int, bytes]: ... - - -async def mkstemp( - suffix: AnyStr | None = None, - prefix: AnyStr | None = None, - dir: AnyStr | None = None, - text: bool = False, -) -> tuple[int, str | bytes]: - """ - Asynchronously create a temporary file and return an OS-level handle and the file - name. - - This function wraps `tempfile.mkstemp` and executes it in a background thread. - - :param suffix: Suffix to be added to the file name. - :param prefix: Prefix to be added to the file name. - :param dir: Directory in which the temporary file is created. - :param text: Whether the file is opened in text mode. - :return: A tuple containing the file descriptor and the file name. - - """ - return await to_thread.run_sync(tempfile.mkstemp, suffix, prefix, dir, text) - - -@overload -async def mkdtemp( - suffix: str | None = None, - prefix: str | None = None, - dir: str | None = None, -) -> str: ... - - -@overload -async def mkdtemp( - suffix: bytes | None = None, - prefix: bytes | None = None, - dir: bytes | None = None, -) -> bytes: ... - - -async def mkdtemp( - suffix: AnyStr | None = None, - prefix: AnyStr | None = None, - dir: AnyStr | None = None, -) -> str | bytes: - """ - Asynchronously create a temporary directory and return its path. - - This function wraps `tempfile.mkdtemp` and executes it in a background thread. - - :param suffix: Suffix to be added to the directory name. - :param prefix: Prefix to be added to the directory name. - :param dir: Parent directory where the temporary directory is created. - :return: The path of the created temporary directory. - - """ - return await to_thread.run_sync(tempfile.mkdtemp, suffix, prefix, dir) - - -async def gettempdir() -> str: - """ - Asynchronously return the name of the directory used for temporary files. - - This function wraps `tempfile.gettempdir` and executes it in a background thread. - - :return: The path of the temporary directory as a string. - - """ - return await to_thread.run_sync(tempfile.gettempdir) - - -async def gettempdirb() -> bytes: - """ - Asynchronously return the name of the directory used for temporary files in bytes. - - This function wraps `tempfile.gettempdirb` and executes it in a background thread. - - :return: The path of the temporary directory as bytes. - - """ - return await to_thread.run_sync(tempfile.gettempdirb) diff --git a/functions/python/data-processor/anyio/_core/_testing.py b/functions/python/data-processor/anyio/_core/_testing.py deleted file mode 100644 index 369e65c06..000000000 --- a/functions/python/data-processor/anyio/_core/_testing.py +++ /dev/null @@ -1,82 +0,0 @@ -from __future__ import annotations - -from collections.abc import Awaitable, Generator -from typing import Any, cast - -from ._eventloop import get_async_backend - - -class TaskInfo: - """ - Represents an asynchronous task. - - :ivar int id: the unique identifier of the task - :ivar parent_id: the identifier of the parent task, if any - :vartype parent_id: Optional[int] - :ivar str name: the description of the task (if any) - :ivar ~collections.abc.Coroutine coro: the coroutine object of the task - """ - - __slots__ = "_name", "id", "parent_id", "name", "coro" - - def __init__( - self, - id: int, - parent_id: int | None, - name: str | None, - coro: Generator[Any, Any, Any] | Awaitable[Any], - ): - func = get_current_task - self._name = f"{func.__module__}.{func.__qualname__}" - self.id: int = id - self.parent_id: int | None = parent_id - self.name: str | None = name - self.coro: Generator[Any, Any, Any] | Awaitable[Any] = coro - - def __eq__(self, other: object) -> bool: - if isinstance(other, TaskInfo): - return self.id == other.id - - return NotImplemented - - def __hash__(self) -> int: - return hash(self.id) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(id={self.id!r}, name={self.name!r})" - - def has_pending_cancellation(self) -> bool: - """ - Return ``True`` if the task has a cancellation pending, ``False`` otherwise. - - """ - return False - - -def get_current_task() -> TaskInfo: - """ - Return the current task. - - :return: a representation of the current task - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().get_current_task() - - -def get_running_tasks() -> list[TaskInfo]: - """ - Return a list of running tasks in the current event loop. - - :return: a list of task info objects - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return cast("list[TaskInfo]", get_async_backend().get_running_tasks()) - - -async def wait_all_tasks_blocked() -> None: - """Wait until all other tasks are waiting for something.""" - await get_async_backend().wait_all_tasks_blocked() diff --git a/functions/python/data-processor/anyio/_core/_typedattr.py b/functions/python/data-processor/anyio/_core/_typedattr.py deleted file mode 100644 index f358a448c..000000000 --- a/functions/python/data-processor/anyio/_core/_typedattr.py +++ /dev/null @@ -1,81 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping -from typing import Any, TypeVar, final, overload - -from ._exceptions import TypedAttributeLookupError - -T_Attr = TypeVar("T_Attr") -T_Default = TypeVar("T_Default") -undefined = object() - - -def typed_attribute() -> Any: - """Return a unique object, used to mark typed attributes.""" - return object() - - -class TypedAttributeSet: - """ - Superclass for typed attribute collections. - - Checks that every public attribute of every subclass has a type annotation. - """ - - def __init_subclass__(cls) -> None: - annotations: dict[str, Any] = getattr(cls, "__annotations__", {}) - for attrname in dir(cls): - if not attrname.startswith("_") and attrname not in annotations: - raise TypeError( - f"Attribute {attrname!r} is missing its type annotation" - ) - - super().__init_subclass__() - - -class TypedAttributeProvider: - """Base class for classes that wish to provide typed extra attributes.""" - - @property - def extra_attributes(self) -> Mapping[T_Attr, Callable[[], T_Attr]]: - """ - A mapping of the extra attributes to callables that return the corresponding - values. - - If the provider wraps another provider, the attributes from that wrapper should - also be included in the returned mapping (but the wrapper may override the - callables from the wrapped instance). - - """ - return {} - - @overload - def extra(self, attribute: T_Attr) -> T_Attr: ... - - @overload - def extra(self, attribute: T_Attr, default: T_Default) -> T_Attr | T_Default: ... - - @final - def extra(self, attribute: Any, default: object = undefined) -> object: - """ - extra(attribute, default=undefined) - - Return the value of the given typed extra attribute. - - :param attribute: the attribute (member of a :class:`~TypedAttributeSet`) to - look for - :param default: the value that should be returned if no value is found for the - attribute - :raises ~anyio.TypedAttributeLookupError: if the search failed and no default - value was given - - """ - try: - getter = self.extra_attributes[attribute] - except KeyError: - if default is undefined: - raise TypedAttributeLookupError("Attribute not found") from None - else: - return default - - return getter() diff --git a/functions/python/data-processor/anyio/abc/__init__.py b/functions/python/data-processor/anyio/abc/__init__.py deleted file mode 100644 index d560ce3f1..000000000 --- a/functions/python/data-processor/anyio/abc/__init__.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -from ._eventloop import AsyncBackend as AsyncBackend -from ._resources import AsyncResource as AsyncResource -from ._sockets import ConnectedUDPSocket as ConnectedUDPSocket -from ._sockets import ConnectedUNIXDatagramSocket as ConnectedUNIXDatagramSocket -from ._sockets import IPAddressType as IPAddressType -from ._sockets import IPSockAddrType as IPSockAddrType -from ._sockets import SocketAttribute as SocketAttribute -from ._sockets import SocketListener as SocketListener -from ._sockets import SocketStream as SocketStream -from ._sockets import UDPPacketType as UDPPacketType -from ._sockets import UDPSocket as UDPSocket -from ._sockets import UNIXDatagramPacketType as UNIXDatagramPacketType -from ._sockets import UNIXDatagramSocket as UNIXDatagramSocket -from ._sockets import UNIXSocketStream as UNIXSocketStream -from ._streams import AnyByteReceiveStream as AnyByteReceiveStream -from ._streams import AnyByteSendStream as AnyByteSendStream -from ._streams import AnyByteStream as AnyByteStream -from ._streams import AnyByteStreamConnectable as AnyByteStreamConnectable -from ._streams import AnyUnreliableByteReceiveStream as AnyUnreliableByteReceiveStream -from ._streams import AnyUnreliableByteSendStream as AnyUnreliableByteSendStream -from ._streams import AnyUnreliableByteStream as AnyUnreliableByteStream -from ._streams import ByteReceiveStream as ByteReceiveStream -from ._streams import ByteSendStream as ByteSendStream -from ._streams import ByteStream as ByteStream -from ._streams import ByteStreamConnectable as ByteStreamConnectable -from ._streams import Listener as Listener -from ._streams import ObjectReceiveStream as ObjectReceiveStream -from ._streams import ObjectSendStream as ObjectSendStream -from ._streams import ObjectStream as ObjectStream -from ._streams import ObjectStreamConnectable as ObjectStreamConnectable -from ._streams import UnreliableObjectReceiveStream as UnreliableObjectReceiveStream -from ._streams import UnreliableObjectSendStream as UnreliableObjectSendStream -from ._streams import UnreliableObjectStream as UnreliableObjectStream -from ._subprocesses import Process as Process -from ._tasks import TaskGroup as TaskGroup -from ._tasks import TaskStatus as TaskStatus -from ._testing import TestRunner as TestRunner - -# Re-exported here, for backwards compatibility -# isort: off -from .._core._synchronization import ( - CapacityLimiter as CapacityLimiter, - Condition as Condition, - Event as Event, - Lock as Lock, - Semaphore as Semaphore, -) -from .._core._tasks import CancelScope as CancelScope -from ..from_thread import BlockingPortal as BlockingPortal - -# Re-export imports so they look like they live directly in this package -for __value in list(locals().values()): - if getattr(__value, "__module__", "").startswith("anyio.abc."): - __value.__module__ = __name__ - -del __value diff --git a/functions/python/data-processor/anyio/abc/_eventloop.py b/functions/python/data-processor/anyio/abc/_eventloop.py deleted file mode 100644 index ae0628807..000000000 --- a/functions/python/data-processor/anyio/abc/_eventloop.py +++ /dev/null @@ -1,409 +0,0 @@ -from __future__ import annotations - -import math -import sys -from abc import ABCMeta, abstractmethod -from collections.abc import AsyncIterator, Awaitable, Callable, Sequence -from contextlib import AbstractContextManager -from os import PathLike -from signal import Signals -from socket import AddressFamily, SocketKind, socket -from typing import ( - IO, - TYPE_CHECKING, - Any, - TypeAlias, - TypeVar, - overload, -) - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if TYPE_CHECKING: - from _typeshed import FileDescriptorLike - - from .._core._synchronization import CapacityLimiter, Event, Lock, Semaphore - from .._core._tasks import CancelScope - from .._core._testing import TaskInfo - from ._sockets import ( - ConnectedUDPSocket, - ConnectedUNIXDatagramSocket, - IPSockAddrType, - SocketListener, - SocketStream, - UDPSocket, - UNIXDatagramSocket, - UNIXSocketStream, - ) - from ._subprocesses import Process - from ._tasks import TaskGroup - from ._testing import TestRunner - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") -StrOrBytesPath: TypeAlias = str | bytes | PathLike[str] | PathLike[bytes] - - -class AsyncBackend(metaclass=ABCMeta): - @classmethod - @abstractmethod - def run( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - options: dict[str, Any], - ) -> T_Retval: - """ - Run the given coroutine function in an asynchronous event loop. - - The current thread must not be already running an event loop. - - :param func: a coroutine function - :param args: positional arguments to ``func`` - :param kwargs: positional arguments to ``func`` - :param options: keyword arguments to call the backend ``run()`` implementation - with - :return: the return value of the coroutine function - """ - - @classmethod - @abstractmethod - def current_token(cls) -> object: - """ - Return an object that allows other threads to run code inside the event loop. - - :return: a token object, specific to the event loop running in the current - thread - """ - - @classmethod - @abstractmethod - def current_time(cls) -> float: - """ - Return the current value of the event loop's internal clock. - - :return: the clock value (seconds) - """ - - @classmethod - @abstractmethod - def cancelled_exception_class(cls) -> type[BaseException]: - """Return the exception class that is raised in a task if it's cancelled.""" - - @classmethod - @abstractmethod - async def checkpoint(cls) -> None: - """ - Check if the task has been cancelled, and allow rescheduling of other tasks. - - This is effectively the same as running :meth:`checkpoint_if_cancelled` and then - :meth:`cancel_shielded_checkpoint`. - """ - - @classmethod - async def checkpoint_if_cancelled(cls) -> None: - """ - Check if the current task group has been cancelled. - - This will check if the task has been cancelled, but will not allow other tasks - to be scheduled if not. - - """ - if cls.current_effective_deadline() == -math.inf: - await cls.checkpoint() - - @classmethod - async def cancel_shielded_checkpoint(cls) -> None: - """ - Allow the rescheduling of other tasks. - - This will give other tasks the opportunity to run, but without checking if the - current task group has been cancelled, unlike with :meth:`checkpoint`. - - """ - with cls.create_cancel_scope(shield=True): - await cls.sleep(0) - - @classmethod - @abstractmethod - async def sleep(cls, delay: float) -> None: - """ - Pause the current task for the specified duration. - - :param delay: the duration, in seconds - """ - - @classmethod - @abstractmethod - def create_cancel_scope( - cls, *, deadline: float = math.inf, shield: bool = False - ) -> CancelScope: - pass - - @classmethod - @abstractmethod - def current_effective_deadline(cls) -> float: - """ - Return the nearest deadline among all the cancel scopes effective for the - current task. - - :return: - - a clock value from the event loop's internal clock - - ``inf`` if there is no deadline in effect - - ``-inf`` if the current scope has been cancelled - :rtype: float - """ - - @classmethod - @abstractmethod - def create_task_group(cls) -> TaskGroup: - pass - - @classmethod - @abstractmethod - def create_event(cls) -> Event: - pass - - @classmethod - @abstractmethod - def create_lock(cls, *, fast_acquire: bool) -> Lock: - pass - - @classmethod - @abstractmethod - def create_semaphore( - cls, - initial_value: int, - *, - max_value: int | None = None, - fast_acquire: bool = False, - ) -> Semaphore: - pass - - @classmethod - @abstractmethod - def create_capacity_limiter(cls, total_tokens: float) -> CapacityLimiter: - pass - - @classmethod - @abstractmethod - async def run_sync_in_worker_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - abandon_on_cancel: bool = False, - limiter: CapacityLimiter | None = None, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - def check_cancelled(cls) -> None: - pass - - @classmethod - @abstractmethod - def run_async_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - def run_sync_from_thread( - cls, - func: Callable[[Unpack[PosArgsT]], T_Retval], - args: tuple[Unpack[PosArgsT]], - token: object, - ) -> T_Retval: - pass - - @classmethod - @abstractmethod - async def open_process( - cls, - command: StrOrBytesPath | Sequence[StrOrBytesPath], - *, - stdin: int | IO[Any] | None, - stdout: int | IO[Any] | None, - stderr: int | IO[Any] | None, - **kwargs: Any, - ) -> Process: - pass - - @classmethod - @abstractmethod - def setup_process_pool_exit_at_shutdown(cls, workers: set[Process]) -> None: - pass - - @classmethod - @abstractmethod - async def connect_tcp( - cls, host: str, port: int, local_address: IPSockAddrType | None = None - ) -> SocketStream: - pass - - @classmethod - @abstractmethod - async def connect_unix(cls, path: str | bytes) -> UNIXSocketStream: - pass - - @classmethod - @abstractmethod - def create_tcp_listener(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - def create_unix_listener(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - async def create_udp_socket( - cls, - family: AddressFamily, - local_address: IPSockAddrType | None, - remote_address: IPSockAddrType | None, - reuse_port: bool, - ) -> UDPSocket | ConnectedUDPSocket: - pass - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: None - ) -> UNIXDatagramSocket: ... - - @classmethod - @overload - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: str | bytes - ) -> ConnectedUNIXDatagramSocket: ... - - @classmethod - @abstractmethod - async def create_unix_datagram_socket( - cls, raw_socket: socket, remote_path: str | bytes | None - ) -> UNIXDatagramSocket | ConnectedUNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - async def getaddrinfo( - cls, - host: bytes | str | None, - port: str | int | None, - *, - family: int | AddressFamily = 0, - type: int | SocketKind = 0, - proto: int = 0, - flags: int = 0, - ) -> Sequence[ - tuple[ - AddressFamily, - SocketKind, - int, - str, - tuple[str, int] | tuple[str, int, int, int] | tuple[int, bytes], - ] - ]: - pass - - @classmethod - @abstractmethod - async def getnameinfo( - cls, sockaddr: IPSockAddrType, flags: int = 0 - ) -> tuple[str, str]: - pass - - @classmethod - @abstractmethod - async def wait_readable(cls, obj: FileDescriptorLike) -> None: - pass - - @classmethod - @abstractmethod - async def wait_writable(cls, obj: FileDescriptorLike) -> None: - pass - - @classmethod - @abstractmethod - def notify_closing(cls, obj: FileDescriptorLike) -> None: - pass - - @classmethod - @abstractmethod - async def wrap_listener_socket(cls, sock: socket) -> SocketListener: - pass - - @classmethod - @abstractmethod - async def wrap_stream_socket(cls, sock: socket) -> SocketStream: - pass - - @classmethod - @abstractmethod - async def wrap_unix_stream_socket(cls, sock: socket) -> UNIXSocketStream: - pass - - @classmethod - @abstractmethod - async def wrap_udp_socket(cls, sock: socket) -> UDPSocket: - pass - - @classmethod - @abstractmethod - async def wrap_connected_udp_socket(cls, sock: socket) -> ConnectedUDPSocket: - pass - - @classmethod - @abstractmethod - async def wrap_unix_datagram_socket(cls, sock: socket) -> UNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - async def wrap_connected_unix_datagram_socket( - cls, sock: socket - ) -> ConnectedUNIXDatagramSocket: - pass - - @classmethod - @abstractmethod - def current_default_thread_limiter(cls) -> CapacityLimiter: - pass - - @classmethod - @abstractmethod - def open_signal_receiver( - cls, *signals: Signals - ) -> AbstractContextManager[AsyncIterator[Signals]]: - pass - - @classmethod - @abstractmethod - def get_current_task(cls) -> TaskInfo: - pass - - @classmethod - @abstractmethod - def get_running_tasks(cls) -> Sequence[TaskInfo]: - pass - - @classmethod - @abstractmethod - async def wait_all_tasks_blocked(cls) -> None: - pass - - @classmethod - @abstractmethod - def create_test_runner(cls, options: dict[str, Any]) -> TestRunner: - pass diff --git a/functions/python/data-processor/anyio/abc/_resources.py b/functions/python/data-processor/anyio/abc/_resources.py deleted file mode 100644 index 10df115a7..000000000 --- a/functions/python/data-processor/anyio/abc/_resources.py +++ /dev/null @@ -1,33 +0,0 @@ -from __future__ import annotations - -from abc import ABCMeta, abstractmethod -from types import TracebackType -from typing import TypeVar - -T = TypeVar("T") - - -class AsyncResource(metaclass=ABCMeta): - """ - Abstract base class for all closeable asynchronous resources. - - Works as an asynchronous context manager which returns the instance itself on enter, - and calls :meth:`aclose` on exit. - """ - - __slots__ = () - - async def __aenter__(self: T) -> T: - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - await self.aclose() - - @abstractmethod - async def aclose(self) -> None: - """Close the resource.""" diff --git a/functions/python/data-processor/anyio/abc/_sockets.py b/functions/python/data-processor/anyio/abc/_sockets.py deleted file mode 100644 index feb26bd44..000000000 --- a/functions/python/data-processor/anyio/abc/_sockets.py +++ /dev/null @@ -1,399 +0,0 @@ -from __future__ import annotations - -import errno -import socket -from abc import abstractmethod -from collections.abc import Callable, Collection, Mapping -from contextlib import AsyncExitStack -from io import IOBase -from ipaddress import IPv4Address, IPv6Address -from socket import AddressFamily -from typing import Any, TypeAlias, TypeVar - -from .._core._eventloop import get_async_backend -from .._core._typedattr import ( - TypedAttributeProvider, - TypedAttributeSet, - typed_attribute, -) -from ._streams import ByteStream, Listener, UnreliableObjectStream -from ._tasks import TaskGroup - -IPAddressType: TypeAlias = str | IPv4Address | IPv6Address -IPSockAddrType: TypeAlias = tuple[str, int] -SockAddrType: TypeAlias = IPSockAddrType | str -UDPPacketType: TypeAlias = tuple[bytes, IPSockAddrType] -UNIXDatagramPacketType: TypeAlias = tuple[bytes, str] -T_Retval = TypeVar("T_Retval") - - -def _validate_socket( - sock_or_fd: socket.socket | int, - sock_type: socket.SocketKind, - addr_family: socket.AddressFamily = socket.AF_UNSPEC, - *, - require_connected: bool = False, - require_bound: bool = False, -) -> socket.socket: - if isinstance(sock_or_fd, int): - try: - sock = socket.socket(fileno=sock_or_fd) - except OSError as exc: - if exc.errno == errno.ENOTSOCK: - raise ValueError( - "the file descriptor does not refer to a socket" - ) from exc - elif require_connected: - raise ValueError("the socket must be connected") from exc - elif require_bound: - raise ValueError("the socket must be bound to a local address") from exc - else: - raise - elif isinstance(sock_or_fd, socket.socket): - sock = sock_or_fd - else: - raise TypeError( - f"expected an int or socket, got {type(sock_or_fd).__qualname__} instead" - ) - - try: - if require_connected: - try: - sock.getpeername() - except OSError as exc: - raise ValueError("the socket must be connected") from exc - - if require_bound: - try: - if sock.family in (socket.AF_INET, socket.AF_INET6): - bound_addr = sock.getsockname()[1] - else: - bound_addr = sock.getsockname() - except OSError: - bound_addr = None - - if not bound_addr: - raise ValueError("the socket must be bound to a local address") - - if addr_family != socket.AF_UNSPEC and sock.family != addr_family: - raise ValueError( - f"address family mismatch: expected {addr_family.name}, got " - f"{sock.family.name}" - ) - - if sock.type != sock_type: - raise ValueError( - f"socket type mismatch: expected {sock_type.name}, got {sock.type.name}" - ) - except BaseException: - # Avoid ResourceWarning from the locally constructed socket object - if isinstance(sock_or_fd, int): - sock.detach() - - raise - - sock.setblocking(False) - return sock - - -class SocketAttribute(TypedAttributeSet): - """ - .. attribute:: family - :type: socket.AddressFamily - - the address family of the underlying socket - - .. attribute:: local_address - :type: tuple[str, int] | str - - the local address the underlying socket is connected to - - .. attribute:: local_port - :type: int - - for IP based sockets, the local port the underlying socket is bound to - - .. attribute:: raw_socket - :type: socket.socket - - the underlying stdlib socket object - - .. attribute:: remote_address - :type: tuple[str, int] | str - - the remote address the underlying socket is connected to - - .. attribute:: remote_port - :type: int - - for IP based sockets, the remote port the underlying socket is connected to - """ - - family: AddressFamily = typed_attribute() - local_address: SockAddrType = typed_attribute() - local_port: int = typed_attribute() - raw_socket: socket.socket = typed_attribute() - remote_address: SockAddrType = typed_attribute() - remote_port: int = typed_attribute() - - -class _SocketProvider(TypedAttributeProvider): - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - from .._core._sockets import convert_ipv6_sockaddr as convert - - attributes: dict[Any, Callable[[], Any]] = { - SocketAttribute.family: lambda: self._raw_socket.family, - SocketAttribute.local_address: lambda: convert( - self._raw_socket.getsockname() - ), - SocketAttribute.raw_socket: lambda: self._raw_socket, - } - try: - peername: tuple[str, int] | None = convert(self._raw_socket.getpeername()) - except OSError: - peername = None - - # Provide the remote address for connected sockets - if peername is not None: - attributes[SocketAttribute.remote_address] = lambda: peername - - # Provide local and remote ports for IP based sockets - if self._raw_socket.family in (AddressFamily.AF_INET, AddressFamily.AF_INET6): - attributes[SocketAttribute.local_port] = lambda: ( - self._raw_socket.getsockname()[1] - ) - if peername is not None: - remote_port = peername[1] - attributes[SocketAttribute.remote_port] = lambda: remote_port - - return attributes - - @property - @abstractmethod - def _raw_socket(self) -> socket.socket: - pass - - -class SocketStream(ByteStream, _SocketProvider): - """ - Transports bytes over a socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> SocketStream: - """ - Wrap an existing socket object or file descriptor as a socket stream. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a socket stream - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_STREAM, require_connected=True) - return await get_async_backend().wrap_stream_socket(sock) - - -class UNIXSocketStream(SocketStream): - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> UNIXSocketStream: - """ - Wrap an existing socket object or file descriptor as a UNIX socket stream. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a UNIX socket stream - - """ - sock = _validate_socket( - sock_or_fd, socket.SOCK_STREAM, socket.AF_UNIX, require_connected=True - ) - return await get_async_backend().wrap_unix_stream_socket(sock) - - @abstractmethod - async def send_fds(self, message: bytes, fds: Collection[int | IOBase]) -> None: - """ - Send file descriptors along with a message to the peer. - - :param message: a non-empty bytestring - :param fds: a collection of files (either numeric file descriptors or open file - or socket objects) - """ - - @abstractmethod - async def receive_fds(self, msglen: int, maxfds: int) -> tuple[bytes, list[int]]: - """ - Receive file descriptors along with a message from the peer. - - :param msglen: length of the message to expect from the peer - :param maxfds: maximum number of file descriptors to expect from the peer - :return: a tuple of (message, file descriptors) - """ - - -class SocketListener(Listener[SocketStream], _SocketProvider): - """ - Listens to incoming socket connections. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket( - cls, - sock_or_fd: socket.socket | int, - ) -> SocketListener: - """ - Wrap an existing socket object or file descriptor as a socket listener. - - The newly created listener takes ownership of the socket being passed in. - - :param sock_or_fd: a socket object or file descriptor - :return: a socket listener - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_STREAM, require_bound=True) - return await get_async_backend().wrap_listener_socket(sock) - - @abstractmethod - async def accept(self) -> SocketStream: - """Accept an incoming connection.""" - - async def serve( - self, - handler: Callable[[SocketStream], Any], - task_group: TaskGroup | None = None, - ) -> None: - from .. import create_task_group - - async with AsyncExitStack() as stack: - if task_group is None: - task_group = await stack.enter_async_context(create_task_group()) - - while True: - stream = await self.accept() - task_group.start_soon(handler, stream) - - -class UDPSocket(UnreliableObjectStream[UDPPacketType], _SocketProvider): - """ - Represents an unconnected UDP socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> UDPSocket: - """ - Wrap an existing socket object or file descriptor as a UDP socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must be bound to a local address. - - :param sock_or_fd: a socket object or file descriptor - :return: a UDP socket - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_DGRAM, require_bound=True) - return await get_async_backend().wrap_udp_socket(sock) - - async def sendto(self, data: bytes, host: str, port: int) -> None: - """ - Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, (host, port))). - - """ - return await self.send((data, (host, port))) - - -class ConnectedUDPSocket(UnreliableObjectStream[bytes], _SocketProvider): - """ - Represents an connected UDP socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket(cls, sock_or_fd: socket.socket | int) -> ConnectedUDPSocket: - """ - Wrap an existing socket object or file descriptor as a connected UDP socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a connected UDP socket - - """ - sock = _validate_socket( - sock_or_fd, - socket.SOCK_DGRAM, - require_connected=True, - ) - return await get_async_backend().wrap_connected_udp_socket(sock) - - -class UNIXDatagramSocket( - UnreliableObjectStream[UNIXDatagramPacketType], _SocketProvider -): - """ - Represents an unconnected Unix datagram socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket( - cls, - sock_or_fd: socket.socket | int, - ) -> UNIXDatagramSocket: - """ - Wrap an existing socket object or file descriptor as a UNIX datagram - socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - - :param sock_or_fd: a socket object or file descriptor - :return: a UNIX datagram socket - - """ - sock = _validate_socket(sock_or_fd, socket.SOCK_DGRAM, socket.AF_UNIX) - return await get_async_backend().wrap_unix_datagram_socket(sock) - - async def sendto(self, data: bytes, path: str) -> None: - """Alias for :meth:`~.UnreliableObjectSendStream.send` ((data, path)).""" - return await self.send((data, path)) - - -class ConnectedUNIXDatagramSocket(UnreliableObjectStream[bytes], _SocketProvider): - """ - Represents a connected Unix datagram socket. - - Supports all relevant extra attributes from :class:`~SocketAttribute`. - """ - - @classmethod - async def from_socket( - cls, - sock_or_fd: socket.socket | int, - ) -> ConnectedUNIXDatagramSocket: - """ - Wrap an existing socket object or file descriptor as a connected UNIX datagram - socket. - - The newly created socket wrapper takes ownership of the socket being passed in. - The existing socket must already be connected. - - :param sock_or_fd: a socket object or file descriptor - :return: a connected UNIX datagram socket - - """ - sock = _validate_socket( - sock_or_fd, socket.SOCK_DGRAM, socket.AF_UNIX, require_connected=True - ) - return await get_async_backend().wrap_connected_unix_datagram_socket(sock) diff --git a/functions/python/data-processor/anyio/abc/_streams.py b/functions/python/data-processor/anyio/abc/_streams.py deleted file mode 100644 index 186e3f503..000000000 --- a/functions/python/data-processor/anyio/abc/_streams.py +++ /dev/null @@ -1,233 +0,0 @@ -from __future__ import annotations - -from abc import ABCMeta, abstractmethod -from collections.abc import Callable -from typing import Any, Generic, TypeAlias, TypeVar - -from .._core._exceptions import EndOfStream -from .._core._typedattr import TypedAttributeProvider -from ._resources import AsyncResource -from ._tasks import TaskGroup - -T_Item = TypeVar("T_Item") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) - - -class UnreliableObjectReceiveStream( - Generic[T_co], AsyncResource, TypedAttributeProvider -): - """ - An interface for receiving objects. - - This interface makes no guarantees that the received messages arrive in the order in - which they were sent, or that no messages are missed. - - Asynchronously iterating over objects of this type will yield objects matching the - given type parameter. - """ - - def __aiter__(self) -> UnreliableObjectReceiveStream[T_co]: - return self - - async def __anext__(self) -> T_co: - try: - return await self.receive() - except EndOfStream: - raise StopAsyncIteration from None - - @abstractmethod - async def receive(self) -> T_co: - """ - Receive the next item. - - :raises ~anyio.ClosedResourceError: if the receive stream has been explicitly - closed - :raises ~anyio.EndOfStream: if this stream has been closed from the other end - :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable - due to external causes - """ - - -class UnreliableObjectSendStream( - Generic[T_contra], AsyncResource, TypedAttributeProvider -): - """ - An interface for sending objects. - - This interface makes no guarantees that the messages sent will reach the - recipient(s) in the same order in which they were sent, or at all. - """ - - @abstractmethod - async def send(self, item: T_contra) -> None: - """ - Send an item to the peer(s). - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if the send stream has been explicitly - closed - :raises ~anyio.BrokenResourceError: if this stream has been rendered unusable - due to external causes - """ - - -class UnreliableObjectStream( - UnreliableObjectReceiveStream[T_Item], UnreliableObjectSendStream[T_Item] -): - """ - A bidirectional message stream which does not guarantee the order or reliability of - message delivery. - """ - - -class ObjectReceiveStream(UnreliableObjectReceiveStream[T_co]): - """ - A receive message stream which guarantees that messages are received in the same - order in which they were sent, and that no messages are missed. - """ - - -class ObjectSendStream(UnreliableObjectSendStream[T_contra]): - """ - A send message stream which guarantees that messages are delivered in the same order - in which they were sent, without missing any messages in the middle. - """ - - -class ObjectStream( - ObjectReceiveStream[T_Item], - ObjectSendStream[T_Item], - UnreliableObjectStream[T_Item], -): - """ - A bidirectional message stream which guarantees the order and reliability of message - delivery. - """ - - @abstractmethod - async def send_eof(self) -> None: - """ - Send an end-of-file indication to the peer. - - You should not try to send any further data to this stream after calling this - method. This method is idempotent (does nothing on successive calls). - """ - - -class ByteReceiveStream(AsyncResource, TypedAttributeProvider): - """ - An interface for receiving bytes from a single peer. - - Iterating this byte stream will yield a byte string of arbitrary length, but no more - than 65536 bytes. - """ - - def __aiter__(self) -> ByteReceiveStream: - return self - - async def __anext__(self) -> bytes: - try: - return await self.receive() - except EndOfStream: - raise StopAsyncIteration from None - - @abstractmethod - async def receive(self, max_bytes: int = 65536) -> bytes: - """ - Receive at most ``max_bytes`` bytes from the peer. - - .. note:: Implementers of this interface should not return an empty - :class:`bytes` object, and users should ignore them. - - :param max_bytes: maximum number of bytes to receive - :return: the received bytes - :raises ~anyio.EndOfStream: if this stream has been closed from the other end - """ - - -class ByteSendStream(AsyncResource, TypedAttributeProvider): - """An interface for sending bytes to a single peer.""" - - @abstractmethod - async def send(self, item: bytes) -> None: - """ - Send the given bytes to the peer. - - :param item: the bytes to send - """ - - -class ByteStream(ByteReceiveStream, ByteSendStream): - """A bidirectional byte stream.""" - - @abstractmethod - async def send_eof(self) -> None: - """ - Send an end-of-file indication to the peer. - - You should not try to send any further data to this stream after calling this - method. This method is idempotent (does nothing on successive calls). - """ - - -#: Type alias for all unreliable bytes-oriented receive streams. -AnyUnreliableByteReceiveStream: TypeAlias = ( - UnreliableObjectReceiveStream[bytes] | ByteReceiveStream -) -#: Type alias for all unreliable bytes-oriented send streams. -AnyUnreliableByteSendStream: TypeAlias = ( - UnreliableObjectSendStream[bytes] | ByteSendStream -) -#: Type alias for all unreliable bytes-oriented streams. -AnyUnreliableByteStream: TypeAlias = UnreliableObjectStream[bytes] | ByteStream -#: Type alias for all bytes-oriented receive streams. -AnyByteReceiveStream: TypeAlias = ObjectReceiveStream[bytes] | ByteReceiveStream -#: Type alias for all bytes-oriented send streams. -AnyByteSendStream: TypeAlias = ObjectSendStream[bytes] | ByteSendStream -#: Type alias for all bytes-oriented streams. -AnyByteStream: TypeAlias = ObjectStream[bytes] | ByteStream - - -class Listener(Generic[T_co], AsyncResource, TypedAttributeProvider): - """An interface for objects that let you accept incoming connections.""" - - @abstractmethod - async def serve( - self, handler: Callable[[T_co], Any], task_group: TaskGroup | None = None - ) -> None: - """ - Accept incoming connections as they come in and start tasks to handle them. - - :param handler: a callable that will be used to handle each accepted connection - :param task_group: the task group that will be used to start tasks for handling - each accepted connection (if omitted, an ad-hoc task group will be created) - """ - - -class ObjectStreamConnectable(Generic[T_co], metaclass=ABCMeta): - @abstractmethod - async def connect(self) -> ObjectStream[T_co]: - """ - Connect to the remote endpoint. - - :return: an object stream connected to the remote end - :raises ConnectionFailed: if the connection fails - """ - - -class ByteStreamConnectable(metaclass=ABCMeta): - @abstractmethod - async def connect(self) -> ByteStream: - """ - Connect to the remote endpoint. - - :return: a bytestream connected to the remote end - :raises ConnectionFailed: if the connection fails - """ - - -#: Type alias for all connectables returning bytestreams or bytes-oriented object streams -AnyByteStreamConnectable: TypeAlias = ( - ObjectStreamConnectable[bytes] | ByteStreamConnectable -) diff --git a/functions/python/data-processor/anyio/abc/_subprocesses.py b/functions/python/data-processor/anyio/abc/_subprocesses.py deleted file mode 100644 index ce0564cea..000000000 --- a/functions/python/data-processor/anyio/abc/_subprocesses.py +++ /dev/null @@ -1,79 +0,0 @@ -from __future__ import annotations - -from abc import abstractmethod -from signal import Signals - -from ._resources import AsyncResource -from ._streams import ByteReceiveStream, ByteSendStream - - -class Process(AsyncResource): - """An asynchronous version of :class:`subprocess.Popen`.""" - - @abstractmethod - async def wait(self) -> int: - """ - Wait until the process exits. - - :return: the exit code of the process - """ - - @abstractmethod - def terminate(self) -> None: - """ - Terminates the process, gracefully if possible. - - On Windows, this calls ``TerminateProcess()``. - On POSIX systems, this sends ``SIGTERM`` to the process. - - .. seealso:: :meth:`subprocess.Popen.terminate` - """ - - @abstractmethod - def kill(self) -> None: - """ - Kills the process. - - On Windows, this calls ``TerminateProcess()``. - On POSIX systems, this sends ``SIGKILL`` to the process. - - .. seealso:: :meth:`subprocess.Popen.kill` - """ - - @abstractmethod - def send_signal(self, signal: Signals) -> None: - """ - Send a signal to the subprocess. - - .. seealso:: :meth:`subprocess.Popen.send_signal` - - :param signal: the signal number (e.g. :data:`signal.SIGHUP`) - """ - - @property - @abstractmethod - def pid(self) -> int: - """The process ID of the process.""" - - @property - @abstractmethod - def returncode(self) -> int | None: - """ - The return code of the process. If the process has not yet terminated, this will - be ``None``. - """ - - @property - @abstractmethod - def stdin(self) -> ByteSendStream | None: - """The stream for the standard input of the process.""" - - @property - @abstractmethod - def stdout(self) -> ByteReceiveStream | None: - """The stream for the standard output of the process.""" - - @property - @abstractmethod - def stderr(self) -> ByteReceiveStream | None: - """The stream for the standard error output of the process.""" diff --git a/functions/python/data-processor/anyio/abc/_tasks.py b/functions/python/data-processor/anyio/abc/_tasks.py deleted file mode 100644 index 516b3ec3b..000000000 --- a/functions/python/data-processor/anyio/abc/_tasks.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import sys -from abc import ABCMeta, abstractmethod -from collections.abc import Awaitable, Callable -from types import TracebackType -from typing import TYPE_CHECKING, Any, Protocol, overload - -if sys.version_info >= (3, 13): - from typing import TypeVar -else: - from typing_extensions import TypeVar - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if TYPE_CHECKING: - from .._core._tasks import CancelScope - -T_Retval = TypeVar("T_Retval") -T_contra = TypeVar("T_contra", contravariant=True, default=None) -PosArgsT = TypeVarTuple("PosArgsT") - - -class TaskStatus(Protocol[T_contra]): - @overload - def started(self: TaskStatus[None]) -> None: ... - - @overload - def started(self, value: T_contra) -> None: ... - - def started(self, value: T_contra | None = None) -> None: - """ - Signal that the task has started. - - :param value: object passed back to the starter of the task - """ - - -class TaskGroup(metaclass=ABCMeta): - """ - Groups several asynchronous tasks together. - - :ivar cancel_scope: the cancel scope inherited by all child tasks - :vartype cancel_scope: CancelScope - - .. note:: On asyncio, support for eager task factories is considered to be - **experimental**. In particular, they don't follow the usual semantics of new - tasks being scheduled on the next iteration of the event loop, and may thus - cause unexpected behavior in code that wasn't written with such semantics in - mind. - """ - - cancel_scope: CancelScope - - @abstractmethod - def start_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[Any]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> None: - """ - Start a new task in this task group. - - :param func: a coroutine function - :param args: positional arguments to call the function with - :param name: name of the task, for the purposes of introspection and debugging - - .. versionadded:: 3.0 - """ - - @abstractmethod - async def start( - self, - func: Callable[..., Awaitable[Any]], - *args: object, - name: object = None, - ) -> Any: - """ - Start a new task and wait until it signals for readiness. - - The target callable must accept a keyword argument ``task_status`` (of type - :class:`TaskStatus`). Awaiting on this method will return whatever was passed to - ``task_status.started()`` (``None`` by default). - - .. note:: The :class:`TaskStatus` class is generic, and the type argument should - indicate the type of the value that will be passed to - ``task_status.started()``. - - :param func: a coroutine function that accepts the ``task_status`` keyword - argument - :param args: positional arguments to call the function with - :param name: an optional name for the task, for introspection and debugging - :return: the value passed to ``task_status.started()`` - :raises RuntimeError: if the task finishes without calling - ``task_status.started()`` - - .. seealso:: :ref:`start_initialize` - - .. versionadded:: 3.0 - """ - - @abstractmethod - async def __aenter__(self) -> TaskGroup: - """Enter the task group context and allow starting new tasks.""" - - @abstractmethod - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - """Exit the task group context waiting for all tasks to finish.""" diff --git a/functions/python/data-processor/anyio/abc/_testing.py b/functions/python/data-processor/anyio/abc/_testing.py deleted file mode 100644 index 7c50ed76d..000000000 --- a/functions/python/data-processor/anyio/abc/_testing.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -import types -from abc import ABCMeta, abstractmethod -from collections.abc import AsyncGenerator, Callable, Coroutine, Iterable -from typing import Any, TypeVar - -_T = TypeVar("_T") - - -class TestRunner(metaclass=ABCMeta): - """ - Encapsulates a running event loop. Every call made through this object will use the - same event loop. - """ - - def __enter__(self) -> TestRunner: - return self - - @abstractmethod - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: types.TracebackType | None, - ) -> bool | None: ... - - @abstractmethod - def run_asyncgen_fixture( - self, - fixture_func: Callable[..., AsyncGenerator[_T, Any]], - kwargs: dict[str, Any], - ) -> Iterable[_T]: - """ - Run an async generator fixture. - - :param fixture_func: the fixture function - :param kwargs: keyword arguments to call the fixture function with - :return: an iterator yielding the value yielded from the async generator - """ - - @abstractmethod - def run_fixture( - self, - fixture_func: Callable[..., Coroutine[Any, Any, _T]], - kwargs: dict[str, Any], - ) -> _T: - """ - Run an async fixture. - - :param fixture_func: the fixture function - :param kwargs: keyword arguments to call the fixture function with - :return: the return value of the fixture function - """ - - @abstractmethod - def run_test( - self, test_func: Callable[..., Coroutine[Any, Any, Any]], kwargs: dict[str, Any] - ) -> None: - """ - Run an async test function. - - :param test_func: the test function - :param kwargs: keyword arguments to call the test function with - """ diff --git a/functions/python/data-processor/anyio/from_thread.py b/functions/python/data-processor/anyio/from_thread.py deleted file mode 100644 index 837de5e96..000000000 --- a/functions/python/data-processor/anyio/from_thread.py +++ /dev/null @@ -1,578 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "BlockingPortal", - "BlockingPortalProvider", - "check_cancelled", - "run", - "run_sync", - "start_blocking_portal", -) - -import sys -from collections.abc import Awaitable, Callable, Generator -from concurrent.futures import Future -from contextlib import ( - AbstractAsyncContextManager, - AbstractContextManager, - contextmanager, -) -from dataclasses import dataclass, field -from functools import partial -from inspect import isawaitable -from threading import Lock, Thread, current_thread, get_ident -from types import TracebackType -from typing import ( - Any, - Generic, - TypeVar, - cast, - overload, -) - -from ._core._eventloop import ( - get_cancelled_exc_class, - threadlocals, -) -from ._core._eventloop import run as run_eventloop -from ._core._exceptions import NoEventLoopError -from ._core._synchronization import Event -from ._core._tasks import CancelScope, create_task_group -from .abc._tasks import TaskStatus -from .lowlevel import EventLoopToken, current_token - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -T_co = TypeVar("T_co", covariant=True) -PosArgsT = TypeVarTuple("PosArgsT") - - -def _token_or_error(token: EventLoopToken | None) -> EventLoopToken: - if token is not None: - return token - - try: - return threadlocals.current_token - except AttributeError: - raise NoEventLoopError( - "Not running inside an AnyIO worker thread, and no event loop token was " - "provided" - ) from None - - -def run( - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - token: EventLoopToken | None = None, -) -> T_Retval: - """ - Call a coroutine function from a worker thread. - - :param func: a coroutine function - :param args: positional arguments for the callable - :param token: an event loop token to use to get back to the event loop thread - (required if calling this function from outside an AnyIO worker thread) - :return: the return value of the coroutine function - :raises MissingTokenError: if no token was provided and called from outside an - AnyIO worker thread - :raises RunFinishedError: if the event loop tied to ``token`` is no longer running - - .. versionchanged:: 4.11.0 - Added the ``token`` parameter. - - """ - explicit_token = token is not None - token = _token_or_error(token) - return token.backend_class.run_async_from_thread( - func, args, token=token.native_token if explicit_token else None - ) - - -def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - token: EventLoopToken | None = None, -) -> T_Retval: - """ - Call a function in the event loop thread from a worker thread. - - :param func: a callable - :param args: positional arguments for the callable - :param token: an event loop token to use to get back to the event loop thread - (required if calling this function from outside an AnyIO worker thread) - :return: the return value of the callable - :raises MissingTokenError: if no token was provided and called from outside an - AnyIO worker thread - :raises RunFinishedError: if the event loop tied to ``token`` is no longer running - - .. versionchanged:: 4.11.0 - Added the ``token`` parameter. - - """ - explicit_token = token is not None - token = _token_or_error(token) - return token.backend_class.run_sync_from_thread( - func, args, token=token.native_token if explicit_token else None - ) - - -class _BlockingAsyncContextManager(Generic[T_co], AbstractContextManager): - _enter_future: Future[T_co] - _exit_future: Future[bool | None] - _exit_event: Event - _exit_exc_info: tuple[ - type[BaseException] | None, BaseException | None, TracebackType | None - ] = (None, None, None) - - def __init__( - self, async_cm: AbstractAsyncContextManager[T_co], portal: BlockingPortal - ): - self._async_cm = async_cm - self._portal = portal - - async def run_async_cm(self) -> bool | None: - try: - self._exit_event = Event() - value = await self._async_cm.__aenter__() - except BaseException as exc: - self._enter_future.set_exception(exc) - raise - else: - self._enter_future.set_result(value) - - try: - # Wait for the sync context manager to exit. - # This next statement can raise `get_cancelled_exc_class()` if - # something went wrong in a task group in this async context - # manager. - await self._exit_event.wait() - finally: - # In case of cancellation, it could be that we end up here before - # `_BlockingAsyncContextManager.__exit__` is called, and an - # `_exit_exc_info` has been set. - result = await self._async_cm.__aexit__(*self._exit_exc_info) - - return result - - def __enter__(self) -> T_co: - self._enter_future = Future() - self._exit_future = self._portal.start_task_soon(self.run_async_cm) - return self._enter_future.result() - - def __exit__( - self, - __exc_type: type[BaseException] | None, - __exc_value: BaseException | None, - __traceback: TracebackType | None, - ) -> bool | None: - self._exit_exc_info = __exc_type, __exc_value, __traceback - self._portal.call(self._exit_event.set) - return self._exit_future.result() - - -class _BlockingPortalTaskStatus(TaskStatus): - def __init__(self, future: Future): - self._future = future - - def started(self, value: object = None) -> None: - self._future.set_result(value) - - -class BlockingPortal: - """ - An object that lets external threads run code in an asynchronous event loop. - - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - """ - - def __init__(self) -> None: - self._token = current_token() - self._event_loop_thread_id: int | None = get_ident() - self._stop_event = Event() - self._task_group = create_task_group() - - async def __aenter__(self) -> BlockingPortal: - await self._task_group.__aenter__() - return self - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> bool: - await self.stop() - return await self._task_group.__aexit__(exc_type, exc_val, exc_tb) - - def _check_running(self) -> None: - if self._event_loop_thread_id is None: - raise RuntimeError("This portal is not running") - if self._event_loop_thread_id == get_ident(): - raise RuntimeError( - "This method cannot be called from the event loop thread" - ) - - async def sleep_until_stopped(self) -> None: - """Sleep until :meth:`stop` is called.""" - await self._stop_event.wait() - - async def stop(self, cancel_remaining: bool = False) -> None: - """ - Signal the portal to shut down. - - This marks the portal as no longer accepting new calls and exits from - :meth:`sleep_until_stopped`. - - :param cancel_remaining: ``True`` to cancel all the remaining tasks, ``False`` - to let them finish before returning - - """ - self._event_loop_thread_id = None - self._stop_event.set() - if cancel_remaining: - self._task_group.cancel_scope.cancel("the blocking portal is shutting down") - - async def _call_func( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - future: Future[T_Retval], - ) -> None: - def callback(f: Future[T_Retval]) -> None: - if f.cancelled(): - if self._event_loop_thread_id == get_ident(): - scope.cancel("the future was cancelled") - elif self._event_loop_thread_id is not None: - self.call(scope.cancel, "the future was cancelled") - - try: - retval_or_awaitable = func(*args, **kwargs) - if isawaitable(retval_or_awaitable): - with CancelScope() as scope: - future.add_done_callback(callback) - retval = await retval_or_awaitable - else: - retval = retval_or_awaitable - except get_cancelled_exc_class(): - future.cancel() - future.set_running_or_notify_cancel() - except BaseException as exc: - if not future.cancelled(): - future.set_exception(exc) - - # Let base exceptions fall through - if not isinstance(exc, Exception): - raise - else: - if not future.cancelled(): - future.set_result(retval) - finally: - scope = None # type: ignore[assignment] - - def _spawn_task_from_thread( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - args: tuple[Unpack[PosArgsT]], - kwargs: dict[str, Any], - name: object, - future: Future[T_Retval], - ) -> None: - """ - Spawn a new task using the given callable. - - :param func: a callable - :param args: positional arguments to be passed to the callable - :param kwargs: keyword arguments to be passed to the callable - :param name: name of the task (will be coerced to a string if not ``None``) - :param future: a future that will resolve to the return value of the callable, - or the exception raised during its execution - - """ - run_sync( - partial(self._task_group.start_soon, name=name), - self._call_func, - func, - args, - kwargs, - future, - token=self._token, - ) - - @overload - def call( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - ) -> T_Retval: ... - - @overload - def call( - self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] - ) -> T_Retval: ... - - def call( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - *args: Unpack[PosArgsT], - ) -> T_Retval: - """ - Call the given function in the event loop thread. - - If the callable returns a coroutine object, it is awaited on. - - :param func: any callable - :raises RuntimeError: if the portal is not running or if this method is called - from within the event loop thread - - """ - return cast(T_Retval, self.start_task_soon(func, *args).result()) - - @overload - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval]], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: ... - - @overload - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: ... - - def start_task_soon( - self, - func: Callable[[Unpack[PosArgsT]], Awaitable[T_Retval] | T_Retval], - *args: Unpack[PosArgsT], - name: object = None, - ) -> Future[T_Retval]: - """ - Start a task in the portal's task group. - - The task will be run inside a cancel scope which can be cancelled by cancelling - the returned future. - - :param func: the target function - :param args: positional arguments passed to ``func`` - :param name: name of the task (will be coerced to a string if not ``None``) - :return: a future that resolves with the return value of the callable if the - task completes successfully, or with the exception raised in the task - :raises RuntimeError: if the portal is not running or if this method is called - from within the event loop thread - :rtype: concurrent.futures.Future[T_Retval] - - .. versionadded:: 3.0 - - """ - self._check_running() - f: Future[T_Retval] = Future() - self._spawn_task_from_thread(func, args, {}, name, f) - return f - - def start_task( - self, - func: Callable[..., Awaitable[T_Retval]], - *args: object, - name: object = None, - ) -> tuple[Future[T_Retval], Any]: - """ - Start a task in the portal's task group and wait until it signals for readiness. - - This method works the same way as :meth:`.abc.TaskGroup.start`. - - :param func: the target function - :param args: positional arguments passed to ``func`` - :param name: name of the task (will be coerced to a string if not ``None``) - :return: a tuple of (future, task_status_value) where the ``task_status_value`` - is the value passed to ``task_status.started()`` from within the target - function - :rtype: tuple[concurrent.futures.Future[T_Retval], Any] - - .. versionadded:: 3.0 - - """ - - def task_done(future: Future[T_Retval]) -> None: - if not task_status_future.done(): - if future.cancelled(): - task_status_future.cancel() - elif future.exception(): - task_status_future.set_exception(future.exception()) - else: - exc = RuntimeError( - "Task exited without calling task_status.started()" - ) - task_status_future.set_exception(exc) - - self._check_running() - task_status_future: Future = Future() - task_status = _BlockingPortalTaskStatus(task_status_future) - f: Future = Future() - f.add_done_callback(task_done) - self._spawn_task_from_thread(func, args, {"task_status": task_status}, name, f) - return f, task_status_future.result() - - def wrap_async_context_manager( - self, cm: AbstractAsyncContextManager[T_co] - ) -> AbstractContextManager[T_co]: - """ - Wrap an async context manager as a synchronous context manager via this portal. - - Spawns a task that will call both ``__aenter__()`` and ``__aexit__()``, stopping - in the middle until the synchronous context manager exits. - - :param cm: an asynchronous context manager - :return: a synchronous context manager - - .. versionadded:: 2.1 - - """ - return _BlockingAsyncContextManager(cm, self) - - -@dataclass -class BlockingPortalProvider: - """ - A manager for a blocking portal. Used as a context manager. The first thread to - enter this context manager causes a blocking portal to be started with the specific - parameters, and the last thread to exit causes the portal to be shut down. Thus, - there will be exactly one blocking portal running in this context as long as at - least one thread has entered this context manager. - - The parameters are the same as for :func:`~anyio.run`. - - :param backend: name of the backend - :param backend_options: backend options - - .. versionadded:: 4.4 - """ - - backend: str = "asyncio" - backend_options: dict[str, Any] | None = None - _lock: Lock = field(init=False, default_factory=Lock) - _leases: int = field(init=False, default=0) - _portal: BlockingPortal = field(init=False) - _portal_cm: AbstractContextManager[BlockingPortal] | None = field( - init=False, default=None - ) - - def __enter__(self) -> BlockingPortal: - with self._lock: - if self._portal_cm is None: - self._portal_cm = start_blocking_portal( - self.backend, self.backend_options - ) - self._portal = self._portal_cm.__enter__() - - self._leases += 1 - return self._portal - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - portal_cm: AbstractContextManager[BlockingPortal] | None = None - with self._lock: - assert self._portal_cm - assert self._leases > 0 - self._leases -= 1 - if not self._leases: - portal_cm = self._portal_cm - self._portal_cm = None - del self._portal - - if portal_cm: - portal_cm.__exit__(None, None, None) - - -@contextmanager -def start_blocking_portal( - backend: str = "asyncio", - backend_options: dict[str, Any] | None = None, - *, - name: str | None = None, -) -> Generator[BlockingPortal, Any, None]: - """ - Start a new event loop in a new thread and run a blocking portal in its main task. - - The parameters are the same as for :func:`~anyio.run`. - - :param backend: name of the backend - :param backend_options: backend options - :param name: name of the thread - :return: a context manager that yields a blocking portal - - .. versionchanged:: 3.0 - Usage as a context manager is now required. - - """ - - async def run_portal() -> None: - async with BlockingPortal() as portal_: - if name is None: - current_thread().name = f"{backend}-portal-{id(portal_):x}" - - future.set_result(portal_) - await portal_.sleep_until_stopped() - - def run_blocking_portal() -> None: - if future.set_running_or_notify_cancel(): - try: - run_eventloop( - run_portal, backend=backend, backend_options=backend_options - ) - except BaseException as exc: - if not future.done(): - future.set_exception(exc) - - future: Future[BlockingPortal] = Future() - thread = Thread(target=run_blocking_portal, daemon=True, name=name) - thread.start() - try: - cancel_remaining_tasks = False - portal = future.result() - try: - yield portal - except BaseException: - cancel_remaining_tasks = True - raise - finally: - try: - portal.call(portal.stop, cancel_remaining_tasks) - except RuntimeError: - pass - finally: - thread.join() - - -def check_cancelled() -> None: - """ - Check if the cancel scope of the host task's running the current worker thread has - been cancelled. - - If the host task's current cancel scope has indeed been cancelled, the - backend-specific cancellation exception will be raised. - - :raises RuntimeError: if the current thread was not spawned by - :func:`.to_thread.run_sync` - - """ - try: - token: EventLoopToken = threadlocals.current_token - except AttributeError: - raise NoEventLoopError( - "This function can only be called inside an AnyIO worker thread" - ) from None - - token.backend_class.check_cancelled() diff --git a/functions/python/data-processor/anyio/functools.py b/functions/python/data-processor/anyio/functools.py deleted file mode 100644 index f1d6c7c55..000000000 --- a/functions/python/data-processor/anyio/functools.py +++ /dev/null @@ -1,409 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "AsyncCacheInfo", - "AsyncCacheParameters", - "AsyncLRUCacheWrapper", - "cache", - "lru_cache", - "reduce", -) - -import functools -import sys -from collections import OrderedDict -from collections.abc import ( - AsyncIterable, - Awaitable, - Callable, - Coroutine, - Hashable, - Iterable, -) -from functools import update_wrapper -from inspect import iscoroutinefunction -from typing import ( - Any, - Generic, - NamedTuple, - TypedDict, - TypeVar, - cast, - final, - overload, -) -from weakref import WeakKeyDictionary - -from ._core._eventloop import current_time -from ._core._synchronization import Lock -from .lowlevel import RunVar, checkpoint - -if sys.version_info >= (3, 11): - from typing import ParamSpec -else: - from typing_extensions import ParamSpec - -T = TypeVar("T") -S = TypeVar("S") -P = ParamSpec("P") -lru_cache_items: RunVar[ - WeakKeyDictionary[ - AsyncLRUCacheWrapper[Any, Any], - OrderedDict[ - Hashable, - tuple[_InitialMissingType, Lock, float | None] - | tuple[Any, None, float | None], - ], - ] -] = RunVar("lru_cache_items") - - -class _InitialMissingType: - pass - - -initial_missing: _InitialMissingType = _InitialMissingType() - - -class AsyncCacheInfo(NamedTuple): - hits: int - misses: int - maxsize: int | None - currsize: int - ttl: int | None - - -class AsyncCacheParameters(TypedDict): - maxsize: int | None - typed: bool - always_checkpoint: bool - ttl: int | None - - -class _LRUMethodWrapper(Generic[T]): - def __init__(self, wrapper: AsyncLRUCacheWrapper[..., T], instance: object): - self.__wrapper = wrapper - self.__instance = instance - - def cache_info(self) -> AsyncCacheInfo: - return self.__wrapper.cache_info() - - def cache_parameters(self) -> AsyncCacheParameters: - return self.__wrapper.cache_parameters() - - def cache_clear(self) -> None: - self.__wrapper.cache_clear() - - async def __call__(self, *args: Any, **kwargs: Any) -> T: - if self.__instance is None: - return await self.__wrapper(*args, **kwargs) - - return await self.__wrapper(self.__instance, *args, **kwargs) - - -@final -class AsyncLRUCacheWrapper(Generic[P, T]): - def __init__( - self, - func: Callable[P, Awaitable[T]], - maxsize: int | None, - typed: bool, - always_checkpoint: bool, - ttl: int | None, - ): - self.__wrapped__ = func - self._hits: int = 0 - self._misses: int = 0 - self._maxsize = max(maxsize, 0) if maxsize is not None else None - self._currsize: int = 0 - self._typed = typed - self._always_checkpoint = always_checkpoint - self._ttl = ttl - update_wrapper(self, func) - - def cache_info(self) -> AsyncCacheInfo: - return AsyncCacheInfo( - self._hits, self._misses, self._maxsize, self._currsize, self._ttl - ) - - def cache_parameters(self) -> AsyncCacheParameters: - return { - "maxsize": self._maxsize, - "typed": self._typed, - "always_checkpoint": self._always_checkpoint, - "ttl": self._ttl, - } - - def cache_clear(self) -> None: - if cache := lru_cache_items.get(None): - cache.pop(self, None) - self._hits = self._misses = self._currsize = 0 - - async def __call__(self, *args: P.args, **kwargs: P.kwargs) -> T: - # Easy case first: if maxsize == 0, no caching is done - if self._maxsize == 0: - value = await self.__wrapped__(*args, **kwargs) - self._misses += 1 - return value - - # The key is constructed as a flat tuple to avoid memory overhead - key: tuple[Any, ...] = args - if kwargs: - # initial_missing is used as a separator - key += (initial_missing,) + sum(kwargs.items(), ()) - - if self._typed: - key += tuple(type(arg) for arg in args) - if kwargs: - key += (initial_missing,) + tuple(type(val) for val in kwargs.values()) - - try: - cache = lru_cache_items.get() - except LookupError: - cache = WeakKeyDictionary() - lru_cache_items.set(cache) - - try: - cache_entry = cache[self] - except KeyError: - cache_entry = cache[self] = OrderedDict() - - cached_value: T | _InitialMissingType - try: - cached_value, lock, expires_at = cache_entry[key] - except KeyError: - # We're the first task to call this function - cached_value, lock, expires_at = ( - initial_missing, - Lock(fast_acquire=not self._always_checkpoint), - None, - ) - cache_entry[key] = cached_value, lock, expires_at - - if lock is None: - if expires_at is not None and current_time() >= expires_at: - self._currsize -= 1 - cached_value, lock, expires_at = ( - initial_missing, - Lock(fast_acquire=not self._always_checkpoint), - None, - ) - cache_entry[key] = cached_value, lock, expires_at - else: - # The value was already cached - self._hits += 1 - cache_entry.move_to_end(key) - if self._always_checkpoint: - await checkpoint() - - return cast(T, cached_value) - - async with lock: - # Check if another task filled the cache while we acquired the lock - if (cached_value := cache_entry[key][0]) is initial_missing: - self._misses += 1 - if self._maxsize is not None and self._currsize >= self._maxsize: - cache_entry.popitem(last=False) - else: - self._currsize += 1 - - value = await self.__wrapped__(*args, **kwargs) - expires_at = ( - current_time() + self._ttl if self._ttl is not None else None - ) - cache_entry[key] = value, None, expires_at - else: - # Another task filled the cache while we were waiting for the lock - self._hits += 1 - cache_entry.move_to_end(key) - value = cast(T, cached_value) - - return value - - def __get__( - self, instance: object, owner: type | None = None - ) -> _LRUMethodWrapper[T]: - wrapper = _LRUMethodWrapper(self, instance) - update_wrapper(wrapper, self.__wrapped__) - return wrapper - - -class _LRUCacheWrapper(Generic[T]): - def __init__( - self, maxsize: int | None, typed: bool, always_checkpoint: bool, ttl: int | None - ): - self._maxsize = maxsize - self._typed = typed - self._always_checkpoint = always_checkpoint - self._ttl = ttl - - @overload - def __call__( # type: ignore[overload-overlap] - self, func: Callable[P, Coroutine[Any, Any, T]], / - ) -> AsyncLRUCacheWrapper[P, T]: ... - - @overload - def __call__( - self, func: Callable[..., T], / - ) -> functools._lru_cache_wrapper[T]: ... - - def __call__( - self, f: Callable[P, Coroutine[Any, Any, T]] | Callable[..., T], / - ) -> AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T]: - if iscoroutinefunction(f): - return AsyncLRUCacheWrapper( - f, self._maxsize, self._typed, self._always_checkpoint, self._ttl - ) - - return functools.lru_cache(maxsize=self._maxsize, typed=self._typed)(f) # type: ignore[arg-type] - - -@overload -def cache( # type: ignore[overload-overlap] - func: Callable[P, Coroutine[Any, Any, T]], / -) -> AsyncLRUCacheWrapper[P, T]: ... - - -@overload -def cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... - - -def cache( - func: Callable[..., T] | Callable[P, Coroutine[Any, Any, T]], / -) -> AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T]: - """ - A convenient shortcut for :func:`lru_cache` with ``maxsize=None``. - - This is the asynchronous equivalent to :func:`functools.cache`. - - """ - return lru_cache(maxsize=None)(func) - - -@overload -def lru_cache( - *, - maxsize: int | None = ..., - typed: bool = ..., - always_checkpoint: bool = ..., - ttl: int | None = ..., -) -> _LRUCacheWrapper[Any]: ... - - -@overload -def lru_cache( # type: ignore[overload-overlap] - func: Callable[P, Coroutine[Any, Any, T]], / -) -> AsyncLRUCacheWrapper[P, T]: ... - - -@overload -def lru_cache(func: Callable[..., T], /) -> functools._lru_cache_wrapper[T]: ... - - -def lru_cache( - func: Callable[P, Coroutine[Any, Any, T]] | Callable[..., T] | None = None, - /, - *, - maxsize: int | None = 128, - typed: bool = False, - always_checkpoint: bool = False, - ttl: int | None = None, -) -> ( - AsyncLRUCacheWrapper[P, T] | functools._lru_cache_wrapper[T] | _LRUCacheWrapper[Any] -): - """ - An asynchronous version of :func:`functools.lru_cache`. - - If a synchronous function is passed, the standard library - :func:`functools.lru_cache` is applied instead. - - :param always_checkpoint: if ``True``, every call to the cached function will be - guaranteed to yield control to the event loop at least once - :param ttl: time in seconds after which to invalidate cache entries - - .. note:: Caches and locks are managed on a per-event loop basis. - - """ - if func is None: - return _LRUCacheWrapper[Any](maxsize, typed, always_checkpoint, ttl) - - if not callable(func): - raise TypeError("the first argument must be callable") - - return _LRUCacheWrapper[T](maxsize, typed, always_checkpoint, ttl)(func) - - -@overload -async def reduce( - function: Callable[[T, S], Awaitable[T]], - iterable: Iterable[S] | AsyncIterable[S], - /, - initial: T, -) -> T: ... - - -@overload -async def reduce( - function: Callable[[T, T], Awaitable[T]], - iterable: Iterable[T] | AsyncIterable[T], - /, -) -> T: ... - - -async def reduce( # type: ignore[misc] - function: Callable[[T, T], Awaitable[T]] | Callable[[T, S], Awaitable[T]], - iterable: Iterable[T] | Iterable[S] | AsyncIterable[T] | AsyncIterable[S], - /, - initial: T | _InitialMissingType = initial_missing, -) -> T: - """ - Asynchronous version of :func:`functools.reduce`. - - :param function: a coroutine function that takes two arguments: the accumulated - value and the next element from the iterable - :param iterable: an iterable or async iterable - :param initial: the initial value (if missing, the first element of the iterable is - used as the initial value) - - """ - element: Any - function_called = False - if isinstance(iterable, AsyncIterable): - async_it = iterable.__aiter__() - if initial is initial_missing: - try: - value = cast(T, await async_it.__anext__()) - except StopAsyncIteration: - raise TypeError( - "reduce() of empty sequence with no initial value" - ) from None - else: - value = cast(T, initial) - - async for element in async_it: - value = await function(value, element) - function_called = True - elif isinstance(iterable, Iterable): - it = iter(iterable) - if initial is initial_missing: - try: - value = cast(T, next(it)) - except StopIteration: - raise TypeError( - "reduce() of empty sequence with no initial value" - ) from None - else: - value = cast(T, initial) - - for element in it: - value = await function(value, element) - function_called = True - else: - raise TypeError("reduce() argument 2 must be an iterable or async iterable") - - # Make sure there is at least one checkpoint, even if an empty iterable and an - # initial value were given - if not function_called: - await checkpoint() - - return value diff --git a/functions/python/data-processor/anyio/lowlevel.py b/functions/python/data-processor/anyio/lowlevel.py deleted file mode 100644 index ffbb75a70..000000000 --- a/functions/python/data-processor/anyio/lowlevel.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "EventLoopToken", - "RunvarToken", - "RunVar", - "checkpoint", - "checkpoint_if_cancelled", - "cancel_shielded_checkpoint", - "current_token", -) - -import enum -from dataclasses import dataclass -from types import TracebackType -from typing import Any, Generic, Literal, TypeVar, final, overload -from weakref import WeakKeyDictionary - -from ._core._eventloop import get_async_backend -from .abc import AsyncBackend - -T = TypeVar("T") -D = TypeVar("D") - - -async def checkpoint() -> None: - """ - Check for cancellation and allow the scheduler to switch to another task. - - Equivalent to (but more efficient than):: - - await checkpoint_if_cancelled() - await cancel_shielded_checkpoint() - - .. versionadded:: 3.0 - - """ - await get_async_backend().checkpoint() - - -async def checkpoint_if_cancelled() -> None: - """ - Enter a checkpoint if the enclosing cancel scope has been cancelled. - - This does not allow the scheduler to switch to a different task. - - .. versionadded:: 3.0 - - """ - await get_async_backend().checkpoint_if_cancelled() - - -async def cancel_shielded_checkpoint() -> None: - """ - Allow the scheduler to switch to another task but without checking for cancellation. - - Equivalent to (but potentially more efficient than):: - - with CancelScope(shield=True): - await checkpoint() - - .. versionadded:: 3.0 - - """ - await get_async_backend().cancel_shielded_checkpoint() - - -@final -@dataclass(frozen=True, repr=False) -class EventLoopToken: - """ - An opaque object that holds a reference to an event loop. - - .. versionadded:: 4.11.0 - """ - - backend_class: type[AsyncBackend] - native_token: object - - -def current_token() -> EventLoopToken: - """ - Return a token object that can be used to call code in the current event loop from - another thread. - - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - .. versionadded:: 4.11.0 - - """ - backend_class = get_async_backend() - raw_token = backend_class.current_token() - return EventLoopToken(backend_class, raw_token) - - -_run_vars: WeakKeyDictionary[object, dict[RunVar[Any], Any]] = WeakKeyDictionary() - - -class _NoValueSet(enum.Enum): - NO_VALUE_SET = enum.auto() - - -class RunvarToken(Generic[T]): - __slots__ = "_var", "_value", "_redeemed" - - def __init__(self, var: RunVar[T], value: T | Literal[_NoValueSet.NO_VALUE_SET]): - self._var = var - self._value: T | Literal[_NoValueSet.NO_VALUE_SET] = value - self._redeemed = False - - def __enter__(self) -> RunvarToken[T]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self._var.reset(self) - - -class RunVar(Generic[T]): - """ - Like a :class:`~contextvars.ContextVar`, except scoped to the running event loop. - - Can be used as a context manager, Just like :class:`~contextvars.ContextVar`, that - will reset the variable to its previous value when the context block is exited. - """ - - __slots__ = "_name", "_default" - - NO_VALUE_SET: Literal[_NoValueSet.NO_VALUE_SET] = _NoValueSet.NO_VALUE_SET - - def __init__( - self, name: str, default: T | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET - ): - self._name = name - self._default = default - - @property - def _current_vars(self) -> dict[RunVar[T], T]: - native_token = current_token().native_token - try: - return _run_vars[native_token] - except KeyError: - run_vars = _run_vars[native_token] = {} - return run_vars - - @overload - def get(self, default: D) -> T | D: ... - - @overload - def get(self) -> T: ... - - def get( - self, default: D | Literal[_NoValueSet.NO_VALUE_SET] = NO_VALUE_SET - ) -> T | D: - try: - return self._current_vars[self] - except KeyError: - if default is not RunVar.NO_VALUE_SET: - return default - elif self._default is not RunVar.NO_VALUE_SET: - return self._default - - raise LookupError( - f'Run variable "{self._name}" has no value and no default set' - ) - - def set(self, value: T) -> RunvarToken[T]: - current_vars = self._current_vars - token = RunvarToken(self, current_vars.get(self, RunVar.NO_VALUE_SET)) - current_vars[self] = value - return token - - def reset(self, token: RunvarToken[T]) -> None: - if token._var is not self: - raise ValueError("This token does not belong to this RunVar") - - if token._redeemed: - raise ValueError("This token has already been used") - - if token._value is _NoValueSet.NO_VALUE_SET: - try: - del self._current_vars[self] - except KeyError: - pass - else: - self._current_vars[self] = token._value - - token._redeemed = True - - def __repr__(self) -> str: - return f"" diff --git a/functions/python/data-processor/anyio/py.typed b/functions/python/data-processor/anyio/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/anyio/pytest_plugin.py b/functions/python/data-processor/anyio/pytest_plugin.py deleted file mode 100644 index a5183f0c8..000000000 --- a/functions/python/data-processor/anyio/pytest_plugin.py +++ /dev/null @@ -1,363 +0,0 @@ -from __future__ import annotations - -import dataclasses -import socket -import sys -from collections.abc import Callable, Generator, Iterator -from contextlib import ExitStack, contextmanager -from inspect import isasyncgenfunction, iscoroutinefunction, ismethod -from typing import Any, cast - -import pytest -from _pytest.fixtures import FuncFixtureInfo, SubRequest -from _pytest.outcomes import Exit -from _pytest.python import CallSpec2 -from _pytest.scope import Scope - -from . import get_available_backends -from ._core._eventloop import ( - current_async_library, - get_async_backend, - reset_current_async_library, - set_current_async_library, -) -from ._core._exceptions import iterate_exceptions -from .abc import TestRunner - -if sys.version_info < (3, 11): - from exceptiongroup import ExceptionGroup - -_current_runner: TestRunner | None = None -_runner_stack: ExitStack | None = None -_runner_leases = 0 - - -def extract_backend_and_options(backend: object) -> tuple[str, dict[str, Any]]: - if isinstance(backend, str): - return backend, {} - elif isinstance(backend, tuple) and len(backend) == 2: - if isinstance(backend[0], str) and isinstance(backend[1], dict): - return cast(tuple[str, dict[str, Any]], backend) - - raise TypeError("anyio_backend must be either a string or tuple of (string, dict)") - - -@contextmanager -def get_runner( - backend_name: str, backend_options: dict[str, Any] -) -> Iterator[TestRunner]: - global _current_runner, _runner_leases, _runner_stack - if _current_runner is None: - asynclib = get_async_backend(backend_name) - _runner_stack = ExitStack() - if current_async_library() is None: - # Since we're in control of the event loop, we can cache the name of the - # async library - token = set_current_async_library(backend_name) - _runner_stack.callback(reset_current_async_library, token) - - backend_options = backend_options or {} - _current_runner = _runner_stack.enter_context( - asynclib.create_test_runner(backend_options) - ) - - _runner_leases += 1 - try: - yield _current_runner - finally: - _runner_leases -= 1 - if not _runner_leases: - assert _runner_stack is not None - _runner_stack.close() - _runner_stack = _current_runner = None - - -def pytest_addoption(parser: pytest.Parser) -> None: - parser.addini( - "anyio_mode", - default="strict", - help='AnyIO plugin mode (either "strict" or "auto")', - ) - - -def pytest_configure(config: pytest.Config) -> None: - config.addinivalue_line( - "markers", - "anyio: mark the (coroutine function) test to be run asynchronously via anyio.", - ) - if ( - config.getini("anyio_mode") == "auto" - and config.pluginmanager.has_plugin("asyncio") - and config.getini("asyncio_mode") == "auto" - ): - config.issue_config_time_warning( - pytest.PytestConfigWarning( - "AnyIO auto mode has been enabled together with pytest-asyncio auto " - "mode. This may cause unexpected behavior." - ), - 1, - ) - - -@pytest.hookimpl(hookwrapper=True) -def pytest_fixture_setup(fixturedef: Any, request: Any) -> Generator[Any]: - def wrapper(anyio_backend: Any, request: SubRequest, **kwargs: Any) -> Any: - # Rebind any fixture methods to the request instance - if ( - request.instance - and ismethod(func) - and type(func.__self__) is type(request.instance) - ): - local_func = func.__func__.__get__(request.instance) - else: - local_func = func - - backend_name, backend_options = extract_backend_and_options(anyio_backend) - if has_backend_arg: - kwargs["anyio_backend"] = anyio_backend - - if has_request_arg: - kwargs["request"] = request - - with get_runner(backend_name, backend_options) as runner: - if isasyncgenfunction(local_func): - yield from runner.run_asyncgen_fixture(local_func, kwargs) - else: - yield runner.run_fixture(local_func, kwargs) - - # Only apply this to coroutine functions and async generator functions in requests - # that involve the anyio_backend fixture - func = fixturedef.func - if isasyncgenfunction(func) or iscoroutinefunction(func): - if "anyio_backend" in request.fixturenames: - fixturedef.func = wrapper - original_argname = fixturedef.argnames - - if not (has_backend_arg := "anyio_backend" in fixturedef.argnames): - fixturedef.argnames += ("anyio_backend",) - - if not (has_request_arg := "request" in fixturedef.argnames): - fixturedef.argnames += ("request",) - - try: - return (yield) - finally: - fixturedef.func = func - fixturedef.argnames = original_argname - - return (yield) - - -@pytest.hookimpl(tryfirst=True) -def pytest_pycollect_makeitem( - collector: pytest.Module | pytest.Class, name: str, obj: object -) -> None: - if collector.istestfunction(obj, name): - inner_func = obj.hypothesis.inner_test if hasattr(obj, "hypothesis") else obj - if iscoroutinefunction(inner_func): - anyio_auto_mode = collector.config.getini("anyio_mode") == "auto" - marker = collector.get_closest_marker("anyio") - own_markers = getattr(obj, "pytestmark", ()) - if ( - anyio_auto_mode - or marker - or any(marker.name == "anyio" for marker in own_markers) - ): - pytest.mark.usefixtures("anyio_backend")(obj) - - -def pytest_collection_finish(session: pytest.Session) -> None: - for i, item in reversed(list(enumerate(session.items))): - if ( - isinstance(item, pytest.Function) - and iscoroutinefunction(item.function) - and item.get_closest_marker("anyio") is not None - and "anyio_backend" not in item.fixturenames - ): - new_items = [] - try: - cs_fields = {f.name for f in dataclasses.fields(CallSpec2)} - except TypeError: - cs_fields = set() - - for param_index, backend in enumerate(get_available_backends()): - if "_arg2scope" in cs_fields: # pytest >= 8 - callspec = CallSpec2( - params={"anyio_backend": backend}, - indices={"anyio_backend": param_index}, - _arg2scope={"anyio_backend": Scope.Module}, - _idlist=[backend], - marks=[], - ) - else: # pytest 7.x - callspec = CallSpec2( # type: ignore[call-arg] - funcargs={}, - params={"anyio_backend": backend}, - indices={"anyio_backend": param_index}, - arg2scope={"anyio_backend": Scope.Module}, - idlist=[backend], - marks=[], - ) - - fi = item._fixtureinfo - new_names_closure = list(fi.names_closure) - if "anyio_backend" not in new_names_closure: - new_names_closure.append("anyio_backend") - - new_fixtureinfo = FuncFixtureInfo( - argnames=fi.argnames, - initialnames=fi.initialnames, - names_closure=new_names_closure, - name2fixturedefs=fi.name2fixturedefs, - ) - new_item = pytest.Function.from_parent( - item.parent, - name=f"{item.originalname}[{backend}]", - callspec=callspec, - callobj=item.obj, - fixtureinfo=new_fixtureinfo, - keywords=item.keywords, - originalname=item.originalname, - ) - new_items.append(new_item) - - session.items[i : i + 1] = new_items - - -@pytest.hookimpl(tryfirst=True) -def pytest_pyfunc_call(pyfuncitem: Any) -> bool | None: - def run_with_hypothesis(**kwargs: Any) -> None: - with get_runner(backend_name, backend_options) as runner: - runner.run_test(original_func, kwargs) - - backend = pyfuncitem.funcargs.get("anyio_backend") - if backend: - backend_name, backend_options = extract_backend_and_options(backend) - - if hasattr(pyfuncitem.obj, "hypothesis"): - # Wrap the inner test function unless it's already wrapped - original_func = pyfuncitem.obj.hypothesis.inner_test - if original_func.__qualname__ != run_with_hypothesis.__qualname__: - if iscoroutinefunction(original_func): - pyfuncitem.obj.hypothesis.inner_test = run_with_hypothesis - - return None - - if iscoroutinefunction(pyfuncitem.obj): - funcargs = pyfuncitem.funcargs - testargs = {arg: funcargs[arg] for arg in pyfuncitem._fixtureinfo.argnames} - with get_runner(backend_name, backend_options) as runner: - try: - runner.run_test(pyfuncitem.obj, testargs) - except ExceptionGroup as excgrp: - for exc in iterate_exceptions(excgrp): - if isinstance(exc, (Exit, KeyboardInterrupt, SystemExit)): - raise exc from excgrp - - raise - - return True - - return None - - -@pytest.fixture(scope="module", params=get_available_backends()) -def anyio_backend(request: Any) -> Any: - return request.param - - -@pytest.fixture -def anyio_backend_name(anyio_backend: Any) -> str: - if isinstance(anyio_backend, str): - return anyio_backend - else: - return anyio_backend[0] - - -@pytest.fixture -def anyio_backend_options(anyio_backend: Any) -> dict[str, Any]: - if isinstance(anyio_backend, str): - return {} - else: - return anyio_backend[1] - - -class FreePortFactory: - """ - Manages port generation based on specified socket kind, ensuring no duplicate - ports are generated. - - This class provides functionality for generating available free ports on the - system. It is initialized with a specific socket kind and can generate ports - for given address families while avoiding reuse of previously generated ports. - - Users should not instantiate this class directly, but use the - ``free_tcp_port_factory`` and ``free_udp_port_factory`` fixtures instead. For simple - uses cases, ``free_tcp_port`` and ``free_udp_port`` can be used instead. - """ - - def __init__(self, kind: socket.SocketKind) -> None: - self._kind = kind - self._generated = set[int]() - - @property - def kind(self) -> socket.SocketKind: - """ - The type of socket connection (e.g., :data:`~socket.SOCK_STREAM` or - :data:`~socket.SOCK_DGRAM`) used to bind for checking port availability - - """ - return self._kind - - def __call__(self, family: socket.AddressFamily | None = None) -> int: - """ - Return an unbound port for the given address family. - - :param family: if omitted, both IPv4 and IPv6 addresses will be tried - :return: a port number - - """ - if family is not None: - families = [family] - else: - families = [socket.AF_INET] - if socket.has_ipv6: - families.append(socket.AF_INET6) - - while True: - port = 0 - with ExitStack() as stack: - for family in families: - sock = stack.enter_context(socket.socket(family, self._kind)) - addr = "::1" if family == socket.AF_INET6 else "127.0.0.1" - try: - sock.bind((addr, port)) - except OSError: - break - - if not port: - port = sock.getsockname()[1] - else: - if port not in self._generated: - self._generated.add(port) - return port - - -@pytest.fixture(scope="session") -def free_tcp_port_factory() -> FreePortFactory: - return FreePortFactory(socket.SOCK_STREAM) - - -@pytest.fixture(scope="session") -def free_udp_port_factory() -> FreePortFactory: - return FreePortFactory(socket.SOCK_DGRAM) - - -@pytest.fixture -def free_tcp_port(free_tcp_port_factory: Callable[[], int]) -> int: - return free_tcp_port_factory() - - -@pytest.fixture -def free_udp_port(free_udp_port_factory: Callable[[], int]) -> int: - return free_udp_port_factory() diff --git a/functions/python/data-processor/anyio/streams/__init__.py b/functions/python/data-processor/anyio/streams/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/anyio/streams/buffered.py b/functions/python/data-processor/anyio/streams/buffered.py deleted file mode 100644 index 57c7cd749..000000000 --- a/functions/python/data-processor/anyio/streams/buffered.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "BufferedByteReceiveStream", - "BufferedByteStream", - "BufferedConnectable", -) - -import sys -from collections.abc import Callable, Iterable, Mapping -from dataclasses import dataclass, field -from typing import Any, SupportsIndex - -from .. import ClosedResourceError, DelimiterNotFound, EndOfStream, IncompleteRead -from ..abc import ( - AnyByteReceiveStream, - AnyByteStream, - AnyByteStreamConnectable, - ByteReceiveStream, - ByteStream, - ByteStreamConnectable, -) - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - - -@dataclass(eq=False) -class BufferedByteReceiveStream(ByteReceiveStream): - """ - Wraps any bytes-based receive stream and uses a buffer to provide sophisticated - receiving capabilities in the form of a byte stream. - """ - - receive_stream: AnyByteReceiveStream - _buffer: bytearray = field(init=False, default_factory=bytearray) - _closed: bool = field(init=False, default=False) - - async def aclose(self) -> None: - await self.receive_stream.aclose() - self._closed = True - - @property - def buffer(self) -> bytes: - """The bytes currently in the buffer.""" - return bytes(self._buffer) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.receive_stream.extra_attributes - - def feed_data(self, data: Iterable[SupportsIndex], /) -> None: - """ - Append data directly into the buffer. - - Any data in the buffer will be consumed by receive operations before receiving - anything from the wrapped stream. - - :param data: the data to append to the buffer (can be bytes or anything else - that supports ``__index__()``) - - """ - self._buffer.extend(data) - - async def receive(self, max_bytes: int = 65536) -> bytes: - if self._closed: - raise ClosedResourceError - - if self._buffer: - chunk = bytes(self._buffer[:max_bytes]) - del self._buffer[:max_bytes] - return chunk - elif isinstance(self.receive_stream, ByteReceiveStream): - return await self.receive_stream.receive(max_bytes) - else: - # With a bytes-oriented object stream, we need to handle any surplus bytes - # we get from the receive() call - chunk = await self.receive_stream.receive() - if len(chunk) > max_bytes: - # Save the surplus bytes in the buffer - self._buffer.extend(chunk[max_bytes:]) - return chunk[:max_bytes] - else: - return chunk - - async def receive_exactly(self, nbytes: int) -> bytes: - """ - Read exactly the given amount of bytes from the stream. - - :param nbytes: the number of bytes to read - :return: the bytes read - :raises ~anyio.IncompleteRead: if the stream was closed before the requested - amount of bytes could be read from the stream - - """ - while True: - remaining = nbytes - len(self._buffer) - if remaining <= 0: - retval = self._buffer[:nbytes] - del self._buffer[:nbytes] - return bytes(retval) - - try: - if isinstance(self.receive_stream, ByteReceiveStream): - chunk = await self.receive_stream.receive(remaining) - else: - chunk = await self.receive_stream.receive() - except EndOfStream as exc: - raise IncompleteRead from exc - - self._buffer.extend(chunk) - - async def receive_until(self, delimiter: bytes, max_bytes: int) -> bytes: - """ - Read from the stream until the delimiter is found or max_bytes have been read. - - :param delimiter: the marker to look for in the stream - :param max_bytes: maximum number of bytes that will be read before raising - :exc:`~anyio.DelimiterNotFound` - :return: the bytes read (not including the delimiter) - :raises ~anyio.IncompleteRead: if the stream was closed before the delimiter - was found - :raises ~anyio.DelimiterNotFound: if the delimiter is not found within the - bytes read up to the maximum allowed - - """ - delimiter_size = len(delimiter) - offset = 0 - while True: - # Check if the delimiter can be found in the current buffer - index = self._buffer.find(delimiter, offset) - if index >= 0: - found = self._buffer[:index] - del self._buffer[: index + len(delimiter) :] - return bytes(found) - - # Check if the buffer is already at or over the limit - if len(self._buffer) >= max_bytes: - raise DelimiterNotFound(max_bytes) - - # Read more data into the buffer from the socket - try: - data = await self.receive_stream.receive() - except EndOfStream as exc: - raise IncompleteRead from exc - - # Move the offset forward and add the new data to the buffer - offset = max(len(self._buffer) - delimiter_size + 1, 0) - self._buffer.extend(data) - - -class BufferedByteStream(BufferedByteReceiveStream, ByteStream): - """ - A full-duplex variant of :class:`BufferedByteReceiveStream`. All writes are passed - through to the wrapped stream as-is. - """ - - def __init__(self, stream: AnyByteStream): - """ - :param stream: the stream to be wrapped - - """ - super().__init__(stream) - self._stream = stream - - @override - async def send_eof(self) -> None: - await self._stream.send_eof() - - @override - async def send(self, item: bytes) -> None: - await self._stream.send(item) - - -class BufferedConnectable(ByteStreamConnectable): - def __init__(self, connectable: AnyByteStreamConnectable): - """ - :param connectable: the connectable to wrap - - """ - self.connectable = connectable - - @override - async def connect(self) -> BufferedByteStream: - stream = await self.connectable.connect() - return BufferedByteStream(stream) diff --git a/functions/python/data-processor/anyio/streams/file.py b/functions/python/data-processor/anyio/streams/file.py deleted file mode 100644 index 79c3d500b..000000000 --- a/functions/python/data-processor/anyio/streams/file.py +++ /dev/null @@ -1,154 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "FileReadStream", - "FileStreamAttribute", - "FileWriteStream", -) - -from collections.abc import Callable, Mapping -from io import SEEK_SET, UnsupportedOperation -from os import PathLike -from pathlib import Path -from typing import IO, Any - -from .. import ( - BrokenResourceError, - ClosedResourceError, - EndOfStream, - TypedAttributeSet, - to_thread, - typed_attribute, -) -from ..abc import ByteReceiveStream, ByteSendStream - - -class FileStreamAttribute(TypedAttributeSet): - #: the open file descriptor - file: IO[bytes] = typed_attribute() - #: the path of the file on the file system, if available (file must be a real file) - path: Path = typed_attribute() - #: the file number, if available (file must be a real file or a TTY) - fileno: int = typed_attribute() - - -class _BaseFileStream: - def __init__(self, file: IO[bytes]): - self._file = file - - async def aclose(self) -> None: - await to_thread.run_sync(self._file.close) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - attributes: dict[Any, Callable[[], Any]] = { - FileStreamAttribute.file: lambda: self._file, - } - - if hasattr(self._file, "name"): - attributes[FileStreamAttribute.path] = lambda: Path(self._file.name) - - try: - self._file.fileno() - except UnsupportedOperation: - pass - else: - attributes[FileStreamAttribute.fileno] = lambda: self._file.fileno() - - return attributes - - -class FileReadStream(_BaseFileStream, ByteReceiveStream): - """ - A byte stream that reads from a file in the file system. - - :param file: a file that has been opened for reading in binary mode - - .. versionadded:: 3.0 - """ - - @classmethod - async def from_path(cls, path: str | PathLike[str]) -> FileReadStream: - """ - Create a file read stream by opening the given file. - - :param path: path of the file to read from - - """ - file = await to_thread.run_sync(Path(path).open, "rb") - return cls(file) - - async def receive(self, max_bytes: int = 65536) -> bytes: - try: - data = await to_thread.run_sync(self._file.read, max_bytes) - except ValueError: - raise ClosedResourceError from None - except OSError as exc: - raise BrokenResourceError from exc - - if data: - return data - else: - raise EndOfStream - - async def seek(self, position: int, whence: int = SEEK_SET) -> int: - """ - Seek the file to the given position. - - .. seealso:: :meth:`io.IOBase.seek` - - .. note:: Not all file descriptors are seekable. - - :param position: position to seek the file to - :param whence: controls how ``position`` is interpreted - :return: the new absolute position - :raises OSError: if the file is not seekable - - """ - return await to_thread.run_sync(self._file.seek, position, whence) - - async def tell(self) -> int: - """ - Return the current stream position. - - .. note:: Not all file descriptors are seekable. - - :return: the current absolute position - :raises OSError: if the file is not seekable - - """ - return await to_thread.run_sync(self._file.tell) - - -class FileWriteStream(_BaseFileStream, ByteSendStream): - """ - A byte stream that writes to a file in the file system. - - :param file: a file that has been opened for writing in binary mode - - .. versionadded:: 3.0 - """ - - @classmethod - async def from_path( - cls, path: str | PathLike[str], append: bool = False - ) -> FileWriteStream: - """ - Create a file write stream by opening the given file for writing. - - :param path: path of the file to write to - :param append: if ``True``, open the file for appending; if ``False``, any - existing file at the given path will be truncated - - """ - mode = "ab" if append else "wb" - file = await to_thread.run_sync(Path(path).open, mode) - return cls(file) - - async def send(self, item: bytes) -> None: - try: - await to_thread.run_sync(self._file.write, item) - except ValueError: - raise ClosedResourceError from None - except OSError as exc: - raise BrokenResourceError from exc diff --git a/functions/python/data-processor/anyio/streams/memory.py b/functions/python/data-processor/anyio/streams/memory.py deleted file mode 100644 index a3fa0c3d9..000000000 --- a/functions/python/data-processor/anyio/streams/memory.py +++ /dev/null @@ -1,325 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "MemoryObjectReceiveStream", - "MemoryObjectSendStream", - "MemoryObjectStreamStatistics", -) - -import warnings -from collections import OrderedDict, deque -from dataclasses import dataclass, field -from types import TracebackType -from typing import Generic, NamedTuple, TypeVar - -from .. import ( - BrokenResourceError, - ClosedResourceError, - EndOfStream, - WouldBlock, -) -from .._core._testing import TaskInfo, get_current_task -from ..abc import Event, ObjectReceiveStream, ObjectSendStream -from ..lowlevel import checkpoint - -T_Item = TypeVar("T_Item") -T_co = TypeVar("T_co", covariant=True) -T_contra = TypeVar("T_contra", contravariant=True) - - -class MemoryObjectStreamStatistics(NamedTuple): - current_buffer_used: int #: number of items stored in the buffer - #: maximum number of items that can be stored on this stream (or :data:`math.inf`) - max_buffer_size: float - open_send_streams: int #: number of unclosed clones of the send stream - open_receive_streams: int #: number of unclosed clones of the receive stream - #: number of tasks blocked on :meth:`MemoryObjectSendStream.send` - tasks_waiting_send: int - #: number of tasks blocked on :meth:`MemoryObjectReceiveStream.receive` - tasks_waiting_receive: int - - -@dataclass(eq=False) -class _MemoryObjectItemReceiver(Generic[T_Item]): - task_info: TaskInfo = field(init=False, default_factory=get_current_task) - item: T_Item = field(init=False) - - def __repr__(self) -> str: - # When item is not defined, we get following error with default __repr__: - # AttributeError: 'MemoryObjectItemReceiver' object has no attribute 'item' - item = getattr(self, "item", None) - return f"{self.__class__.__name__}(task_info={self.task_info}, item={item!r})" - - -@dataclass(eq=False) -class _MemoryObjectStreamState(Generic[T_Item]): - max_buffer_size: float = field() - buffer: deque[T_Item] = field(init=False, default_factory=deque) - open_send_channels: int = field(init=False, default=0) - open_receive_channels: int = field(init=False, default=0) - waiting_receivers: OrderedDict[Event, _MemoryObjectItemReceiver[T_Item]] = field( - init=False, default_factory=OrderedDict - ) - waiting_senders: OrderedDict[Event, T_Item] = field( - init=False, default_factory=OrderedDict - ) - - def statistics(self) -> MemoryObjectStreamStatistics: - return MemoryObjectStreamStatistics( - len(self.buffer), - self.max_buffer_size, - self.open_send_channels, - self.open_receive_channels, - len(self.waiting_senders), - len(self.waiting_receivers), - ) - - -@dataclass(eq=False) -class MemoryObjectReceiveStream(Generic[T_co], ObjectReceiveStream[T_co]): - _state: _MemoryObjectStreamState[T_co] - _closed: bool = field(init=False, default=False) - - def __post_init__(self) -> None: - self._state.open_receive_channels += 1 - - def receive_nowait(self) -> T_co: - """ - Receive the next item if it can be done without waiting. - - :return: the received item - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.EndOfStream: if the buffer is empty and this stream has been - closed from the sending end - :raises ~anyio.WouldBlock: if there are no items in the buffer and no tasks - waiting to send - - """ - if self._closed: - raise ClosedResourceError - - if self._state.waiting_senders: - # Get the item from the next sender - send_event, item = self._state.waiting_senders.popitem(last=False) - self._state.buffer.append(item) - send_event.set() - - if self._state.buffer: - return self._state.buffer.popleft() - elif not self._state.open_send_channels: - raise EndOfStream - - raise WouldBlock - - async def receive(self) -> T_co: - await checkpoint() - try: - return self.receive_nowait() - except WouldBlock: - # Add ourselves in the queue - receive_event = Event() - receiver = _MemoryObjectItemReceiver[T_co]() - self._state.waiting_receivers[receive_event] = receiver - - try: - await receive_event.wait() - finally: - self._state.waiting_receivers.pop(receive_event, None) - - try: - return receiver.item - except AttributeError: - raise EndOfStream from None - - def clone(self) -> MemoryObjectReceiveStream[T_co]: - """ - Create a clone of this receive stream. - - Each clone can be closed separately. Only when all clones have been closed will - the receiving end of the memory stream be considered closed by the sending ends. - - :return: the cloned stream - - """ - if self._closed: - raise ClosedResourceError - - return MemoryObjectReceiveStream(_state=self._state) - - def close(self) -> None: - """ - Close the stream. - - This works the exact same way as :meth:`aclose`, but is provided as a special - case for the benefit of synchronous callbacks. - - """ - if not self._closed: - self._closed = True - self._state.open_receive_channels -= 1 - if self._state.open_receive_channels == 0: - send_events = list(self._state.waiting_senders.keys()) - for event in send_events: - event.set() - - async def aclose(self) -> None: - self.close() - - def statistics(self) -> MemoryObjectStreamStatistics: - """ - Return statistics about the current state of this stream. - - .. versionadded:: 3.0 - """ - return self._state.statistics() - - def __enter__(self) -> MemoryObjectReceiveStream[T_co]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def __del__(self) -> None: - if not self._closed: - warnings.warn( - f"Unclosed <{self.__class__.__name__} at {id(self):x}>", - ResourceWarning, - stacklevel=1, - source=self, - ) - - -@dataclass(eq=False) -class MemoryObjectSendStream(Generic[T_contra], ObjectSendStream[T_contra]): - _state: _MemoryObjectStreamState[T_contra] - _closed: bool = field(init=False, default=False) - - def __post_init__(self) -> None: - self._state.open_send_channels += 1 - - def send_nowait(self, item: T_contra) -> None: - """ - Send an item immediately if it can be done without waiting. - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.BrokenResourceError: if the stream has been closed from the - receiving end - :raises ~anyio.WouldBlock: if the buffer is full and there are no tasks waiting - to receive - - """ - if self._closed: - raise ClosedResourceError - if not self._state.open_receive_channels: - raise BrokenResourceError - - while self._state.waiting_receivers: - receive_event, receiver = self._state.waiting_receivers.popitem(last=False) - if not receiver.task_info.has_pending_cancellation(): - receiver.item = item - receive_event.set() - return - - if len(self._state.buffer) < self._state.max_buffer_size: - self._state.buffer.append(item) - else: - raise WouldBlock - - async def send(self, item: T_contra) -> None: - """ - Send an item to the stream. - - If the buffer is full, this method blocks until there is again room in the - buffer or the item can be sent directly to a receiver. - - :param item: the item to send - :raises ~anyio.ClosedResourceError: if this send stream has been closed - :raises ~anyio.BrokenResourceError: if the stream has been closed from the - receiving end - - """ - await checkpoint() - try: - self.send_nowait(item) - except WouldBlock: - # Wait until there's someone on the receiving end - send_event = Event() - self._state.waiting_senders[send_event] = item - try: - await send_event.wait() - except BaseException: - self._state.waiting_senders.pop(send_event, None) - raise - - if send_event in self._state.waiting_senders: - del self._state.waiting_senders[send_event] - raise BrokenResourceError from None - - def clone(self) -> MemoryObjectSendStream[T_contra]: - """ - Create a clone of this send stream. - - Each clone can be closed separately. Only when all clones have been closed will - the sending end of the memory stream be considered closed by the receiving ends. - - :return: the cloned stream - - """ - if self._closed: - raise ClosedResourceError - - return MemoryObjectSendStream(_state=self._state) - - def close(self) -> None: - """ - Close the stream. - - This works the exact same way as :meth:`aclose`, but is provided as a special - case for the benefit of synchronous callbacks. - - """ - if not self._closed: - self._closed = True - self._state.open_send_channels -= 1 - if self._state.open_send_channels == 0: - receive_events = list(self._state.waiting_receivers.keys()) - self._state.waiting_receivers.clear() - for event in receive_events: - event.set() - - async def aclose(self) -> None: - self.close() - - def statistics(self) -> MemoryObjectStreamStatistics: - """ - Return statistics about the current state of this stream. - - .. versionadded:: 3.0 - """ - return self._state.statistics() - - def __enter__(self) -> MemoryObjectSendStream[T_contra]: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_val: BaseException | None, - exc_tb: TracebackType | None, - ) -> None: - self.close() - - def __del__(self) -> None: - if not self._closed: - warnings.warn( - f"Unclosed <{self.__class__.__name__} at {id(self):x}>", - ResourceWarning, - stacklevel=1, - source=self, - ) diff --git a/functions/python/data-processor/anyio/streams/stapled.py b/functions/python/data-processor/anyio/streams/stapled.py deleted file mode 100644 index 9248b68ab..000000000 --- a/functions/python/data-processor/anyio/streams/stapled.py +++ /dev/null @@ -1,147 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "MultiListener", - "StapledByteStream", - "StapledObjectStream", -) - -from collections.abc import Callable, Mapping, Sequence -from dataclasses import dataclass -from typing import Any, Generic, TypeVar - -from ..abc import ( - ByteReceiveStream, - ByteSendStream, - ByteStream, - Listener, - ObjectReceiveStream, - ObjectSendStream, - ObjectStream, - TaskGroup, -) - -T_Item = TypeVar("T_Item") -T_Stream = TypeVar("T_Stream") - - -@dataclass(eq=False) -class StapledByteStream(ByteStream): - """ - Combines two byte streams into a single, bidirectional byte stream. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param ByteSendStream send_stream: the sending byte stream - :param ByteReceiveStream receive_stream: the receiving byte stream - """ - - send_stream: ByteSendStream - receive_stream: ByteReceiveStream - - async def receive(self, max_bytes: int = 65536) -> bytes: - return await self.receive_stream.receive(max_bytes) - - async def send(self, item: bytes) -> None: - await self.send_stream.send(item) - - async def send_eof(self) -> None: - await self.send_stream.aclose() - - async def aclose(self) -> None: - await self.send_stream.aclose() - await self.receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.send_stream.extra_attributes, - **self.receive_stream.extra_attributes, - } - - -@dataclass(eq=False) -class StapledObjectStream(Generic[T_Item], ObjectStream[T_Item]): - """ - Combines two object streams into a single, bidirectional object stream. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param ObjectSendStream send_stream: the sending object stream - :param ObjectReceiveStream receive_stream: the receiving object stream - """ - - send_stream: ObjectSendStream[T_Item] - receive_stream: ObjectReceiveStream[T_Item] - - async def receive(self) -> T_Item: - return await self.receive_stream.receive() - - async def send(self, item: T_Item) -> None: - await self.send_stream.send(item) - - async def send_eof(self) -> None: - await self.send_stream.aclose() - - async def aclose(self) -> None: - await self.send_stream.aclose() - await self.receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.send_stream.extra_attributes, - **self.receive_stream.extra_attributes, - } - - -@dataclass(eq=False) -class MultiListener(Generic[T_Stream], Listener[T_Stream]): - """ - Combines multiple listeners into one, serving connections from all of them at once. - - Any MultiListeners in the given collection of listeners will have their listeners - moved into this one. - - Extra attributes are provided from each listener, with each successive listener - overriding any conflicting attributes from the previous one. - - :param listeners: listeners to serve - :type listeners: Sequence[Listener[T_Stream]] - """ - - listeners: Sequence[Listener[T_Stream]] - - def __post_init__(self) -> None: - listeners: list[Listener[T_Stream]] = [] - for listener in self.listeners: - if isinstance(listener, MultiListener): - listeners.extend(listener.listeners) - del listener.listeners[:] # type: ignore[attr-defined] - else: - listeners.append(listener) - - self.listeners = listeners - - async def serve( - self, handler: Callable[[T_Stream], Any], task_group: TaskGroup | None = None - ) -> None: - from .. import create_task_group - - async with create_task_group() as tg: - for listener in self.listeners: - tg.start_soon(listener.serve, handler, task_group) - - async def aclose(self) -> None: - for listener in self.listeners: - await listener.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - attributes: dict = {} - for listener in self.listeners: - attributes.update(listener.extra_attributes) - - return attributes diff --git a/functions/python/data-processor/anyio/streams/text.py b/functions/python/data-processor/anyio/streams/text.py deleted file mode 100644 index 296cd2504..000000000 --- a/functions/python/data-processor/anyio/streams/text.py +++ /dev/null @@ -1,176 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "TextConnectable", - "TextReceiveStream", - "TextSendStream", - "TextStream", -) - -import codecs -import sys -from collections.abc import Callable, Mapping -from dataclasses import InitVar, dataclass, field -from typing import Any - -from ..abc import ( - AnyByteReceiveStream, - AnyByteSendStream, - AnyByteStream, - AnyByteStreamConnectable, - ObjectReceiveStream, - ObjectSendStream, - ObjectStream, - ObjectStreamConnectable, -) - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - - -@dataclass(eq=False) -class TextReceiveStream(ObjectReceiveStream[str]): - """ - Stream wrapper that decodes bytes to strings using the given encoding. - - Decoding is done using :class:`~codecs.IncrementalDecoder` which returns any - completely received unicode characters as soon as they come in. - - :param transport_stream: any bytes-based receive stream - :param encoding: character encoding to use for decoding bytes to strings (defaults - to ``utf-8``) - :param errors: handling scheme for decoding errors (defaults to ``strict``; see the - `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteReceiveStream - encoding: InitVar[str] = "utf-8" - errors: InitVar[str] = "strict" - _decoder: codecs.IncrementalDecoder = field(init=False) - - def __post_init__(self, encoding: str, errors: str) -> None: - decoder_class = codecs.getincrementaldecoder(encoding) - self._decoder = decoder_class(errors=errors) - - async def receive(self) -> str: - while True: - chunk = await self.transport_stream.receive() - decoded = self._decoder.decode(chunk) - if decoded: - return decoded - - async def aclose(self) -> None: - await self.transport_stream.aclose() - self._decoder.reset() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.transport_stream.extra_attributes - - -@dataclass(eq=False) -class TextSendStream(ObjectSendStream[str]): - """ - Sends strings to the wrapped stream as bytes using the given encoding. - - :param AnyByteSendStream transport_stream: any bytes-based send stream - :param str encoding: character encoding to use for encoding strings to bytes - (defaults to ``utf-8``) - :param str errors: handling scheme for encoding errors (defaults to ``strict``; see - the `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteSendStream - encoding: InitVar[str] = "utf-8" - errors: str = "strict" - _encoder: Callable[..., tuple[bytes, int]] = field(init=False) - - def __post_init__(self, encoding: str) -> None: - self._encoder = codecs.getencoder(encoding) - - async def send(self, item: str) -> None: - encoded = self._encoder(item, self.errors)[0] - await self.transport_stream.send(encoded) - - async def aclose(self) -> None: - await self.transport_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return self.transport_stream.extra_attributes - - -@dataclass(eq=False) -class TextStream(ObjectStream[str]): - """ - A bidirectional stream that decodes bytes to strings on receive and encodes strings - to bytes on send. - - Extra attributes will be provided from both streams, with the receive stream - providing the values in case of a conflict. - - :param AnyByteStream transport_stream: any bytes-based stream - :param str encoding: character encoding to use for encoding/decoding strings to/from - bytes (defaults to ``utf-8``) - :param str errors: handling scheme for encoding errors (defaults to ``strict``; see - the `codecs module documentation`_ for a comprehensive list of options) - - .. _codecs module documentation: - https://docs.python.org/3/library/codecs.html#codec-objects - """ - - transport_stream: AnyByteStream - encoding: InitVar[str] = "utf-8" - errors: InitVar[str] = "strict" - _receive_stream: TextReceiveStream = field(init=False) - _send_stream: TextSendStream = field(init=False) - - def __post_init__(self, encoding: str, errors: str) -> None: - self._receive_stream = TextReceiveStream( - self.transport_stream, encoding=encoding, errors=errors - ) - self._send_stream = TextSendStream( - self.transport_stream, encoding=encoding, errors=errors - ) - - async def receive(self) -> str: - return await self._receive_stream.receive() - - async def send(self, item: str) -> None: - await self._send_stream.send(item) - - async def send_eof(self) -> None: - await self.transport_stream.send_eof() - - async def aclose(self) -> None: - await self._send_stream.aclose() - await self._receive_stream.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self._send_stream.extra_attributes, - **self._receive_stream.extra_attributes, - } - - -class TextConnectable(ObjectStreamConnectable[str]): - def __init__(self, connectable: AnyByteStreamConnectable): - """ - :param connectable: the bytestream endpoint to wrap - - """ - self.connectable = connectable - - @override - async def connect(self) -> TextStream: - stream = await self.connectable.connect() - return TextStream(stream) diff --git a/functions/python/data-processor/anyio/streams/tls.py b/functions/python/data-processor/anyio/streams/tls.py deleted file mode 100644 index e2a7ca5b1..000000000 --- a/functions/python/data-processor/anyio/streams/tls.py +++ /dev/null @@ -1,421 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "TLSAttribute", - "TLSConnectable", - "TLSListener", - "TLSStream", -) - -import logging -import re -import ssl -import sys -from collections.abc import Callable, Mapping -from dataclasses import dataclass -from functools import wraps -from ssl import SSLContext -from typing import Any, TypeAlias, TypeVar - -from .. import ( - BrokenResourceError, - EndOfStream, - aclose_forcefully, - get_cancelled_exc_class, - to_thread, -) -from .._core._typedattr import TypedAttributeSet, typed_attribute -from ..abc import ( - AnyByteStream, - AnyByteStreamConnectable, - ByteStream, - ByteStreamConnectable, - Listener, - TaskGroup, -) - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if sys.version_info >= (3, 12): - from typing import override -else: - from typing_extensions import override - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") -_PCTRTT: TypeAlias = tuple[tuple[str, str], ...] -_PCTRTTT: TypeAlias = tuple[_PCTRTT, ...] - - -class TLSAttribute(TypedAttributeSet): - """Contains Transport Layer Security related attributes.""" - - #: the selected ALPN protocol - alpn_protocol: str | None = typed_attribute() - #: the channel binding for type ``tls-unique`` - channel_binding_tls_unique: bytes = typed_attribute() - #: the selected cipher - cipher: tuple[str, str, int] = typed_attribute() - #: the peer certificate in dictionary form (see :meth:`ssl.SSLSocket.getpeercert` - # for more information) - peer_certificate: None | (dict[str, str | _PCTRTTT | _PCTRTT]) = typed_attribute() - #: the peer certificate in binary form - peer_certificate_binary: bytes | None = typed_attribute() - #: ``True`` if this is the server side of the connection - server_side: bool = typed_attribute() - #: ciphers shared by the client during the TLS handshake (``None`` if this is the - #: client side) - shared_ciphers: list[tuple[str, str, int]] | None = typed_attribute() - #: the :class:`~ssl.SSLObject` used for encryption - ssl_object: ssl.SSLObject = typed_attribute() - #: ``True`` if this stream does (and expects) a closing TLS handshake when the - #: stream is being closed - standard_compatible: bool = typed_attribute() - #: the TLS protocol version (e.g. ``TLSv1.2``) - tls_version: str = typed_attribute() - - -@dataclass(eq=False) -class TLSStream(ByteStream): - """ - A stream wrapper that encrypts all sent data and decrypts received data. - - This class has no public initializer; use :meth:`wrap` instead. - All extra attributes from :class:`~TLSAttribute` are supported. - - :var AnyByteStream transport_stream: the wrapped stream - - """ - - transport_stream: AnyByteStream - standard_compatible: bool - _ssl_object: ssl.SSLObject - _read_bio: ssl.MemoryBIO - _write_bio: ssl.MemoryBIO - - @classmethod - async def wrap( - cls, - transport_stream: AnyByteStream, - *, - server_side: bool | None = None, - hostname: str | None = None, - ssl_context: ssl.SSLContext | None = None, - standard_compatible: bool = True, - ) -> TLSStream: - """ - Wrap an existing stream with Transport Layer Security. - - This performs a TLS handshake with the peer. - - :param transport_stream: a bytes-transporting stream to wrap - :param server_side: ``True`` if this is the server side of the connection, - ``False`` if this is the client side (if omitted, will be set to ``False`` - if ``hostname`` has been provided, ``False`` otherwise). Used only to create - a default context when an explicit context has not been provided. - :param hostname: host name of the peer (if host name checking is desired) - :param ssl_context: the SSLContext object to use (if not provided, a secure - default will be created) - :param standard_compatible: if ``False``, skip the closing handshake when - closing the connection, and don't raise an exception if the peer does the - same - :raises ~ssl.SSLError: if the TLS handshake fails - - """ - if server_side is None: - server_side = not hostname - - if not ssl_context: - purpose = ( - ssl.Purpose.CLIENT_AUTH if server_side else ssl.Purpose.SERVER_AUTH - ) - ssl_context = ssl.create_default_context(purpose) - - # Re-enable detection of unexpected EOFs if it was disabled by Python - if hasattr(ssl, "OP_IGNORE_UNEXPECTED_EOF"): - ssl_context.options &= ~ssl.OP_IGNORE_UNEXPECTED_EOF - - bio_in = ssl.MemoryBIO() - bio_out = ssl.MemoryBIO() - - # External SSLContext implementations may do blocking I/O in wrap_bio(), - # but the standard library implementation won't - if type(ssl_context) is ssl.SSLContext: - ssl_object = ssl_context.wrap_bio( - bio_in, bio_out, server_side=server_side, server_hostname=hostname - ) - else: - ssl_object = await to_thread.run_sync( - ssl_context.wrap_bio, - bio_in, - bio_out, - server_side, - hostname, - None, - ) - - wrapper = cls( - transport_stream=transport_stream, - standard_compatible=standard_compatible, - _ssl_object=ssl_object, - _read_bio=bio_in, - _write_bio=bio_out, - ) - await wrapper._call_sslobject_method(ssl_object.do_handshake) - return wrapper - - async def _call_sslobject_method( - self, func: Callable[[Unpack[PosArgsT]], T_Retval], *args: Unpack[PosArgsT] - ) -> T_Retval: - while True: - try: - result = func(*args) - except ssl.SSLWantReadError: - try: - # Flush any pending writes first - if self._write_bio.pending: - await self.transport_stream.send(self._write_bio.read()) - - data = await self.transport_stream.receive() - except EndOfStream: - self._read_bio.write_eof() - except OSError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - raise BrokenResourceError from exc - else: - self._read_bio.write(data) - except ssl.SSLWantWriteError: - await self.transport_stream.send(self._write_bio.read()) - except ssl.SSLSyscallError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - raise BrokenResourceError from exc - except ssl.SSLError as exc: - self._read_bio.write_eof() - self._write_bio.write_eof() - if isinstance(exc, ssl.SSLEOFError) or ( - exc.strerror and "UNEXPECTED_EOF_WHILE_READING" in exc.strerror - ): - if self.standard_compatible: - raise BrokenResourceError from exc - else: - raise EndOfStream from None - - raise - else: - # Flush any pending writes first - if self._write_bio.pending: - await self.transport_stream.send(self._write_bio.read()) - - return result - - async def unwrap(self) -> tuple[AnyByteStream, bytes]: - """ - Does the TLS closing handshake. - - :return: a tuple of (wrapped byte stream, bytes left in the read buffer) - - """ - await self._call_sslobject_method(self._ssl_object.unwrap) - self._read_bio.write_eof() - self._write_bio.write_eof() - return self.transport_stream, self._read_bio.read() - - async def aclose(self) -> None: - if self.standard_compatible: - try: - await self.unwrap() - except BaseException: - await aclose_forcefully(self.transport_stream) - raise - - await self.transport_stream.aclose() - - async def receive(self, max_bytes: int = 65536) -> bytes: - data = await self._call_sslobject_method(self._ssl_object.read, max_bytes) - if not data: - raise EndOfStream - - return data - - async def send(self, item: bytes) -> None: - await self._call_sslobject_method(self._ssl_object.write, item) - - async def send_eof(self) -> None: - tls_version = self.extra(TLSAttribute.tls_version) - match = re.match(r"TLSv(\d+)(?:\.(\d+))?", tls_version) - if match: - major, minor = int(match.group(1)), int(match.group(2) or 0) - if (major, minor) < (1, 3): - raise NotImplementedError( - f"send_eof() requires at least TLSv1.3; current " - f"session uses {tls_version}" - ) - - raise NotImplementedError( - "send_eof() has not yet been implemented for TLS streams" - ) - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - **self.transport_stream.extra_attributes, - TLSAttribute.alpn_protocol: self._ssl_object.selected_alpn_protocol, - TLSAttribute.channel_binding_tls_unique: ( - self._ssl_object.get_channel_binding - ), - TLSAttribute.cipher: self._ssl_object.cipher, - TLSAttribute.peer_certificate: lambda: self._ssl_object.getpeercert(False), - TLSAttribute.peer_certificate_binary: lambda: self._ssl_object.getpeercert( - True - ), - TLSAttribute.server_side: lambda: self._ssl_object.server_side, - TLSAttribute.shared_ciphers: lambda: ( - self._ssl_object.shared_ciphers() - if self._ssl_object.server_side - else None - ), - TLSAttribute.standard_compatible: lambda: self.standard_compatible, - TLSAttribute.ssl_object: lambda: self._ssl_object, - TLSAttribute.tls_version: self._ssl_object.version, - } - - -@dataclass(eq=False) -class TLSListener(Listener[TLSStream]): - """ - A convenience listener that wraps another listener and auto-negotiates a TLS session - on every accepted connection. - - If the TLS handshake times out or raises an exception, - :meth:`handle_handshake_error` is called to do whatever post-mortem processing is - deemed necessary. - - Supports only the :attr:`~TLSAttribute.standard_compatible` extra attribute. - - :param Listener listener: the listener to wrap - :param ssl_context: the SSL context object - :param standard_compatible: a flag passed through to :meth:`TLSStream.wrap` - :param handshake_timeout: time limit for the TLS handshake - (passed to :func:`~anyio.fail_after`) - """ - - listener: Listener[Any] - ssl_context: ssl.SSLContext - standard_compatible: bool = True - handshake_timeout: float = 30 - - @staticmethod - async def handle_handshake_error(exc: BaseException, stream: AnyByteStream) -> None: - """ - Handle an exception raised during the TLS handshake. - - This method does 3 things: - - #. Forcefully closes the original stream - #. Logs the exception (unless it was a cancellation exception) using the - ``anyio.streams.tls`` logger - #. Reraises the exception if it was a base exception or a cancellation exception - - :param exc: the exception - :param stream: the original stream - - """ - await aclose_forcefully(stream) - - # Log all except cancellation exceptions - if not isinstance(exc, get_cancelled_exc_class()): - # CPython (as of 3.11.5) returns incorrect `sys.exc_info()` here when using - # any asyncio implementation, so we explicitly pass the exception to log - # (https://github.com/python/cpython/issues/108668). Trio does not have this - # issue because it works around the CPython bug. - logging.getLogger(__name__).exception( - "Error during TLS handshake", exc_info=exc - ) - - # Only reraise base exceptions and cancellation exceptions - if not isinstance(exc, Exception) or isinstance(exc, get_cancelled_exc_class()): - raise - - async def serve( - self, - handler: Callable[[TLSStream], Any], - task_group: TaskGroup | None = None, - ) -> None: - @wraps(handler) - async def handler_wrapper(stream: AnyByteStream) -> None: - from .. import fail_after - - try: - with fail_after(self.handshake_timeout): - wrapped_stream = await TLSStream.wrap( - stream, - ssl_context=self.ssl_context, - standard_compatible=self.standard_compatible, - ) - except BaseException as exc: - await self.handle_handshake_error(exc, stream) - else: - await handler(wrapped_stream) - - await self.listener.serve(handler_wrapper, task_group) - - async def aclose(self) -> None: - await self.listener.aclose() - - @property - def extra_attributes(self) -> Mapping[Any, Callable[[], Any]]: - return { - TLSAttribute.standard_compatible: lambda: self.standard_compatible, - } - - -class TLSConnectable(ByteStreamConnectable): - """ - Wraps another connectable and does TLS negotiation after a successful connection. - - :param connectable: the connectable to wrap - :param hostname: host name of the server (if host name checking is desired) - :param ssl_context: the SSLContext object to use (if not provided, a secure default - will be created) - :param standard_compatible: if ``False``, skip the closing handshake when closing - the connection, and don't raise an exception if the server does the same - """ - - def __init__( - self, - connectable: AnyByteStreamConnectable, - *, - hostname: str | None = None, - ssl_context: ssl.SSLContext | None = None, - standard_compatible: bool = True, - ) -> None: - self.connectable = connectable - self.ssl_context: SSLContext = ssl_context or ssl.create_default_context( - ssl.Purpose.SERVER_AUTH - ) - if not isinstance(self.ssl_context, ssl.SSLContext): - raise TypeError( - "ssl_context must be an instance of ssl.SSLContext, not " - f"{type(self.ssl_context).__name__}" - ) - self.hostname = hostname - self.standard_compatible = standard_compatible - - @override - async def connect(self) -> TLSStream: - stream = await self.connectable.connect() - try: - return await TLSStream.wrap( - stream, - hostname=self.hostname, - ssl_context=self.ssl_context, - standard_compatible=self.standard_compatible, - ) - except BaseException: - await aclose_forcefully(stream) - raise diff --git a/functions/python/data-processor/anyio/to_interpreter.py b/functions/python/data-processor/anyio/to_interpreter.py deleted file mode 100644 index 694dbe77b..000000000 --- a/functions/python/data-processor/anyio/to_interpreter.py +++ /dev/null @@ -1,246 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "run_sync", - "current_default_interpreter_limiter", -) - -import atexit -import os -import sys -from collections import deque -from collections.abc import Callable -from typing import Any, Final, TypeVar - -from . import current_time, to_thread -from ._core._exceptions import BrokenWorkerInterpreter -from ._core._synchronization import CapacityLimiter -from .lowlevel import RunVar - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -if sys.version_info >= (3, 14): - from concurrent.interpreters import ExecutionFailed, create - - def _interp_call( - func: Callable[..., Any], args: tuple[Any, ...] - ) -> tuple[Any, bool]: - try: - retval = func(*args) - except BaseException as exc: - return exc, True - else: - return retval, False - - class _Worker: - last_used: float = 0 - - def __init__(self) -> None: - self._interpreter = create() - - def destroy(self) -> None: - self._interpreter.close() - - def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any, ...], - ) -> T_Retval: - try: - res, is_exception = self._interpreter.call(_interp_call, func, args) - except ExecutionFailed as exc: - raise BrokenWorkerInterpreter(exc.excinfo) from exc - - if is_exception: - raise res - - return res -elif sys.version_info >= (3, 13): - import _interpqueues - import _interpreters - - UNBOUND: Final = 2 # I have no clue how this works, but it was used in the stdlib - FMT_UNPICKLED: Final = 0 - FMT_PICKLED: Final = 1 - QUEUE_PICKLE_ARGS: Final = (FMT_PICKLED, UNBOUND) - QUEUE_UNPICKLE_ARGS: Final = (FMT_UNPICKLED, UNBOUND) - - _run_func = compile( - """ -import _interpqueues -from _interpreters import NotShareableError -from pickle import loads, dumps, HIGHEST_PROTOCOL - -QUEUE_PICKLE_ARGS = (1, 2) -QUEUE_UNPICKLE_ARGS = (0, 2) - -item = _interpqueues.get(queue_id)[0] -try: - func, args = loads(item) - retval = func(*args) -except BaseException as exc: - is_exception = True - retval = exc -else: - is_exception = False - -try: - _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_UNPICKLE_ARGS) -except NotShareableError: - retval = dumps(retval, HIGHEST_PROTOCOL) - _interpqueues.put(queue_id, (retval, is_exception), *QUEUE_PICKLE_ARGS) - """, - "", - "exec", - ) - - class _Worker: - last_used: float = 0 - - def __init__(self) -> None: - self._interpreter_id = _interpreters.create() - self._queue_id = _interpqueues.create(1, *QUEUE_UNPICKLE_ARGS) - _interpreters.set___main___attrs( - self._interpreter_id, {"queue_id": self._queue_id} - ) - - def destroy(self) -> None: - _interpqueues.destroy(self._queue_id) - _interpreters.destroy(self._interpreter_id) - - def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any, ...], - ) -> T_Retval: - import pickle - - item = pickle.dumps((func, args), pickle.HIGHEST_PROTOCOL) - _interpqueues.put(self._queue_id, item, *QUEUE_PICKLE_ARGS) - exc_info = _interpreters.exec(self._interpreter_id, _run_func) - if exc_info: - raise BrokenWorkerInterpreter(exc_info) - - res = _interpqueues.get(self._queue_id) - (res, is_exception), fmt = res[:2] - if fmt == FMT_PICKLED: - res = pickle.loads(res) - - if is_exception: - raise res - - return res -else: - - class _Worker: - last_used: float = 0 - - def __init__(self) -> None: - raise RuntimeError("subinterpreters require at least Python 3.13") - - def call( - self, - func: Callable[..., T_Retval], - args: tuple[Any, ...], - ) -> T_Retval: - raise NotImplementedError - - def destroy(self) -> None: - pass - - -DEFAULT_CPU_COUNT: Final = 8 # this is just an arbitrarily selected value -MAX_WORKER_IDLE_TIME = ( - 30 # seconds a subinterpreter can be idle before becoming eligible for pruning -) - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -_idle_workers = RunVar[deque[_Worker]]("_available_workers") -_default_interpreter_limiter = RunVar[CapacityLimiter]("_default_interpreter_limiter") - - -def _stop_workers(workers: deque[_Worker]) -> None: - for worker in workers: - worker.destroy() - - workers.clear() - - -async def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a subinterpreter. - - .. warning:: On Python 3.13, the :mod:`concurrent.interpreters` module was not yet - available, so the code path for that Python version relies on an undocumented, - private API. As such, it is recommended to not rely on this function for anything - mission-critical on Python 3.13. - - :param func: a callable - :param args: the positional arguments for the callable - :param limiter: capacity limiter to use to limit the total number of subinterpreters - running (if omitted, the default limiter is used) - :return: the result of the call - :raises BrokenWorkerInterpreter: if there's an internal error in a subinterpreter - - """ - if limiter is None: - limiter = current_default_interpreter_limiter() - - try: - idle_workers = _idle_workers.get() - except LookupError: - idle_workers = deque() - _idle_workers.set(idle_workers) - atexit.register(_stop_workers, idle_workers) - - async with limiter: - try: - worker = idle_workers.pop() - except IndexError: - worker = _Worker() - - try: - return await to_thread.run_sync( - worker.call, - func, - args, - limiter=limiter, - ) - finally: - # Prune workers that have been idle for too long - now = current_time() - while idle_workers: - if now - idle_workers[0].last_used <= MAX_WORKER_IDLE_TIME: - break - - await to_thread.run_sync(idle_workers.popleft().destroy, limiter=limiter) - - worker.last_used = current_time() - idle_workers.append(worker) - - -def current_default_interpreter_limiter() -> CapacityLimiter: - """ - Return the capacity limiter used by default to limit the number of concurrently - running subinterpreters. - - Defaults to the number of CPU cores. - - :return: a capacity limiter object - - """ - try: - return _default_interpreter_limiter.get() - except LookupError: - limiter = CapacityLimiter(os.cpu_count() or DEFAULT_CPU_COUNT) - _default_interpreter_limiter.set(limiter) - return limiter diff --git a/functions/python/data-processor/anyio/to_process.py b/functions/python/data-processor/anyio/to_process.py deleted file mode 100644 index b289234ec..000000000 --- a/functions/python/data-processor/anyio/to_process.py +++ /dev/null @@ -1,266 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "current_default_process_limiter", - "process_worker", - "run_sync", -) - -import os -import pickle -import subprocess -import sys -from collections import deque -from collections.abc import Callable -from importlib.util import module_from_spec, spec_from_file_location -from typing import TypeVar, cast - -from ._core._eventloop import current_time, get_async_backend, get_cancelled_exc_class -from ._core._exceptions import BrokenWorkerProcess -from ._core._subprocesses import open_process -from ._core._synchronization import CapacityLimiter -from ._core._tasks import CancelScope, fail_after -from .abc import ByteReceiveStream, ByteSendStream, Process -from .lowlevel import RunVar, checkpoint_if_cancelled -from .streams.buffered import BufferedByteReceiveStream - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -WORKER_MAX_IDLE_TIME = 300 # 5 minutes - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - -_process_pool_workers: RunVar[set[Process]] = RunVar("_process_pool_workers") -_process_pool_idle_workers: RunVar[deque[tuple[Process, float]]] = RunVar( - "_process_pool_idle_workers" -) -_default_process_limiter: RunVar[CapacityLimiter] = RunVar("_default_process_limiter") - - -async def run_sync( # type: ignore[return] - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - cancellable: bool = False, - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a worker process. - - If the ``cancellable`` option is enabled and the task waiting for its completion is - cancelled, the worker process running it will be abruptly terminated using SIGKILL - (or ``terminateProcess()`` on Windows). - - :param func: a callable - :param args: positional arguments for the callable - :param cancellable: ``True`` to allow cancellation of the operation while it's - running - :param limiter: capacity limiter to use to limit the total amount of processes - running (if omitted, the default limiter is used) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - :return: an awaitable that yields the return value of the function. - - """ - - async def send_raw_command(pickled_cmd: bytes) -> object: - try: - await stdin.send(pickled_cmd) - response = await buffered.receive_until(b"\n", 50) - status, length = response.split(b" ") - if status not in (b"RETURN", b"EXCEPTION"): - raise RuntimeError( - f"Worker process returned unexpected response: {response!r}" - ) - - pickled_response = await buffered.receive_exactly(int(length)) - except BaseException as exc: - workers.discard(process) - try: - process.kill() - with CancelScope(shield=True): - await process.aclose() - except ProcessLookupError: - pass - - if isinstance(exc, get_cancelled_exc_class()): - raise - else: - raise BrokenWorkerProcess from exc - - retval = pickle.loads(pickled_response) - if status == b"EXCEPTION": - assert isinstance(retval, BaseException) - raise retval - else: - return retval - - # First pickle the request before trying to reserve a worker process - await checkpoint_if_cancelled() - request = pickle.dumps(("run", func, args), protocol=pickle.HIGHEST_PROTOCOL) - - # If this is the first run in this event loop thread, set up the necessary variables - try: - workers = _process_pool_workers.get() - idle_workers = _process_pool_idle_workers.get() - except LookupError: - workers = set() - idle_workers = deque() - _process_pool_workers.set(workers) - _process_pool_idle_workers.set(idle_workers) - get_async_backend().setup_process_pool_exit_at_shutdown(workers) - - async with limiter or current_default_process_limiter(): - # Pop processes from the pool (starting from the most recently used) until we - # find one that hasn't exited yet - process: Process - while idle_workers: - process, idle_since = idle_workers.pop() - if process.returncode is None: - stdin = cast(ByteSendStream, process.stdin) - buffered = BufferedByteReceiveStream( - cast(ByteReceiveStream, process.stdout) - ) - - # Prune any other workers that have been idle for WORKER_MAX_IDLE_TIME - # seconds or longer - now = current_time() - killed_processes: list[Process] = [] - while idle_workers: - if now - idle_workers[0][1] < WORKER_MAX_IDLE_TIME: - break - - process_to_kill, idle_since = idle_workers.popleft() - process_to_kill.kill() - workers.remove(process_to_kill) - killed_processes.append(process_to_kill) - - with CancelScope(shield=True): - for killed_process in killed_processes: - await killed_process.aclose() - - break - - workers.remove(process) - else: - command = [sys.executable, "-u", "-m", __name__] - process = await open_process( - command, stdin=subprocess.PIPE, stdout=subprocess.PIPE - ) - try: - stdin = cast(ByteSendStream, process.stdin) - buffered = BufferedByteReceiveStream( - cast(ByteReceiveStream, process.stdout) - ) - with fail_after(20): - message = await buffered.receive(6) - - if message != b"READY\n": - raise BrokenWorkerProcess( - f"Worker process returned unexpected response: {message!r}" - ) - - main_module_path = getattr(sys.modules["__main__"], "__file__", None) - pickled = pickle.dumps( - ("init", sys.path, main_module_path), - protocol=pickle.HIGHEST_PROTOCOL, - ) - await send_raw_command(pickled) - except (BrokenWorkerProcess, get_cancelled_exc_class()): - raise - except BaseException as exc: - process.kill() - raise BrokenWorkerProcess( - "Error during worker process initialization" - ) from exc - - workers.add(process) - - with CancelScope(shield=not cancellable): - try: - return cast(T_Retval, await send_raw_command(request)) - finally: - if process in workers: - idle_workers.append((process, current_time())) - - -def current_default_process_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of worker - processes. - - :return: a capacity limiter object - - """ - try: - return _default_process_limiter.get() - except LookupError: - limiter = CapacityLimiter(os.cpu_count() or 2) - _default_process_limiter.set(limiter) - return limiter - - -def process_worker() -> None: - # Redirect standard streams to os.devnull so that user code won't interfere with the - # parent-worker communication - stdin = sys.stdin - stdout = sys.stdout - sys.stdin = open(os.devnull) - sys.stdout = open(os.devnull, "w") - - stdout.buffer.write(b"READY\n") - while True: - retval = exception = None - try: - command, *args = pickle.load(stdin.buffer) - except EOFError: - return - except BaseException as exc: - exception = exc - else: - if command == "run": - func, args = args - try: - retval = func(*args) - except BaseException as exc: - exception = exc - elif command == "init": - main_module_path: str | None - sys.path, main_module_path = args - del sys.modules["__main__"] - if main_module_path and os.path.isfile(main_module_path): - # Load the parent's main module but as __mp_main__ instead of - # __main__ (like multiprocessing does) to avoid infinite recursion - try: - spec = spec_from_file_location("__mp_main__", main_module_path) - if spec and spec.loader: - main = module_from_spec(spec) - spec.loader.exec_module(main) - sys.modules["__main__"] = main - except BaseException as exc: - exception = exc - try: - if exception is not None: - status = b"EXCEPTION" - pickled = pickle.dumps(exception, pickle.HIGHEST_PROTOCOL) - else: - status = b"RETURN" - pickled = pickle.dumps(retval, pickle.HIGHEST_PROTOCOL) - except BaseException as exc: - exception = exc - status = b"EXCEPTION" - pickled = pickle.dumps(exc, pickle.HIGHEST_PROTOCOL) - - stdout.buffer.write(b"%s %d\n" % (status, len(pickled))) - stdout.buffer.write(pickled) - - # Respect SIGTERM - if isinstance(exception, SystemExit): - raise exception - - -if __name__ == "__main__": - process_worker() diff --git a/functions/python/data-processor/anyio/to_thread.py b/functions/python/data-processor/anyio/to_thread.py deleted file mode 100644 index 83c79d1cd..000000000 --- a/functions/python/data-processor/anyio/to_thread.py +++ /dev/null @@ -1,78 +0,0 @@ -from __future__ import annotations - -__all__ = ( - "run_sync", - "current_default_thread_limiter", -) - -import sys -from collections.abc import Callable -from typing import TypeVar -from warnings import warn - -from ._core._eventloop import get_async_backend -from .abc import CapacityLimiter - -if sys.version_info >= (3, 11): - from typing import TypeVarTuple, Unpack -else: - from typing_extensions import TypeVarTuple, Unpack - -T_Retval = TypeVar("T_Retval") -PosArgsT = TypeVarTuple("PosArgsT") - - -async def run_sync( - func: Callable[[Unpack[PosArgsT]], T_Retval], - *args: Unpack[PosArgsT], - abandon_on_cancel: bool = False, - cancellable: bool | None = None, - limiter: CapacityLimiter | None = None, -) -> T_Retval: - """ - Call the given function with the given arguments in a worker thread. - - If the ``abandon_on_cancel`` option is enabled and the task waiting for its - completion is cancelled, the thread will still run its course but its - return value (or any raised exception) will be ignored. - - :param func: a callable - :param args: positional arguments for the callable - :param abandon_on_cancel: ``True`` to abandon the thread (leaving it to run - unchecked on own) if the host task is cancelled, ``False`` to ignore - cancellations in the host task until the operation has completed in the worker - thread - :param cancellable: deprecated alias of ``abandon_on_cancel``; will override - ``abandon_on_cancel`` if both parameters are passed - :param limiter: capacity limiter to use to limit the total amount of threads running - (if omitted, the default limiter is used) - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - :return: an awaitable that yields the return value of the function. - - """ - if cancellable is not None: - abandon_on_cancel = cancellable - warn( - "The `cancellable=` keyword argument to `anyio.to_thread.run_sync` is " - "deprecated since AnyIO 4.1.0; use `abandon_on_cancel=` instead", - DeprecationWarning, - stacklevel=2, - ) - - return await get_async_backend().run_sync_in_worker_thread( - func, args, abandon_on_cancel=abandon_on_cancel, limiter=limiter - ) - - -def current_default_thread_limiter() -> CapacityLimiter: - """ - Return the capacity limiter that is used by default to limit the number of - concurrent threads. - - :return: a capacity limiter object - :raises NoEventLoopError: if no supported asynchronous event loop is running in the - current thread - - """ - return get_async_backend().current_default_thread_limiter() diff --git a/functions/python/data-processor/bin/fastapi b/functions/python/data-processor/bin/fastapi deleted file mode 100755 index 971df26bf..000000000 --- a/functions/python/data-processor/bin/fastapi +++ /dev/null @@ -1,6 +0,0 @@ -#!/opt/homebrew/opt/python@3.14/bin/python3.14 -import sys -from fastapi.cli import main -if __name__ == '__main__': - sys.argv[0] = sys.argv[0].removesuffix('.exe') - sys.exit(main()) diff --git a/functions/python/data-processor/bin/idna b/functions/python/data-processor/bin/idna deleted file mode 100755 index 6f2e3bb71..000000000 --- a/functions/python/data-processor/bin/idna +++ /dev/null @@ -1,6 +0,0 @@ -#!/opt/homebrew/opt/python@3.14/bin/python3.14 -import sys -from idna.cli import main -if __name__ == '__main__': - sys.argv[0] = sys.argv[0].removesuffix('.exe') - sys.exit(main()) diff --git a/functions/python/data-processor/bin/uvicorn b/functions/python/data-processor/bin/uvicorn deleted file mode 100755 index 377286bbf..000000000 --- a/functions/python/data-processor/bin/uvicorn +++ /dev/null @@ -1,6 +0,0 @@ -#!/opt/homebrew/opt/python@3.14/bin/python3.14 -import sys -from uvicorn.main import main -if __name__ == '__main__': - sys.argv[0] = sys.argv[0].removesuffix('.exe') - sys.exit(main()) diff --git a/functions/python/data-processor/click-8.4.1.dist-info/INSTALLER b/functions/python/data-processor/click-8.4.1.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/click-8.4.1.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/click-8.4.1.dist-info/METADATA b/functions/python/data-processor/click-8.4.1.dist-info/METADATA deleted file mode 100644 index 21d7a0c3d..000000000 --- a/functions/python/data-processor/click-8.4.1.dist-info/METADATA +++ /dev/null @@ -1,84 +0,0 @@ -Metadata-Version: 2.4 -Name: click -Version: 8.4.1 -Summary: Composable command line interface toolkit -Maintainer-email: Pallets -Requires-Python: >=3.10 -Description-Content-Type: text/markdown -License-Expression: BSD-3-Clause -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Typing :: Typed -License-File: LICENSE.txt -Requires-Dist: colorama; platform_system == 'Windows' -Project-URL: Changes, https://click.palletsprojects.com/page/changes/ -Project-URL: Chat, https://discord.gg/pallets -Project-URL: Documentation, https://click.palletsprojects.com/ -Project-URL: Donate, https://palletsprojects.com/donate -Project-URL: Source, https://github.com/pallets/click/ - -
- -# Click - -Click is a Python package for creating beautiful command line interfaces -in a composable way with as little code as necessary. It's the "Command -Line Interface Creation Kit". It's highly configurable but comes with -sensible defaults out of the box. - -It aims to make the process of writing command line tools quick and fun -while also preventing any frustration caused by the inability to -implement an intended CLI API. - -Click in three points: - -- Arbitrary nesting of commands -- Automatic help page generation -- Supports lazy loading of subcommands at runtime - - -## A Simple Example - -```python -import click - -@click.command() -@click.option("--count", default=1, help="Number of greetings.") -@click.option("--name", prompt="Your name", help="The person to greet.") -def hello(count, name): - """Simple program that greets NAME for a total of COUNT times.""" - for _ in range(count): - click.echo(f"Hello, {name}!") - -if __name__ == '__main__': - hello() -``` - -``` -$ python hello.py --count=3 -Your name: Click -Hello, Click! -Hello, Click! -Hello, Click! -``` - - -## Donate - -The Pallets organization develops and supports Click and other popular -packages. In order to grow the community of contributors and users, and -allow the maintainers to devote more time to the projects, [please -donate today][]. - -[please donate today]: https://palletsprojects.com/donate - -## Contributing - -See our [detailed contributing documentation][contrib] for many ways to -contribute, including reporting issues, requesting features, asking or answering -questions, and making PRs. - -[contrib]: https://palletsprojects.com/contributing/ - diff --git a/functions/python/data-processor/click-8.4.1.dist-info/RECORD b/functions/python/data-processor/click-8.4.1.dist-info/RECORD deleted file mode 100644 index 178f62ea9..000000000 --- a/functions/python/data-processor/click-8.4.1.dist-info/RECORD +++ /dev/null @@ -1,40 +0,0 @@ -click-8.4.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -click-8.4.1.dist-info/METADATA,sha256=uWALc7Y-Ixjj35al3B2qNsyXk-0tf578YgIsL1Vbn98,2621 -click-8.4.1.dist-info/RECORD,, -click-8.4.1.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -click-8.4.1.dist-info/licenses/LICENSE.txt,sha256=morRBqOU6FO_4h9C9OctWSgZoigF2ZG18ydQKSkrZY0,1475 -click/__init__.py,sha256=FId2fXCSJB3yeWD-e2uON-mBhFa2Yc9MvXGmHu8OXG0,4634 -click/__pycache__/__init__.cpython-314.pyc,, -click/__pycache__/_compat.cpython-314.pyc,, -click/__pycache__/_termui_impl.cpython-314.pyc,, -click/__pycache__/_textwrap.cpython-314.pyc,, -click/__pycache__/_utils.cpython-314.pyc,, -click/__pycache__/_winconsole.cpython-314.pyc,, -click/__pycache__/core.cpython-314.pyc,, -click/__pycache__/decorators.cpython-314.pyc,, -click/__pycache__/exceptions.cpython-314.pyc,, -click/__pycache__/formatting.cpython-314.pyc,, -click/__pycache__/globals.cpython-314.pyc,, -click/__pycache__/parser.cpython-314.pyc,, -click/__pycache__/shell_completion.cpython-314.pyc,, -click/__pycache__/termui.cpython-314.pyc,, -click/__pycache__/testing.cpython-314.pyc,, -click/__pycache__/types.cpython-314.pyc,, -click/__pycache__/utils.cpython-314.pyc,, -click/_compat.py,sha256=XK1woqBSPRb1KNAwWwVsSMfEm8ZlfDwBCjceWzSGmEY,18910 -click/_termui_impl.py,sha256=VNyWY27pm0JUq12WK__pJNYxeUoDyfNwYEgE6vz_cnM,30411 -click/_textwrap.py,sha256=7Z0N7Vmn-66TNSTUwp6OXJbcUXRmYET9h9c2ucD8oQQ,6270 -click/_utils.py,sha256=eCZCtwJtsYD5QYkkNWJ8MY_8ABIjy8MczgMMyVY32rQ,996 -click/_winconsole.py,sha256=KSxfNbMlYRa6GOJuCLgsg2Pb3dVkgJNPqLJPae-Pa10,8543 -click/core.py,sha256=CQaJ0ALw4zyyVp6E7gP6a_G88eiuXE9mBzFnIJQ-igY,137917 -click/decorators.py,sha256=xgR8vZAClhWVkzitP22k6jETFRhKralh81zvA69Csd0,18467 -click/exceptions.py,sha256=FJitDd6MSZLj6CtKQWpkQzHnfsUw3UypxDU0t76bn0c,11294 -click/formatting.py,sha256=r6kGLpPyF0Cva6uSTwSZ_8ONRE7jp93I1C0JVRYQEqE,10370 -click/globals.py,sha256=gM-Nh6A4M0HB_SgkaF5M4ncGGMDHc_flHXu9_oh4GEU,1923 -click/parser.py,sha256=oJ-fU_3mvxugIuNtHaCATZ56lgEmHRggjJiSqEgYrjA,19052 -click/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -click/shell_completion.py,sha256=QxeFb3RapNSP8ZwvK2BDVI2jrEKS1RbJaeuFMrHJAnU,21748 -click/termui.py,sha256=hflZ_tK7P4WUv_ef_RVVRL7n0nRPhTD0wN0pkHE7hDo,33000 -click/testing.py,sha256=0k_v5By_5ZO2McjBT7lTqQ57W8QJuW0_5Sx7d2ZLKec,25717 -click/types.py,sha256=UiVEx0yoQtrWO2wH22NgME4LfDxMNQi8j1qtJupYIpw,42783 -click/utils.py,sha256=YsTNrChEfwDXl39CTC_yWnQ-ZVkZxYjFESRCfEgClY4,20386 diff --git a/functions/python/data-processor/click-8.4.1.dist-info/WHEEL b/functions/python/data-processor/click-8.4.1.dist-info/WHEEL deleted file mode 100644 index d8b9936da..000000000 --- a/functions/python/data-processor/click-8.4.1.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/click-8.4.1.dist-info/licenses/LICENSE.txt b/functions/python/data-processor/click-8.4.1.dist-info/licenses/LICENSE.txt deleted file mode 100644 index d12a84918..000000000 --- a/functions/python/data-processor/click-8.4.1.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,28 +0,0 @@ -Copyright 2014 Pallets - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A -PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/functions/python/data-processor/click/__init__.py b/functions/python/data-processor/click/__init__.py deleted file mode 100644 index 64be7e0c3..000000000 --- a/functions/python/data-processor/click/__init__.py +++ /dev/null @@ -1,126 +0,0 @@ -""" -Click is a simple Python module inspired by the stdlib optparse to make -writing command line scripts fun. Unlike other modules, it's based -around a simple API that does not come with too much magic and is -composable. -""" - -from __future__ import annotations - -from .core import Argument as Argument -from .core import Command as Command -from .core import CommandCollection as CommandCollection -from .core import Context as Context -from .core import Group as Group -from .core import Option as Option -from .core import Parameter as Parameter -from .core import ParameterSource as ParameterSource -from .decorators import argument as argument -from .decorators import command as command -from .decorators import confirmation_option as confirmation_option -from .decorators import group as group -from .decorators import help_option as help_option -from .decorators import make_pass_decorator as make_pass_decorator -from .decorators import option as option -from .decorators import pass_context as pass_context -from .decorators import pass_obj as pass_obj -from .decorators import password_option as password_option -from .decorators import version_option as version_option -from .exceptions import Abort as Abort -from .exceptions import BadArgumentUsage as BadArgumentUsage -from .exceptions import BadOptionUsage as BadOptionUsage -from .exceptions import BadParameter as BadParameter -from .exceptions import ClickException as ClickException -from .exceptions import FileError as FileError -from .exceptions import MissingParameter as MissingParameter -from .exceptions import NoSuchCommand as NoSuchCommand -from .exceptions import NoSuchOption as NoSuchOption -from .exceptions import UsageError as UsageError -from .formatting import HelpFormatter as HelpFormatter -from .formatting import wrap_text as wrap_text -from .globals import get_current_context as get_current_context -from .termui import clear as clear -from .termui import confirm as confirm -from .termui import echo_via_pager as echo_via_pager -from .termui import edit as edit -from .termui import get_pager_file as get_pager_file -from .termui import getchar as getchar -from .termui import launch as launch -from .termui import pause as pause -from .termui import progressbar as progressbar -from .termui import prompt as prompt -from .termui import secho as secho -from .termui import style as style -from .termui import unstyle as unstyle -from .types import BOOL as BOOL -from .types import Choice as Choice -from .types import DateTime as DateTime -from .types import File as File -from .types import FLOAT as FLOAT -from .types import FloatRange as FloatRange -from .types import INT as INT -from .types import IntRange as IntRange -from .types import ParamType as ParamType -from .types import Path as Path -from .types import STRING as STRING -from .types import Tuple as Tuple -from .types import UNPROCESSED as UNPROCESSED -from .types import UUID as UUID -from .utils import echo as echo -from .utils import format_filename as format_filename -from .utils import get_app_dir as get_app_dir -from .utils import get_binary_stream as get_binary_stream -from .utils import get_text_stream as get_text_stream -from .utils import open_file as open_file - - -def __getattr__(name: str) -> object: - import warnings - - if name == "BaseCommand": - from .core import _BaseCommand - - warnings.warn( - "'BaseCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Command' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _BaseCommand - - if name == "MultiCommand": - from .core import _MultiCommand - - warnings.warn( - "'MultiCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Group' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _MultiCommand - - if name == "OptionParser": - from .parser import _OptionParser - - warnings.warn( - "'OptionParser' is deprecated and will be removed in Click 9.0. The" - " old parser is available in 'optparse'.", - DeprecationWarning, - stacklevel=2, - ) - return _OptionParser - - if name == "__version__": - import importlib.metadata - import warnings - - warnings.warn( - "The '__version__' attribute is deprecated and will be removed in" - " Click 9.1. Use feature detection or" - " 'importlib.metadata.version(\"click\")' instead.", - DeprecationWarning, - stacklevel=2, - ) - return importlib.metadata.version("click") - - raise AttributeError(name) diff --git a/functions/python/data-processor/click/_compat.py b/functions/python/data-processor/click/_compat.py deleted file mode 100644 index 36c0e5397..000000000 --- a/functions/python/data-processor/click/_compat.py +++ /dev/null @@ -1,627 +0,0 @@ -from __future__ import annotations - -import codecs -import collections.abc as cabc -import io -import os -import re -import sys -import typing as t -from types import TracebackType -from weakref import WeakKeyDictionary - -CYGWIN = sys.platform.startswith("cygwin") -WIN = sys.platform.startswith("win") -MAC = sys.platform == "darwin" -auto_wrap_for_ansi: t.Callable[[t.TextIO], t.TextIO] | None = None -_ansi_re = re.compile(r"\033\[[;?0-9]*[a-zA-Z]") - - -def _make_text_stream( - stream: t.BinaryIO, - encoding: str | None, - errors: str | None, - force_readable: bool = False, - force_writable: bool = False, -) -> t.TextIO: - if encoding is None: - encoding = get_best_encoding(stream) - if errors is None: - errors = "replace" - return _NonClosingTextIOWrapper( - stream, - encoding, - errors, - line_buffering=True, - force_readable=force_readable, - force_writable=force_writable, - ) - - -def is_ascii_encoding(encoding: str) -> bool: - """Checks if a given encoding is ascii.""" - try: - return codecs.lookup(encoding).name == "ascii" - except LookupError: - return False - - -def get_best_encoding(stream: t.IO[t.Any]) -> str: - """Returns the default stream encoding if not found.""" - rv = getattr(stream, "encoding", None) or sys.getdefaultencoding() - if is_ascii_encoding(rv): - return "utf-8" - return rv - - -class _NonClosingTextIOWrapper(io.TextIOWrapper): - def __init__( - self, - stream: t.BinaryIO, - encoding: str | None, - errors: str | None, - force_readable: bool = False, - force_writable: bool = False, - **extra: t.Any, - ) -> None: - self._stream = stream = t.cast( - t.BinaryIO, _FixupStream(stream, force_readable, force_writable) - ) - super().__init__(stream, encoding, errors, **extra) - - def __del__(self) -> None: - try: - self.detach() - except Exception: - pass - - def isatty(self) -> bool: - # https://bitbucket.org/pypy/pypy/issue/1803 - return self._stream.isatty() - - -class _FixupStream: - """The new io interface needs more from streams than streams - traditionally implement. As such, this fix-up code is necessary in - some circumstances. - - The forcing of readable and writable flags are there because some tools - put badly patched objects on sys (one such offender are certain version - of jupyter notebook). - """ - - def __init__( - self, - stream: t.BinaryIO, - force_readable: bool = False, - force_writable: bool = False, - ): - self._stream = stream - self._force_readable = force_readable - self._force_writable = force_writable - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._stream, name) - - def read1(self, size: int) -> bytes: - f = getattr(self._stream, "read1", None) - - if f is not None: - return t.cast(bytes, f(size)) - - return self._stream.read(size) - - def readable(self) -> bool: - if self._force_readable: - return True - x = getattr(self._stream, "readable", None) - if x is not None: - return t.cast(bool, x()) - try: - self._stream.read(0) - except Exception: - return False - return True - - def writable(self) -> bool: - if self._force_writable: - return True - x = getattr(self._stream, "writable", None) - if x is not None: - return t.cast(bool, x()) - try: - self._stream.write(b"") - except Exception: - try: - self._stream.write(b"") - except Exception: - return False - return True - - def seekable(self) -> bool: - x = getattr(self._stream, "seekable", None) - if x is not None: - return t.cast(bool, x()) - try: - self._stream.seek(self._stream.tell()) - except Exception: - return False - return True - - -def _is_binary_reader(stream: t.IO[t.Any], default: bool = False) -> bool: - try: - return isinstance(stream.read(0), bytes) - except Exception: - return default - # This happens in some cases where the stream was already - # closed. In this case, we assume the default. - - -def _is_binary_writer(stream: t.IO[t.Any], default: bool = False) -> bool: - try: - stream.write(b"") - except Exception: - try: - stream.write("") - return False - except Exception: - pass - return default - return True - - -def _find_binary_reader(stream: t.IO[t.Any]) -> t.BinaryIO | None: - # We need to figure out if the given stream is already binary. - # This can happen because the official docs recommend detaching - # the streams to get binary streams. Some code might do this, so - # we need to deal with this case explicitly. - if _is_binary_reader(stream, False): - return t.cast(t.BinaryIO, stream) - - buf = getattr(stream, "buffer", None) - - # Same situation here; this time we assume that the buffer is - # actually binary in case it's closed. - if buf is not None and _is_binary_reader(buf, True): - return t.cast(t.BinaryIO, buf) - - return None - - -def _find_binary_writer(stream: t.IO[t.Any]) -> t.BinaryIO | None: - # We need to figure out if the given stream is already binary. - # This can happen because the official docs recommend detaching - # the streams to get binary streams. Some code might do this, so - # we need to deal with this case explicitly. - if _is_binary_writer(stream, False): - return t.cast(t.BinaryIO, stream) - - buf = getattr(stream, "buffer", None) - - # Same situation here; this time we assume that the buffer is - # actually binary in case it's closed. - if buf is not None and _is_binary_writer(buf, True): - return t.cast(t.BinaryIO, buf) - - return None - - -def _stream_is_misconfigured(stream: t.TextIO) -> bool: - """A stream is misconfigured if its encoding is ASCII.""" - # If the stream does not have an encoding set, we assume it's set - # to ASCII. This appears to happen in certain unittest - # environments. It's not quite clear what the correct behavior is - # but this at least will force Click to recover somehow. - return is_ascii_encoding(getattr(stream, "encoding", None) or "ascii") - - -def _is_compat_stream_attr(stream: t.TextIO, attr: str, value: str | None) -> bool: - """A stream attribute is compatible if it is equal to the - desired value or the desired value is unset and the attribute - has a value. - """ - stream_value = getattr(stream, attr, None) - return stream_value == value or (value is None and stream_value is not None) - - -def _is_compatible_text_stream( - stream: t.TextIO, encoding: str | None, errors: str | None -) -> bool: - """Check if a stream's encoding and errors attributes are - compatible with the desired values. - """ - return _is_compat_stream_attr( - stream, "encoding", encoding - ) and _is_compat_stream_attr(stream, "errors", errors) - - -def _force_correct_text_stream( - text_stream: t.IO[t.Any], - encoding: str | None, - errors: str | None, - is_binary: t.Callable[[t.IO[t.Any], bool], bool], - find_binary: t.Callable[[t.IO[t.Any]], t.BinaryIO | None], - force_readable: bool = False, - force_writable: bool = False, -) -> t.TextIO: - if is_binary(text_stream, False): - binary_reader = t.cast(t.BinaryIO, text_stream) - else: - text_stream = t.cast(t.TextIO, text_stream) - # If the stream looks compatible, and won't default to a - # misconfigured ascii encoding, return it as-is. - if _is_compatible_text_stream(text_stream, encoding, errors) and not ( - encoding is None and _stream_is_misconfigured(text_stream) - ): - return text_stream - - # Otherwise, get the underlying binary reader. - possible_binary_reader = find_binary(text_stream) - - # If that's not possible, silently use the original reader - # and get mojibake instead of exceptions. - if possible_binary_reader is None: - return text_stream - - binary_reader = possible_binary_reader - - # Default errors to replace instead of strict in order to get - # something that works. - if errors is None: - errors = "replace" - - # Wrap the binary stream in a text stream with the correct - # encoding parameters. - return _make_text_stream( - binary_reader, - encoding, - errors, - force_readable=force_readable, - force_writable=force_writable, - ) - - -def _force_correct_text_reader( - text_reader: t.IO[t.Any], - encoding: str | None, - errors: str | None, - force_readable: bool = False, -) -> t.TextIO: - return _force_correct_text_stream( - text_reader, - encoding, - errors, - _is_binary_reader, - _find_binary_reader, - force_readable=force_readable, - ) - - -def _force_correct_text_writer( - text_writer: t.IO[t.Any], - encoding: str | None, - errors: str | None, - force_writable: bool = False, -) -> t.TextIO: - return _force_correct_text_stream( - text_writer, - encoding, - errors, - _is_binary_writer, - _find_binary_writer, - force_writable=force_writable, - ) - - -def get_binary_stdin() -> t.BinaryIO: - reader = _find_binary_reader(sys.stdin) - if reader is None: - raise RuntimeError("Was not able to determine binary stream for sys.stdin.") - return reader - - -def get_binary_stdout() -> t.BinaryIO: - writer = _find_binary_writer(sys.stdout) - if writer is None: - raise RuntimeError("Was not able to determine binary stream for sys.stdout.") - return writer - - -def get_binary_stderr() -> t.BinaryIO: - writer = _find_binary_writer(sys.stderr) - if writer is None: - raise RuntimeError("Was not able to determine binary stream for sys.stderr.") - return writer - - -def get_text_stdin(encoding: str | None = None, errors: str | None = None) -> t.TextIO: - rv = _get_windows_console_stream(sys.stdin, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_reader(sys.stdin, encoding, errors, force_readable=True) - - -def get_text_stdout(encoding: str | None = None, errors: str | None = None) -> t.TextIO: - rv = _get_windows_console_stream(sys.stdout, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_writer(sys.stdout, encoding, errors, force_writable=True) - - -def get_text_stderr(encoding: str | None = None, errors: str | None = None) -> t.TextIO: - rv = _get_windows_console_stream(sys.stderr, encoding, errors) - if rv is not None: - return rv - return _force_correct_text_writer(sys.stderr, encoding, errors, force_writable=True) - - -def _wrap_io_open( - file: str | os.PathLike[str] | int, - mode: str, - encoding: str | None, - errors: str | None, -) -> t.IO[t.Any]: - """Handles not passing ``encoding`` and ``errors`` in binary mode.""" - if "b" in mode: - return open(file, mode) - - return open(file, mode, encoding=encoding, errors=errors) - - -def open_stream( - filename: str | os.PathLike[str], - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - atomic: bool = False, -) -> tuple[t.IO[t.Any], bool]: - binary = "b" in mode - filename = os.fspath(filename) - - # Standard streams first. These are simple because they ignore the - # atomic flag. Use fsdecode to handle Path("-"). - if os.fsdecode(filename) == "-": - if any(m in mode for m in ["w", "a", "x"]): - if binary: - return get_binary_stdout(), False - return get_text_stdout(encoding=encoding, errors=errors), False - if binary: - return get_binary_stdin(), False - return get_text_stdin(encoding=encoding, errors=errors), False - - # Non-atomic writes directly go out through the regular open functions. - if not atomic: - return _wrap_io_open(filename, mode, encoding, errors), True - - # Some usability stuff for atomic writes - if "a" in mode: - raise ValueError( - "Appending to an existing file is not supported, because that" - " would involve an expensive `copy`-operation to a temporary" - " file. Open the file in normal `w`-mode and copy explicitly" - " if that's what you're after." - ) - if "x" in mode: - raise ValueError("Use the `overwrite`-parameter instead.") - if "w" not in mode: - raise ValueError("Atomic writes only make sense with `w`-mode.") - - # Atomic writes are more complicated. They work by opening a file - # as a proxy in the same folder and then using the fdopen - # functionality to wrap it in a Python file. Then we wrap it in an - # atomic file that moves the file over on close. - import errno - import random - - try: - perm: int | None = os.stat(filename).st_mode - except OSError: - perm = None - - flags = os.O_RDWR | os.O_CREAT | os.O_EXCL - - if binary: - flags |= getattr(os, "O_BINARY", 0) - - while True: - tmp_filename = os.path.join( - os.path.dirname(filename), - f".__atomic-write{random.randrange(1 << 32):08x}", - ) - try: - fd = os.open(tmp_filename, flags, 0o666 if perm is None else perm) - break - except OSError as e: - if e.errno == errno.EEXIST or ( - os.name == "nt" - and e.errno == errno.EACCES - and os.path.isdir(e.filename) - and os.access(e.filename, os.W_OK) - ): - continue - raise - - if perm is not None: - os.chmod(tmp_filename, perm) # in case perm includes bits in umask - - f = _wrap_io_open(fd, mode, encoding, errors) - af = _AtomicFile(f, tmp_filename, os.path.realpath(filename)) - return t.cast(t.IO[t.Any], af), True - - -class _AtomicFile: - def __init__(self, f: t.IO[t.Any], tmp_filename: str, real_filename: str) -> None: - self._f = f - self._tmp_filename = tmp_filename - self._real_filename = real_filename - self.closed = False - - @property - def name(self) -> str: - return self._real_filename - - def close(self, delete: bool = False) -> None: - if self.closed: - return - self._f.close() - os.replace(self._tmp_filename, self._real_filename) - self.closed = True - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._f, name) - - def __enter__(self) -> _AtomicFile: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.close(delete=exc_type is not None) - - def __repr__(self) -> str: - return repr(self._f) - - -def strip_ansi(value: str) -> str: - return _ansi_re.sub("", value) - - -def _is_jupyter_kernel_output(stream: t.IO[t.Any]) -> bool: - while isinstance(stream, (_FixupStream, _NonClosingTextIOWrapper)): - stream = stream._stream - - return stream.__class__.__module__.startswith("ipykernel.") - - -def should_strip_ansi( - stream: t.IO[t.Any] | None = None, color: bool | None = None -) -> bool: - if color is None: - if stream is None: - stream = sys.stdin - elif hasattr(stream, "color"): - # ._termui_impl.MaybeStripAnsi handles stripping ansi itself, - # so we don't need to strip it here - return False - return not isatty(stream) and not _is_jupyter_kernel_output(stream) - return not color - - -# On Windows, wrap the output streams with colorama to support ANSI -# color codes. -# NOTE: double check is needed so mypy does not analyze this on Linux -if sys.platform.startswith("win") and WIN: - from ._winconsole import _get_windows_console_stream - - def _get_argv_encoding() -> str: - import locale - - return locale.getpreferredencoding() - - _ansi_stream_wrappers: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() - - def auto_wrap_for_ansi(stream: t.TextIO, color: bool | None = None) -> t.TextIO: - """Support ANSI color and style codes on Windows by wrapping a - stream with colorama. - """ - try: - cached = _ansi_stream_wrappers.get(stream) - except Exception: - cached = None - - if cached is not None: - return cached - - import colorama - - strip = should_strip_ansi(stream, color) - ansi_wrapper = colorama.AnsiToWin32(stream, strip=strip) - rv = t.cast(t.TextIO, ansi_wrapper.stream) - _write = rv.write - - def _safe_write(s: str) -> int: - try: - return _write(s) - except BaseException: - ansi_wrapper.reset_all() - raise - - rv.write = _safe_write # type: ignore[method-assign] - - try: - _ansi_stream_wrappers[stream] = rv - except Exception: - pass - - return rv - -else: - - def _get_argv_encoding() -> str: - return getattr(sys.stdin, "encoding", None) or sys.getfilesystemencoding() - - def _get_windows_console_stream( - f: t.TextIO, encoding: str | None, errors: str | None - ) -> t.TextIO | None: - return None - - -def term_len(x: str) -> int: - return len(strip_ansi(x)) - - -def isatty(stream: t.IO[t.Any]) -> bool: - try: - return stream.isatty() - except Exception: - return False - - -def _make_cached_stream_func( - src_func: t.Callable[[], t.TextIO | None], - wrapper_func: t.Callable[[], t.TextIO], -) -> t.Callable[[], t.TextIO | None]: - cache: cabc.MutableMapping[t.TextIO, t.TextIO] = WeakKeyDictionary() - - def func() -> t.TextIO | None: - stream = src_func() - - if stream is None: - return None - - try: - rv = cache.get(stream) - except Exception: - rv = None - if rv is not None: - return rv - rv = wrapper_func() - try: - cache[stream] = rv - except Exception: - pass - return rv - - return func - - -_default_text_stdin = _make_cached_stream_func(lambda: sys.stdin, get_text_stdin) -_default_text_stdout = _make_cached_stream_func(lambda: sys.stdout, get_text_stdout) -_default_text_stderr = _make_cached_stream_func(lambda: sys.stderr, get_text_stderr) - - -binary_streams: cabc.Mapping[str, t.Callable[[], t.BinaryIO]] = { - "stdin": get_binary_stdin, - "stdout": get_binary_stdout, - "stderr": get_binary_stderr, -} - -text_streams: cabc.Mapping[str, t.Callable[[str | None, str | None], t.TextIO]] = { - "stdin": get_text_stdin, - "stdout": get_text_stdout, - "stderr": get_text_stderr, -} diff --git a/functions/python/data-processor/click/_termui_impl.py b/functions/python/data-processor/click/_termui_impl.py deleted file mode 100644 index 76113e9bb..000000000 --- a/functions/python/data-processor/click/_termui_impl.py +++ /dev/null @@ -1,929 +0,0 @@ -""" -This module contains implementations for the termui module. To keep the -import time of Click down, some infrequently used functionality is -placed in this module and only imported as needed. -""" - -from __future__ import annotations - -import collections.abc as cabc -import contextlib -import io -import math -import os -import shlex -import sys -import time -import typing as t -from gettext import gettext as _ -from io import StringIO -from pathlib import Path -from types import TracebackType - -from ._compat import _default_text_stdout -from ._compat import CYGWIN -from ._compat import get_best_encoding -from ._compat import isatty -from ._compat import strip_ansi -from ._compat import term_len -from ._compat import WIN -from .exceptions import ClickException -from .utils import echo - -V = t.TypeVar("V") - - -class _BufferedTextPagerStream(t.Protocol): - buffer: t.BinaryIO - - -def _has_binary_buffer( - stream: t.BinaryIO | t.TextIO, -) -> t.TypeGuard[_BufferedTextPagerStream]: - # TextIO is wider than TextIOWrapper; text-only streams such as StringIO - # are valid TextIO values but do not expose a binary buffer to wrap. - return getattr(stream, "buffer", None) is not None - - -if os.name == "nt": - BEFORE_BAR = "\r" - AFTER_BAR = "\n" -else: - BEFORE_BAR = "\r\033[?25l" - AFTER_BAR = "\033[?25h\n" - - -class ProgressBar(t.Generic[V]): - def __init__( - self, - iterable: cabc.Iterable[V] | None, - length: int | None = None, - fill_char: str = "#", - empty_char: str = " ", - bar_template: str = "%(bar)s", - info_sep: str = " ", - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - item_show_func: t.Callable[[V | None], str | None] | None = None, - label: str | None = None, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, - width: int = 30, - ) -> None: - self.fill_char = fill_char - self.empty_char = empty_char - self.bar_template = bar_template - self.info_sep = info_sep - self.hidden = hidden - self.show_eta = show_eta - self.show_percent = show_percent - self.show_pos = show_pos - self.item_show_func = item_show_func - self.label: str = label or "" - - if file is None: - file = _default_text_stdout() - - # There are no standard streams attached to write to. For example, - # pythonw on Windows. - if file is None: - file = StringIO() - - self.file = file - self.color = color - self.update_min_steps = update_min_steps - self._completed_intervals = 0 - self.width: int = width - self.autowidth: bool = width == 0 - - if length is None: - from operator import length_hint - - length = length_hint(iterable, -1) - - if length == -1: - length = None - if iterable is None: - if length is None: - raise TypeError("iterable or length is required") - iterable = t.cast("cabc.Iterable[V]", range(length)) - self.iter: cabc.Iterable[V] = iter(iterable) - self.length = length - self.pos: int = 0 - self.avg: list[float] = [] - self.last_eta: float - self.start: float - self.start = self.last_eta = time.time() - self.eta_known: bool = False - self.finished: bool = False - self.max_width: int | None = None - self.entered: bool = False - self.current_item: V | None = None - self._is_atty = isatty(self.file) - self._last_line: str | None = None - - def __enter__(self) -> ProgressBar[V]: - self.entered = True - self.render_progress() - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.render_finish() - - def __iter__(self) -> cabc.Iterator[V]: - if not self.entered: - raise RuntimeError("You need to use progress bars in a with block.") - self.render_progress() - return self.generator() - - def __next__(self) -> V: - # Iteration is defined in terms of a generator function, - # returned by iter(self); use that to define next(). This works - # because `self.iter` is an iterable consumed by that generator, - # so it is re-entry safe. Calling `next(self.generator())` - # twice works and does "what you want". - return next(iter(self)) - - def render_finish(self) -> None: - if self.hidden or not self._is_atty: - return - self.file.write(AFTER_BAR) - self.file.flush() - - @property - def pct(self) -> float: - if self.finished: - return 1.0 - return min(self.pos / (float(self.length or 1) or 1), 1.0) - - @property - def time_per_iteration(self) -> float: - if not self.avg: - return 0.0 - return sum(self.avg) / float(len(self.avg)) - - @property - def eta(self) -> float: - if self.length is not None and not self.finished: - return self.time_per_iteration * (self.length - self.pos) - return 0.0 - - def format_eta(self) -> str: - if self.eta_known: - t = int(self.eta) - seconds = t % 60 - t //= 60 - minutes = t % 60 - t //= 60 - hours = t % 24 - t //= 24 - if t > 0: - return "{d}{day_label} {h:02}:{m:02}:{s:02}".format( - d=t, - day_label=_("d"), - h=hours, - m=minutes, - s=seconds, - ) - else: - return f"{hours:02}:{minutes:02}:{seconds:02}" - return "" - - def format_pos(self) -> str: - pos = str(self.pos) - if self.length is not None: - pos += f"/{self.length}" - return pos - - def format_pct(self) -> str: - return f"{int(self.pct * 100): 4}%"[1:] - - def format_bar(self) -> str: - if self.length is not None: - bar_length = int(self.pct * self.width) - bar = self.fill_char * bar_length - bar += self.empty_char * (self.width - bar_length) - elif self.finished: - bar = self.fill_char * self.width - else: - chars = list(self.empty_char * (self.width or 1)) - if self.time_per_iteration != 0: - chars[ - int( - (math.cos(self.pos * self.time_per_iteration) / 2.0 + 0.5) - * self.width - ) - ] = self.fill_char - bar = "".join(chars) - return bar - - def format_progress_line(self) -> str: - show_percent = self.show_percent - - info_bits = [] - if self.length is not None and show_percent is None: - show_percent = not self.show_pos - - if self.show_pos: - info_bits.append(self.format_pos()) - if show_percent: - info_bits.append(self.format_pct()) - if self.show_eta and self.eta_known and not self.finished: - info_bits.append(self.format_eta()) - if self.item_show_func is not None: - item_info = self.item_show_func(self.current_item) - if item_info is not None: - info_bits.append(item_info) - - return ( - self.bar_template - % { - "label": self.label, - "bar": self.format_bar(), - "info": self.info_sep.join(info_bits), - } - ).rstrip() - - def render_progress(self) -> None: - if self.hidden: - return - - if not self._is_atty: - # Only output the label once if the output is not a TTY. - if self._last_line != self.label: - self._last_line = self.label - echo(self.label, file=self.file, color=self.color) - return - - buf = [] - # Update width in case the terminal has been resized - if self.autowidth: - import shutil - - old_width = self.width - self.width = 0 - clutter_length = term_len(self.format_progress_line()) - new_width = max(0, shutil.get_terminal_size().columns - clutter_length) - if new_width < old_width and self.max_width is not None: - buf.append(BEFORE_BAR) - buf.append(" " * self.max_width) - self.max_width = new_width - self.width = new_width - - clear_width = self.width - if self.max_width is not None: - clear_width = self.max_width - - buf.append(BEFORE_BAR) - line = self.format_progress_line() - line_len = term_len(line) - if self.max_width is None or self.max_width < line_len: - self.max_width = line_len - - buf.append(line) - buf.append(" " * (clear_width - line_len)) - line = "".join(buf) - # Render the line only if it changed. - - if line != self._last_line: - self._last_line = line - echo(line, file=self.file, color=self.color, nl=False) - self.file.flush() - - def make_step(self, n_steps: int) -> None: - self.pos += n_steps - if self.length is not None and self.pos >= self.length: - self.finished = True - - if (time.time() - self.last_eta) < 1.0: - return - - self.last_eta = time.time() - - # self.avg is a rolling list of length <= 7 of steps where steps are - # defined as time elapsed divided by the total progress through - # self.length. - if self.pos: - step = (time.time() - self.start) / self.pos - else: - step = time.time() - self.start - - self.avg = self.avg[-6:] + [step] - - self.eta_known = self.length is not None - - def update(self, n_steps: int, current_item: V | None = None) -> None: - """Update the progress bar by advancing a specified number of - steps, and optionally set the ``current_item`` for this new - position. - - :param n_steps: Number of steps to advance. - :param current_item: Optional item to set as ``current_item`` - for the updated position. - - .. versionchanged:: 8.0 - Added the ``current_item`` optional parameter. - - .. versionchanged:: 8.0 - Only render when the number of steps meets the - ``update_min_steps`` threshold. - """ - if current_item is not None: - self.current_item = current_item - - self._completed_intervals += n_steps - - if self._completed_intervals >= self.update_min_steps: - self.make_step(self._completed_intervals) - self.render_progress() - self._completed_intervals = 0 - - def finish(self) -> None: - self.eta_known = False - self.current_item = None - self.finished = True - - def generator(self) -> cabc.Iterator[V]: - """Return a generator which yields the items added to the bar - during construction, and updates the progress bar *after* the - yielded block returns. - """ - # WARNING: the iterator interface for `ProgressBar` relies on - # this and only works because this is a simple generator which - # doesn't create or manage additional state. If this function - # changes, the impact should be evaluated both against - # `iter(bar)` and `next(bar)`. `next()` in particular may call - # `self.generator()` repeatedly, and this must remain safe in - # order for that interface to work. - if not self.entered: - raise RuntimeError("You need to use progress bars in a with block.") - - if not self._is_atty: - yield from self.iter - else: - for rv in self.iter: - self.current_item = rv - - # This allows show_item_func to be updated before the - # item is processed. Only trigger at the beginning of - # the update interval. - if self._completed_intervals == 0: - self.render_progress() - - yield rv - self.update(1) - - self.finish() - self.render_progress() - - -class MaybeStripAnsi(io.TextIOWrapper): - def __init__(self, stream: t.IO[bytes], *, color: bool, **kwargs: t.Any): - super().__init__(stream, **kwargs) - self.color = color - - def write(self, text: str) -> int: - if not self.color: - text = strip_ansi(text) - return super().write(text) - - -def _pager_contextmanager( - color: bool | None = None, -) -> t.ContextManager[tuple[t.BinaryIO | t.TextIO, str, bool]]: - """Decide what method to use for paging through text.""" - stdout = _default_text_stdout() - - # There are no standard streams attached to write to. For example, - # pythonw on Windows. - if stdout is None: - stdout = StringIO() - - if not isatty(sys.stdin) or not isatty(stdout): - return _nullpager(stdout, color) - - # Split using POSIX mode (the default) so that quote characters are - # stripped from tokens and quoted Windows paths are preserved. - # Non-POSIX mode retains quotes in tokens, and wrapping tokens - # with shlex.quote re-introduces quoting issues on Windows. - pager_cmd_parts = shlex.split(os.environ.get("PAGER", "")) - if pager_cmd_parts: - if WIN: - return _tempfilepager(pager_cmd_parts, color) - return _pipepager(pager_cmd_parts, color) - - if os.environ.get("TERM") in ("dumb", "emacs"): - return _nullpager(stdout, color) - if WIN or sys.platform.startswith("os2"): - return _tempfilepager(["more"], color) - return _pipepager(["less"], color) - - -@contextlib.contextmanager -def get_pager_file(color: bool | None = None) -> t.Generator[t.TextIO, None, None]: - """Context manager. - Yields a writable file-like object which can be used as an output pager. - .. versionadded:: 8.4 - :param color: controls if the pager supports ANSI colors or not. The - default is autodetection. - """ - with _pager_contextmanager(color=color) as (stream, encoding, color): - # Split streams by capabilities rather than the abstract TextIO / - # BinaryIO annotations: buffered text streams can be unwrapped to bytes, - # while text-only streams are yielded as-is. - if _has_binary_buffer(stream): - # Text stream backed by a binary buffer. - stream = MaybeStripAnsi(stream.buffer, color=color, encoding=encoding) - elif isinstance(stream, t.BinaryIO): - # Binary stream - stream = MaybeStripAnsi(stream, color=color, encoding=encoding) - try: - yield stream - finally: - stream.flush() - - -@contextlib.contextmanager -def _pipepager( - cmd_parts: list[str], color: bool | None = None -) -> t.Iterator[tuple[t.BinaryIO | t.TextIO, str, bool]]: - """Page through text by feeding it to another program. - - Invokes the pager via :class:`subprocess.Popen` with an ``argv`` list - produced by :func:`shlex.split`. The command is resolved to an absolute - path with :func:`shutil.which` as recommended by the - :mod:`subprocess` docs for Windows compatibility. - - Invoking a pager through this might support colors: if piping to - ``less`` and the user hasn't decided on colors, ``LESS=-R`` is set - automatically. - """ - # Split the command into the invoked CLI and its parameters. - if not cmd_parts: - stdout = _default_text_stdout() or StringIO() - yield stdout, "utf-8", False - return - - import shutil - - cmd = cmd_parts[0] - cmd_params = cmd_parts[1:] - - cmd_filepath = shutil.which(cmd) - if not cmd_filepath: - stdout = _default_text_stdout() or StringIO() - yield stdout, "utf-8", False - return - - # Produces a normalized absolute path string. - # multi-call binaries such as busybox derive their identity from the symlink - # less -> busybox. resolve() causes them to misbehave. (eg. less becomes busybox) - cmd_path = Path(cmd_filepath).absolute() - cmd_name = cmd_path.name - - import subprocess - - # Make a local copy of the environment to not affect the global one. - env = dict(os.environ) - - # If we're piping to less and the user hasn't decided on colors, we enable - # them by default we find the -R flag in the command line arguments. - if color is None and cmd_name == "less": - less_flags = f"{os.environ.get('LESS', '')}{' '.join(cmd_params)}" - if not less_flags: - env["LESS"] = "-R" - color = True - elif "r" in less_flags or "R" in less_flags: - color = True - - if color is None: - color = False - - c = subprocess.Popen( - [str(cmd_path)] + cmd_params, - shell=False, - stdin=subprocess.PIPE, - env=env, - errors="replace", - text=True, - ) - stdin = t.cast(t.BinaryIO, c.stdin) - encoding = get_best_encoding(stdin) - try: - yield stdin, encoding, color - except BrokenPipeError: - # In case the pager exited unexpectedly, ignore the broken pipe error. - pass - except Exception as e: - # In case there is an exception we want to close the pager immediately - # and let the caller handle it. - # Otherwise the pager will keep running, and the user may not notice - # the error message, or worse yet it may leave the terminal in a broken state. - c.terminate() - raise e - finally: - # We must close stdin and wait for the pager to exit before we continue - try: - stdin.close() - # Close implies flush, so it might throw a BrokenPipeError if the pager - # process exited already. - except BrokenPipeError: - pass - - # Less doesn't respect ^C, but catches it for its own UI purposes (aborting - # search or other commands inside less). - # - # That means when the user hits ^C, the parent process (click) terminates, - # but less is still alive, paging the output and messing up the terminal. - # - # If the user wants to make the pager exit on ^C, they should set - # `LESS='-K'`. It's not our decision to make. - while True: - try: - c.wait() - except KeyboardInterrupt: - pass - else: - break - - -@contextlib.contextmanager -def _tempfilepager( - cmd_parts: list[str], color: bool | None = None -) -> t.Iterator[tuple[t.BinaryIO | t.TextIO, str, bool]]: - """Page through text by invoking a program on a temporary file. - - Used as the primary pager strategy on Windows (where piping to - ``more`` adds spurious ``\\r\\n``), and as a fallback on other - platforms. The command is resolved to an absolute path with - :func:`shutil.which`. - """ - # Split the command into the invoked CLI and its parameters. - if not cmd_parts: - stdout = _default_text_stdout() or StringIO() - yield stdout, "utf-8", False - return - - import shutil - import subprocess - - cmd = cmd_parts[0] - - cmd_filepath = shutil.which(cmd) - if not cmd_filepath: - stdout = _default_text_stdout() or StringIO() - yield stdout, "utf-8", False - return - - # Produces a normalized absolute path string. - # multi-call binaries such as busybox derive their identity from the symlink - # less -> busybox. resolve() causes them to misbehave. (eg. less becomes busybox) - cmd_path = Path(cmd_filepath).absolute() - - import tempfile - - encoding = get_best_encoding(sys.stdout) - if color is None: - color = False - # On Windows, NamedTemporaryFile cannot be opened by another process - # while Python still has it open, so we use delete=False and clean up manually - # rather than using a contextmanager here. - f = tempfile.NamedTemporaryFile(mode="wb", delete=False) - try: - yield t.cast(t.BinaryIO, f), encoding, color - f.flush() - f.close() - subprocess.call([str(cmd_path), f.name]) - finally: - os.unlink(f.name) - - -class _SkipClose: - def __init__(self, stream: t.IO[t.Any]) -> None: - self.stream = stream - - def __getattr__(self, name: str) -> t.Any: - return getattr(self.stream, name) - - @property - def buffer(self) -> t.BinaryIO: - return _SkipClose(self.stream.buffer) # type: ignore[attr-defined, return-value] - - def close(self) -> None: - pass - - -@contextlib.contextmanager -def _nullpager( - stream: t.TextIO, color: bool | None = None -) -> t.Iterator[tuple[t.TextIO, str, bool]]: - """Simply print unformatted text. This is the ultimate fallback. Don't close the - output stream in this case, since it's coming from elsewhere rather than our - internal helpers. - """ - encoding = get_best_encoding(stream) - - if color is None: - color = False - - yield _SkipClose(stream), encoding, color # type: ignore[misc] - - -class Editor: - def __init__( - self, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", - ) -> None: - self.editor = editor - self.env = env - self.require_save = require_save - self.extension = extension - - def get_editor(self) -> str: - if self.editor is not None: - return self.editor - for key in "VISUAL", "EDITOR": - rv = os.environ.get(key) - if rv: - return rv - if WIN: - return "notepad" - - from shutil import which - - for editor in "sensible-editor", "vim", "nano": - if which(editor) is not None: - return editor - return "vi" - - def edit_files(self, filenames: cabc.Iterable[str]) -> None: - """Open files in the user's editor.""" - import shlex - import subprocess - - editor = self.get_editor() - environ: dict[str, str] | None = None - - if self.env: - environ = os.environ.copy() - environ.update(self.env) - - try: - # Split in POSIX mode (the default) for the same reasons as - # in pager(): strips quotes from tokens and preserves quoted - # Windows paths. - c = subprocess.Popen( - args=shlex.split(editor) + list(filenames), - env=environ, - ) - exit_code = c.wait() - if exit_code != 0: - raise ClickException( - _("{editor}: Editing failed").format(editor=editor) - ) - except OSError as e: - raise ClickException( - _("{editor}: Editing failed: {e}").format(editor=editor, e=e) - ) from e - - @t.overload - def edit(self, text: bytes | bytearray) -> bytes | None: ... - - # We cannot know whether or not the type expected is str or bytes when None - # is passed, so str is returned as that was what was done before. - @t.overload - def edit(self, text: str | None) -> str | None: ... - - def edit(self, text: str | bytes | bytearray | None) -> str | bytes | None: - import tempfile - - if text is None: - data: bytes | bytearray = b"" - elif isinstance(text, (bytes, bytearray)): - data = text - else: - if text and not text.endswith("\n"): - text += "\n" - - if WIN: - data = text.replace("\n", "\r\n").encode("utf-8-sig") - else: - data = text.encode("utf-8") - - fd, name = tempfile.mkstemp(prefix="editor-", suffix=self.extension) - f: t.BinaryIO - - try: - with os.fdopen(fd, "wb") as f: - f.write(data) - - # If the filesystem resolution is 1 second, like Mac OS - # 10.12 Extended, or 2 seconds, like FAT32, and the editor - # closes very fast, require_save can fail. Set the modified - # time to be 2 seconds in the past to work around this. - os.utime(name, (os.path.getatime(name), os.path.getmtime(name) - 2)) - # Depending on the resolution, the exact value might not be - # recorded, so get the new recorded value. - timestamp = os.path.getmtime(name) - - self.edit_files((name,)) - - if self.require_save and os.path.getmtime(name) == timestamp: - return None - - with open(name, "rb") as f: - rv = f.read() - - if isinstance(text, (bytes, bytearray)): - return rv - - return rv.decode("utf-8-sig").replace("\r\n", "\n") - finally: - os.unlink(name) - - -def open_url(url: str, wait: bool = False, locate: bool = False) -> int: - import subprocess - - def _unquote_file(url: str) -> str: - from urllib.parse import unquote - - if url.startswith("file://"): - url = unquote(url[7:]) - - return url - - if sys.platform == "darwin": - args = ["open"] - if wait: - args.append("-W") - if locate: - args.append("-R") - args.append(_unquote_file(url)) - null = open("/dev/null", "w") - try: - return subprocess.Popen(args, stderr=null).wait() - finally: - null.close() - elif WIN: - if locate: - url = _unquote_file(url) - args = ["explorer", "/select,", url] - try: - return subprocess.call(args) - except OSError: - return 127 - else: - try: - os.startfile(url) # type: ignore[attr-defined] - except OSError: - return 127 - return 0 - elif CYGWIN: - if locate: - url = _unquote_file(url) - args = ["cygstart", os.path.dirname(url)] - else: - args = ["cygstart"] - if wait: - args.append("-w") - args.append(url) - try: - return subprocess.call(args) - except OSError: - # Command not found - return 127 - - try: - if locate: - url = os.path.dirname(_unquote_file(url)) or "." - else: - url = _unquote_file(url) - c = subprocess.Popen(["xdg-open", url]) - if wait: - return c.wait() - return 0 - except OSError: - if url.startswith(("http://", "https://")) and not locate and not wait: - import webbrowser - - webbrowser.open(url) - return 0 - return 1 - - -def _translate_ch_to_exc(ch: str) -> None: - if ch == "\x03": - raise KeyboardInterrupt() - - if ch == "\x04" and not WIN: # Unix-like, Ctrl+D - raise EOFError() - - if ch == "\x1a" and WIN: # Windows, Ctrl+Z - raise EOFError() - - -if sys.platform == "win32": - import msvcrt - - @contextlib.contextmanager - def raw_terminal() -> cabc.Iterator[int]: - yield -1 - - def getchar(echo: bool) -> str: - # The function `getch` will return a bytes object corresponding to - # the pressed character. Since Windows 10 build 1803, it will also - # return \x00 when called a second time after pressing a regular key. - # - # `getwch` does not share this probably-bugged behavior. Moreover, it - # returns a Unicode object by default, which is what we want. - # - # Either of these functions will return \x00 or \xe0 to indicate - # a special key, and you need to call the same function again to get - # the "rest" of the code. The fun part is that \u00e0 is - # "latin small letter a with grave", so if you type that on a French - # keyboard, you _also_ get a \xe0. - # E.g., consider the Up arrow. This returns \xe0 and then \x48. The - # resulting Unicode string reads as "a with grave" + "capital H". - # This is indistinguishable from when the user actually types - # "a with grave" and then "capital H". - # - # When \xe0 is returned, we assume it's part of a special-key sequence - # and call `getwch` again, but that means that when the user types - # the \u00e0 character, `getchar` doesn't return until a second - # character is typed. - # The alternative is returning immediately, but that would mess up - # cross-platform handling of arrow keys and others that start with - # \xe0. Another option is using `getch`, but then we can't reliably - # read non-ASCII characters, because return values of `getch` are - # limited to the current 8-bit codepage. - # - # Anyway, Click doesn't claim to do this Right(tm), and using `getwch` - # is doing the right thing in more situations than with `getch`. - - if echo: - func = t.cast(t.Callable[[], str], msvcrt.getwche) - else: - func = t.cast(t.Callable[[], str], msvcrt.getwch) - - rv = func() - - if rv in ("\x00", "\xe0"): - # \x00 and \xe0 are control characters that indicate special key, - # see above. - rv += func() - - _translate_ch_to_exc(rv) - return rv - -else: - import termios - import tty - - @contextlib.contextmanager - def raw_terminal() -> cabc.Iterator[int]: - f: t.TextIO | None - fd: int - - if not isatty(sys.stdin): - f = open("/dev/tty") - fd = f.fileno() - else: - fd = sys.stdin.fileno() - f = None - - try: - old_settings = termios.tcgetattr(fd) - - try: - tty.setraw(fd) - yield fd - finally: - termios.tcsetattr(fd, termios.TCSADRAIN, old_settings) - sys.stdout.flush() - - if f is not None: - f.close() - except termios.error: - pass - - def getchar(echo: bool) -> str: - with raw_terminal() as fd: - ch = os.read(fd, 32).decode(get_best_encoding(sys.stdin), "replace") - - if echo and isatty(sys.stdout): - sys.stdout.write(ch) - - _translate_ch_to_exc(ch) - return ch diff --git a/functions/python/data-processor/click/_textwrap.py b/functions/python/data-processor/click/_textwrap.py deleted file mode 100644 index 82840f2df..000000000 --- a/functions/python/data-processor/click/_textwrap.py +++ /dev/null @@ -1,188 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import textwrap -from contextlib import contextmanager - -from ._compat import _ansi_re -from ._compat import term_len - - -def _truncate_visible(text: str, n: int) -> str: - """Return the longest prefix of ``text`` containing at most ``n`` visible - characters. - - ANSI escape sequences inside the prefix are kept intact and do not count - toward the visible width. A cut is never placed inside an escape sequence. - """ - if n <= 0: - return "" - - visible = 0 - i = 0 - cut = 0 - end = len(text) - while i < end: - m = _ansi_re.match(text, i) - if m is not None: - i = m.end() - continue - visible += 1 - i += 1 - cut = i - if visible >= n: - break - return text[:cut] - - -class TextWrapper(textwrap.TextWrapper): - """``textwrap.TextWrapper`` variant that measures widths by visible - character count. - - ANSI escape sequences embedded in chunks, indents, or the placeholder are - excluded from the width budget. Without this, styled help text (a styled - ``Usage:`` prefix, a colorized option name, ...) would be wrapped earlier - than its visible length warrants and tokens would split mid-word. - """ - - def _handle_long_word( - self, - reversed_chunks: list[str], - cur_line: list[str], - cur_len: int, - width: int, - ) -> None: - space_left = max(width - cur_len, 1) - - if self.break_long_words: - last = reversed_chunks[-1] - cut = _truncate_visible(last, space_left) - res = last[len(cut) :] - cur_line.append(cut) - reversed_chunks[-1] = res - elif not cur_line: - cur_line.append(reversed_chunks.pop()) - - def _wrap_chunks(self, chunks: list[str]) -> list[str]: - """Wrap chunks counting widths in visible characters. - - Mirrors the algorithm of :meth:`textwrap.TextWrapper._wrap_chunks` - with every width measurement routed through - :func:`click._compat.term_len` instead of :func:`len`, so ANSI escape - bytes in chunks, indents, or the placeholder do not inflate the count. - - .. seealso:: - :class:`textwrap.TextWrapper` in the Python standard library documentation: - https://docs.python.org/3/library/textwrap.html#textwrap.TextWrapper - - Reference implementation in CPython: - https://github.com/python/cpython/blob/main/Lib/textwrap.py - """ - lines: list[str] = [] - if self.width <= 0: - raise ValueError(f"invalid width {self.width!r} (must be > 0)") - if self.max_lines is not None: - if self.max_lines > 1: - indent = self.subsequent_indent - else: - indent = self.initial_indent - if term_len(indent) + term_len(self.placeholder.lstrip()) > self.width: - raise ValueError("placeholder too large for max width") - - chunks.reverse() - - while chunks: - cur_line: list[str] = [] - cur_len = 0 - - if lines: - indent = self.subsequent_indent - else: - indent = self.initial_indent - - width = self.width - term_len(indent) - - if self.drop_whitespace and chunks[-1].strip() == "" and lines: - del chunks[-1] - - while chunks: - n = term_len(chunks[-1]) - - if cur_len + n <= width: - cur_line.append(chunks.pop()) - cur_len += n - - else: - break - - if chunks and term_len(chunks[-1]) > width: - self._handle_long_word(chunks, cur_line, cur_len, width) - cur_len = sum(map(term_len, cur_line)) - - if self.drop_whitespace and cur_line and cur_line[-1].strip() == "": - cur_len -= term_len(cur_line[-1]) - del cur_line[-1] - - if cur_line: - if ( - self.max_lines is None - or len(lines) + 1 < self.max_lines - or ( - not chunks - or self.drop_whitespace - and len(chunks) == 1 - and not chunks[0].strip() - ) - and cur_len <= width - ): - lines.append(indent + "".join(cur_line)) - else: - while cur_line: - if ( - cur_line[-1].strip() - and cur_len + term_len(self.placeholder) <= width - ): - cur_line.append(self.placeholder) - lines.append(indent + "".join(cur_line)) - break - cur_len -= term_len(cur_line[-1]) - del cur_line[-1] - else: - if lines: - prev_line = lines[-1].rstrip() - if ( - term_len(prev_line) + term_len(self.placeholder) - <= self.width - ): - lines[-1] = prev_line + self.placeholder - break - lines.append(indent + self.placeholder.lstrip()) - break - - return lines - - @contextmanager - def extra_indent(self, indent: str) -> cabc.Iterator[None]: - old_initial_indent = self.initial_indent - old_subsequent_indent = self.subsequent_indent - self.initial_indent += indent - self.subsequent_indent += indent - - try: - yield - finally: - self.initial_indent = old_initial_indent - self.subsequent_indent = old_subsequent_indent - - def indent_only(self, text: str) -> str: - rv = [] - - for idx, line in enumerate(text.splitlines()): - indent = self.initial_indent - - if idx > 0: - indent = self.subsequent_indent - - rv.append(f"{indent}{line}") - - return "\n".join(rv) diff --git a/functions/python/data-processor/click/_utils.py b/functions/python/data-processor/click/_utils.py deleted file mode 100644 index 05ee2e997..000000000 --- a/functions/python/data-processor/click/_utils.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -import enum -import typing as t - - -class Sentinel(enum.Enum): - """Enum used to define sentinel values. - - .. seealso:: - - `PEP 661 - Sentinel Values `_. - """ - - UNSET = object() - FLAG_NEEDS_VALUE = object() - - def __repr__(self) -> str: - return f"{self.__class__.__name__}.{self.name}" - - -UNSET: t.Literal[Sentinel.UNSET] = Sentinel.UNSET -"""Sentinel used to indicate that a value is not set.""" - -FLAG_NEEDS_VALUE: t.Literal[Sentinel.FLAG_NEEDS_VALUE] = Sentinel.FLAG_NEEDS_VALUE -"""Sentinel used to indicate an option was passed as a flag without a -value but is not a flag option. - -``Option.consume_value`` uses this to prompt or use the ``flag_value``. -""" - -T_UNSET: t.TypeAlias = t.Literal[Sentinel.UNSET] -"""Type hint for the :data:`UNSET` sentinel value.""" - -T_FLAG_NEEDS_VALUE: t.TypeAlias = t.Literal[Sentinel.FLAG_NEEDS_VALUE] -"""Type hint for the :data:`FLAG_NEEDS_VALUE` sentinel value.""" diff --git a/functions/python/data-processor/click/_winconsole.py b/functions/python/data-processor/click/_winconsole.py deleted file mode 100644 index d25178d66..000000000 --- a/functions/python/data-processor/click/_winconsole.py +++ /dev/null @@ -1,297 +0,0 @@ -# This module is based on the excellent work by Adam Bartoš who -# provided a lot of what went into the implementation here in -# the discussion to issue1602 in the Python bug tracker. -# -# There are some general differences in regards to how this works -# compared to the original patches as we do not need to patch -# the entire interpreter but just work in our little world of -# echo and prompt. -from __future__ import annotations - -import collections.abc as cabc -import io -import sys -import time -import typing as t -from ctypes import Array -from ctypes import byref -from ctypes import c_char -from ctypes import c_char_p -from ctypes import c_int -from ctypes import c_ssize_t -from ctypes import c_ulong -from ctypes import c_void_p -from ctypes import POINTER -from ctypes import py_object -from ctypes import Structure -from ctypes.wintypes import DWORD -from ctypes.wintypes import HANDLE -from ctypes.wintypes import LPCWSTR -from ctypes.wintypes import LPWSTR -from gettext import gettext as _ - -from ._compat import _NonClosingTextIOWrapper - -assert sys.platform == "win32" -import msvcrt # noqa: E402 -from ctypes import windll # noqa: E402 -from ctypes import WINFUNCTYPE # noqa: E402 - -c_ssize_p = POINTER(c_ssize_t) - -kernel32 = windll.kernel32 -GetStdHandle = kernel32.GetStdHandle -ReadConsoleW = kernel32.ReadConsoleW -WriteConsoleW = kernel32.WriteConsoleW -GetConsoleMode = kernel32.GetConsoleMode -GetLastError = kernel32.GetLastError -GetCommandLineW = WINFUNCTYPE(LPWSTR)(("GetCommandLineW", windll.kernel32)) -CommandLineToArgvW = WINFUNCTYPE(POINTER(LPWSTR), LPCWSTR, POINTER(c_int))( - ("CommandLineToArgvW", windll.shell32) -) -LocalFree = WINFUNCTYPE(c_void_p, c_void_p)(("LocalFree", windll.kernel32)) - -STDIN_HANDLE = GetStdHandle(-10) -STDOUT_HANDLE = GetStdHandle(-11) -STDERR_HANDLE = GetStdHandle(-12) - -PyBUF_SIMPLE = 0 -PyBUF_WRITABLE = 1 - -ERROR_SUCCESS = 0 -ERROR_NOT_ENOUGH_MEMORY = 8 -ERROR_OPERATION_ABORTED = 995 - -STDIN_FILENO = 0 -STDOUT_FILENO = 1 -STDERR_FILENO = 2 - -EOF = b"\x1a" -MAX_BYTES_WRITTEN = 32767 - -if t.TYPE_CHECKING: - try: - # Using `typing_extensions.Buffer` instead of `collections.abc` - # on Windows for some reason does not have `Sized` implemented. - from collections.abc import Buffer # type: ignore - except ImportError: - from typing_extensions import Buffer - -try: - from ctypes import pythonapi -except ImportError: - # On PyPy we cannot get buffers so our ability to operate here is - # severely limited. - get_buffer = None -else: - - class Py_buffer(Structure): - _fields_ = [ # noqa: RUF012 - ("buf", c_void_p), - ("obj", py_object), - ("len", c_ssize_t), - ("itemsize", c_ssize_t), - ("readonly", c_int), - ("ndim", c_int), - ("format", c_char_p), - ("shape", c_ssize_p), - ("strides", c_ssize_p), - ("suboffsets", c_ssize_p), - ("internal", c_void_p), - ] - - PyObject_GetBuffer = pythonapi.PyObject_GetBuffer - PyBuffer_Release = pythonapi.PyBuffer_Release - - def get_buffer(obj: Buffer, writable: bool = False) -> Array[c_char]: - buf = Py_buffer() - flags: int = PyBUF_WRITABLE if writable else PyBUF_SIMPLE - PyObject_GetBuffer(py_object(obj), byref(buf), flags) - - try: - buffer_type = c_char * buf.len - out: Array[c_char] = buffer_type.from_address(buf.buf) - return out - finally: - PyBuffer_Release(byref(buf)) - - -class _WindowsConsoleRawIOBase(io.RawIOBase): - def __init__(self, handle: int | None) -> None: - self.handle = handle - - def isatty(self) -> t.Literal[True]: - super().isatty() - return True - - -class _WindowsConsoleReader(_WindowsConsoleRawIOBase): - def readable(self) -> t.Literal[True]: - return True - - def readinto(self, b: Buffer) -> int: - bytes_to_be_read = len(b) - if not bytes_to_be_read: - return 0 - elif bytes_to_be_read % 2: - raise ValueError( - "cannot read odd number of bytes from UTF-16-LE encoded console" - ) - - buffer = get_buffer(b, writable=True) - code_units_to_be_read = bytes_to_be_read // 2 - code_units_read = c_ulong() - - rv = ReadConsoleW( - HANDLE(self.handle), - buffer, - code_units_to_be_read, - byref(code_units_read), - None, - ) - if GetLastError() == ERROR_OPERATION_ABORTED: - # wait for KeyboardInterrupt - time.sleep(0.1) - if not rv: - raise OSError(_("Windows error: {error}").format(error=GetLastError())) - - if buffer[0] == EOF: - return 0 - return 2 * code_units_read.value - - -class _WindowsConsoleWriter(_WindowsConsoleRawIOBase): - def writable(self) -> t.Literal[True]: - return True - - @staticmethod - def _get_error_message(errno: int) -> str: - if errno == ERROR_SUCCESS: - return "ERROR_SUCCESS" - elif errno == ERROR_NOT_ENOUGH_MEMORY: - return "ERROR_NOT_ENOUGH_MEMORY" - return _("Windows error: {error}").format(error=errno) - - def write(self, b: Buffer) -> int: - bytes_to_be_written = len(b) - buf = get_buffer(b) - code_units_to_be_written = min(bytes_to_be_written, MAX_BYTES_WRITTEN) // 2 - code_units_written = c_ulong() - - WriteConsoleW( - HANDLE(self.handle), - buf, - code_units_to_be_written, - byref(code_units_written), - None, - ) - bytes_written = 2 * code_units_written.value - - if bytes_written == 0 and bytes_to_be_written > 0: - raise OSError(self._get_error_message(GetLastError())) - return bytes_written - - -class ConsoleStream: - def __init__(self, text_stream: t.TextIO, byte_stream: t.BinaryIO) -> None: - self._text_stream = text_stream - self.buffer = byte_stream - - @property - def name(self) -> str: - return self.buffer.name - - def write(self, x: t.AnyStr) -> int: - if isinstance(x, str): - return self._text_stream.write(x) - try: - self.flush() - except Exception: - pass - return self.buffer.write(x) - - def writelines(self, lines: cabc.Iterable[t.AnyStr]) -> None: - for line in lines: - self.write(line) - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._text_stream, name) - - def isatty(self) -> bool: - return self.buffer.isatty() - - def __repr__(self) -> str: - return f"" - - -def _get_text_stdin(buffer_stream: t.BinaryIO) -> t.TextIO: - text_stream = _NonClosingTextIOWrapper( - io.BufferedReader(_WindowsConsoleReader(STDIN_HANDLE)), - "utf-16-le", - "strict", - line_buffering=True, - ) - return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) - - -def _get_text_stdout(buffer_stream: t.BinaryIO) -> t.TextIO: - text_stream = _NonClosingTextIOWrapper( - io.BufferedWriter(_WindowsConsoleWriter(STDOUT_HANDLE)), - "utf-16-le", - "strict", - line_buffering=True, - ) - return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) - - -def _get_text_stderr(buffer_stream: t.BinaryIO) -> t.TextIO: - text_stream = _NonClosingTextIOWrapper( - io.BufferedWriter(_WindowsConsoleWriter(STDERR_HANDLE)), - "utf-16-le", - "strict", - line_buffering=True, - ) - return t.cast(t.TextIO, ConsoleStream(text_stream, buffer_stream)) - - -_stream_factories: cabc.Mapping[int, t.Callable[[t.BinaryIO], t.TextIO]] = { - 0: _get_text_stdin, - 1: _get_text_stdout, - 2: _get_text_stderr, -} - - -def _is_console(f: t.TextIO) -> bool: - if not hasattr(f, "fileno"): - return False - - try: - fileno = f.fileno() - except (OSError, io.UnsupportedOperation): - return False - - handle = msvcrt.get_osfhandle(fileno) - return bool(GetConsoleMode(handle, byref(DWORD()))) - - -def _get_windows_console_stream( - f: t.TextIO, encoding: str | None, errors: str | None -) -> t.TextIO | None: - if ( - get_buffer is None - or encoding not in {"utf-16-le", None} - or errors not in {"strict", None} - or not _is_console(f) - ): - return None - - func = _stream_factories.get(f.fileno()) - if func is None: - return None - - b = getattr(f, "buffer", None) - - if b is None: - return None - - return func(b) diff --git a/functions/python/data-processor/click/core.py b/functions/python/data-processor/click/core.py deleted file mode 100644 index bc04f6426..000000000 --- a/functions/python/data-processor/click/core.py +++ /dev/null @@ -1,3542 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import enum -import errno -import inspect -import os -import sys -import typing as t -from abc import ABC -from abc import abstractmethod -from collections import abc -from collections import Counter -from contextlib import AbstractContextManager -from contextlib import contextmanager -from contextlib import ExitStack -from functools import update_wrapper -from gettext import gettext as _ -from gettext import ngettext -from itertools import repeat -from types import TracebackType - -from . import types -from ._utils import FLAG_NEEDS_VALUE -from ._utils import UNSET -from .exceptions import Abort -from .exceptions import BadParameter -from .exceptions import ClickException -from .exceptions import Exit -from .exceptions import MissingParameter -from .exceptions import NoArgsIsHelpError -from .exceptions import NoSuchCommand -from .exceptions import UsageError -from .formatting import HelpFormatter -from .formatting import join_options -from .globals import pop_context -from .globals import push_context -from .parser import _OptionParser -from .parser import _split_opt -from .termui import confirm -from .termui import prompt -from .termui import style -from .utils import _detect_program_name -from .utils import _expand_args -from .utils import echo -from .utils import make_default_short_help -from .utils import make_str -from .utils import PacifyFlushWrapper - -if t.TYPE_CHECKING: - from .shell_completion import CompletionItem - -F = t.TypeVar("F", bound="t.Callable[..., t.Any]") -V = t.TypeVar("V") - - -def _complete_visible_commands( - ctx: Context, incomplete: str -) -> cabc.Iterator[tuple[str, Command]]: - """List all the subcommands of a group that start with the - incomplete value and aren't hidden. - - :param ctx: Invocation context for the group. - :param incomplete: Value being completed. May be empty. - """ - multi = t.cast(Group, ctx.command) - - for name in multi.list_commands(ctx): - if name.startswith(incomplete): - command = multi.get_command(ctx, name) - - if command is not None and not command.hidden: - yield name, command - - -def _check_nested_chain( - base_command: Group, cmd_name: str, cmd: Command, register: bool = False -) -> None: - if not base_command.chain or not isinstance(cmd, Group): - return - - if register: - message = ( - f"It is not possible to add the group {cmd_name!r} to another" - f" group {base_command.name!r} that is in chain mode." - ) - else: - message = ( - f"Found the group {cmd_name!r} as subcommand to another group " - f" {base_command.name!r} that is in chain mode. This is not supported." - ) - - raise RuntimeError(message) - - -def _format_deprecated_label(deprecated: bool | str) -> str: - """Return the parenthesized deprecation label shown in help text.""" - label = _("deprecated").upper() - if isinstance(deprecated, str): - return f"({label}: {deprecated})" - return f"({label})" - - -def _format_deprecated_suffix(deprecated: bool | str) -> str: - """Return the trailing reason for a ``DeprecationWarning`` message, - prefixed with a space, or an empty string when no reason was given. - """ - if isinstance(deprecated, str): - return f" {deprecated}" - return "" - - -def batch(iterable: cabc.Iterable[V], batch_size: int) -> list[tuple[V, ...]]: - return list(zip(*repeat(iter(iterable), batch_size), strict=False)) - - -@contextmanager -def augment_usage_errors( - ctx: Context, param: Parameter | None = None -) -> cabc.Iterator[None]: - """Context manager that attaches extra information to exceptions.""" - try: - yield - except BadParameter as e: - if e.ctx is None: - e.ctx = ctx - if param is not None and e.param is None: - e.param = param - raise - except UsageError as e: - if e.ctx is None: - e.ctx = ctx - raise - - -def iter_params_for_processing( - invocation_order: cabc.Sequence[Parameter], - declaration_order: cabc.Sequence[Parameter], -) -> list[Parameter]: - """Returns all declared parameters in the order they should be processed. - - The declared parameters are re-shuffled depending on the order in which - they were invoked, as well as the eagerness of each parameters. - - The invocation order takes precedence over the declaration order. I.e. the - order in which the user provided them to the CLI is respected. - - This behavior and its effect on callback evaluation is detailed at: - https://click.palletsprojects.com/en/stable/advanced/#callback-evaluation-order - """ - - def sort_key(item: Parameter) -> tuple[bool, float]: - try: - idx: float = invocation_order.index(item) - except ValueError: - idx = float("inf") - - return not item.is_eager, idx - - return sorted(declaration_order, key=sort_key) - - -class ParameterSource(enum.IntEnum): - """This is an :class:`~enum.IntEnum` that indicates the source of a - parameter's value. - - Use :meth:`click.Context.get_parameter_source` to get the - source for a parameter by name. - - Members are ordered from most explicit to least explicit source. - This allows comparison to check if a value was explicitly provided: - - .. code-block:: python - - source = ctx.get_parameter_source("port") - if source < click.ParameterSource.DEFAULT_MAP: - ... # value was explicitly set - - .. versionchanged:: 8.3.3 - Use :class:`~enum.IntEnum` and reorder members from most to - least explicit. Supports comparison operators. - - .. versionchanged:: 8.0 - Use :class:`~enum.Enum` and drop the ``validate`` method. - - .. versionchanged:: 8.0 - Added the ``PROMPT`` value. - """ - - PROMPT = enum.auto() - """Used a prompt to confirm a default or provide a value.""" - COMMANDLINE = enum.auto() - """The value was provided by the command line args.""" - ENVIRONMENT = enum.auto() - """The value was provided with an environment variable.""" - DEFAULT_MAP = enum.auto() - """Used a default provided by :attr:`Context.default_map`.""" - DEFAULT = enum.auto() - """Used the default specified by the parameter.""" - - -class Context: - """The context is a special internal object that holds state relevant - for the script execution at every single level. It's normally invisible - to commands unless they opt-in to getting access to it. - - The context is useful as it can pass internal objects around and can - control special execution features such as reading data from - environment variables. - - A context can be used as context manager in which case it will call - :meth:`close` on teardown. - - :param command: the command class for this context. - :param parent: the parent context. - :param info_name: the info name for this invocation. Generally this - is the most descriptive name for the script or - command. For the toplevel script it is usually - the name of the script, for commands below that it's - the name of the script. - :param obj: an arbitrary object of user data. - :param auto_envvar_prefix: the prefix to use for automatic environment - variables. If this is `None` then reading - from environment variables is disabled. This - does not affect manually set environment - variables which are always read. - :param default_map: a dictionary (like object) with default values - for parameters. - :param terminal_width: the width of the terminal. The default is - inherit from parent context. If no context - defines the terminal width then auto - detection will be applied. - :param max_content_width: the maximum width for content rendered by - Click (this currently only affects help - pages). This defaults to 80 characters if - not overridden. In other words: even if the - terminal is larger than that, Click will not - format things wider than 80 characters by - default. In addition to that, formatters might - add some safety mapping on the right. - :param resilient_parsing: if this flag is enabled then Click will - parse without any interactivity or callback - invocation. Default values will also be - ignored. This is useful for implementing - things such as completion support. - :param allow_extra_args: if this is set to `True` then extra arguments - at the end will not raise an error and will be - kept on the context. The default is to inherit - from the command. - :param allow_interspersed_args: if this is set to `False` then options - and arguments cannot be mixed. The - default is to inherit from the command. - :param ignore_unknown_options: instructs click to ignore options it does - not know and keeps them for later - processing. - :param help_option_names: optionally a list of strings that define how - the default help parameter is named. The - default is ``['--help']``. - :param token_normalize_func: an optional function that is used to - normalize tokens (options, choices, - etc.). This for instance can be used to - implement case insensitive behavior. - :param color: controls if the terminal supports ANSI colors or not. The - default is autodetection. This is only needed if ANSI - codes are used in texts that Click prints which is by - default not the case. This for instance would affect - help output. - :param show_default: Show the default value for commands. If this - value is not set, it defaults to the value from the parent - context. ``Command.show_default`` overrides this default for the - specific command. - - .. versionchanged:: 8.2 - The ``protected_args`` attribute is deprecated and will be removed in - Click 9.0. ``args`` will contain remaining unparsed tokens. - - .. versionchanged:: 8.1 - The ``show_default`` parameter is overridden by - ``Command.show_default``, instead of the other way around. - - .. versionchanged:: 8.0 - The ``show_default`` parameter defaults to the value from the - parent context. - - .. versionchanged:: 7.1 - Added the ``show_default`` parameter. - - .. versionchanged:: 4.0 - Added the ``color``, ``ignore_unknown_options``, and - ``max_content_width`` parameters. - - .. versionchanged:: 3.0 - Added the ``allow_extra_args`` and ``allow_interspersed_args`` - parameters. - - .. versionchanged:: 2.0 - Added the ``resilient_parsing``, ``help_option_names``, and - ``token_normalize_func`` parameters. - """ - - #: The formatter class to create with :meth:`make_formatter`. - #: - #: .. versionadded:: 8.0 - formatter_class: type[HelpFormatter] = HelpFormatter - - def __init__( - self, - command: Command, - parent: Context | None = None, - info_name: str | None = None, - obj: t.Any | None = None, - auto_envvar_prefix: str | None = None, - default_map: cabc.MutableMapping[str, t.Any] | None = None, - terminal_width: int | None = None, - max_content_width: int | None = None, - resilient_parsing: bool = False, - allow_extra_args: bool | None = None, - allow_interspersed_args: bool | None = None, - ignore_unknown_options: bool | None = None, - help_option_names: list[str] | None = None, - token_normalize_func: t.Callable[[str], str] | None = None, - color: bool | None = None, - show_default: bool | None = None, - ) -> None: - #: the parent context or `None` if none exists. - self.parent = parent - #: the :class:`Command` for this context. - self.command = command - #: the descriptive information name - self.info_name = info_name - #: Map of parameter names to their parsed values. Parameters - #: with ``expose_value=False`` are not stored. - self.params: dict[str, t.Any] = {} - #: the leftover arguments. - self.args: list[str] = [] - #: protected arguments. These are arguments that are prepended - #: to `args` when certain parsing scenarios are encountered but - #: must be never propagated to another arguments. This is used - #: to implement nested parsing. - self._protected_args: list[str] = [] - #: the collected prefixes of the command's options. - self._opt_prefixes: set[str] = set(parent._opt_prefixes) if parent else set() - - if obj is None and parent is not None: - obj = parent.obj - - #: the user object stored. - self.obj: t.Any = obj - self._meta: dict[str, t.Any] = getattr(parent, "meta", {}) - - #: A dictionary (-like object) with defaults for parameters. - if ( - default_map is None - and info_name is not None - and parent is not None - and parent.default_map is not None - ): - default_map = parent.default_map.get(info_name) - - self.default_map: cabc.MutableMapping[str, t.Any] | None = default_map - - #: This flag indicates if a subcommand is going to be executed. A - #: group callback can use this information to figure out if it's - #: being executed directly or because the execution flow passes - #: onwards to a subcommand. By default it's None, but it can be - #: the name of the subcommand to execute. - #: - #: If chaining is enabled this will be set to ``'*'`` in case - #: any commands are executed. It is however not possible to - #: figure out which ones. If you require this knowledge you - #: should use a :func:`result_callback`. - self.invoked_subcommand: str | None = None - - if terminal_width is None and parent is not None: - terminal_width = parent.terminal_width - - #: The width of the terminal (None is autodetection). - self.terminal_width: int | None = terminal_width - - if max_content_width is None and parent is not None: - max_content_width = parent.max_content_width - - #: The maximum width of formatted content (None implies a sensible - #: default which is 80 for most things). - self.max_content_width: int | None = max_content_width - - if allow_extra_args is None: - allow_extra_args = command.allow_extra_args - - #: Indicates if the context allows extra args or if it should - #: fail on parsing. - #: - #: .. versionadded:: 3.0 - self.allow_extra_args = allow_extra_args - - if allow_interspersed_args is None: - allow_interspersed_args = command.allow_interspersed_args - - #: Indicates if the context allows mixing of arguments and - #: options or not. - #: - #: .. versionadded:: 3.0 - self.allow_interspersed_args: bool = allow_interspersed_args - - if ignore_unknown_options is None: - ignore_unknown_options = command.ignore_unknown_options - - #: Instructs click to ignore options that a command does not - #: understand and will store it on the context for later - #: processing. This is primarily useful for situations where you - #: want to call into external programs. Generally this pattern is - #: strongly discouraged because it's not possibly to losslessly - #: forward all arguments. - #: - #: .. versionadded:: 4.0 - self.ignore_unknown_options: bool = ignore_unknown_options - - if help_option_names is None: - if parent is not None: - help_option_names = parent.help_option_names - else: - help_option_names = ["--help"] - - #: The names for the help options. - self.help_option_names: list[str] = help_option_names - - if token_normalize_func is None and parent is not None: - token_normalize_func = parent.token_normalize_func - - #: An optional normalization function for tokens. This is - #: options, choices, commands etc. - self.token_normalize_func: t.Callable[[str], str] | None = token_normalize_func - - #: Indicates if resilient parsing is enabled. In that case Click - #: will do its best to not cause any failures and default values - #: will be ignored. Useful for completion. - self.resilient_parsing: bool = resilient_parsing - - # If there is no envvar prefix yet, but the parent has one and - # the command on this level has a name, we can expand the envvar - # prefix automatically. - if auto_envvar_prefix is None: - if ( - parent is not None - and parent.auto_envvar_prefix is not None - and self.info_name is not None - ): - auto_envvar_prefix = ( - f"{parent.auto_envvar_prefix}_{self.info_name.upper()}" - ) - else: - auto_envvar_prefix = auto_envvar_prefix.upper() - - if auto_envvar_prefix is not None: - auto_envvar_prefix = auto_envvar_prefix.replace("-", "_") - - self.auto_envvar_prefix: str | None = auto_envvar_prefix - - if color is None and parent is not None: - color = parent.color - - #: Controls if styling output is wanted or not. - self.color: bool | None = color - - if show_default is None and parent is not None: - show_default = parent.show_default - - #: Show option default values when formatting help text. - self.show_default: bool | None = show_default - - self._close_callbacks: list[t.Callable[[], t.Any]] = [] - self._depth = 0 - self._parameter_source: dict[str, ParameterSource] = {} - # Tracks whether the option that currently owns each parameter slot in - # :attr:`params` had its ``default`` set explicitly by the user. Used - # to tie-break feature-switch groups where multiple options share a - # parameter name and both fall back to their default value. - # Refs: https://github.com/pallets/click/issues/3403 - self._param_default_explicit: dict[str, bool] = {} - self._exit_stack = ExitStack() - - @property - def protected_args(self) -> list[str]: - import warnings - - warnings.warn( - "'protected_args' is deprecated and will be removed in Click 9.0." - " 'args' will contain remaining unparsed tokens.", - DeprecationWarning, - stacklevel=2, - ) - return self._protected_args - - def to_info_dict(self) -> dict[str, t.Any]: - """Gather information that could be useful for a tool generating - user-facing documentation. This traverses the entire CLI - structure. - - .. code-block:: python - - with Context(cli) as ctx: - info = ctx.to_info_dict() - - .. versionadded:: 8.0 - """ - return { - "command": self.command.to_info_dict(self), - "info_name": self.info_name, - "allow_extra_args": self.allow_extra_args, - "allow_interspersed_args": self.allow_interspersed_args, - "ignore_unknown_options": self.ignore_unknown_options, - "auto_envvar_prefix": self.auto_envvar_prefix, - } - - def __enter__(self) -> Context: - self._depth += 1 - push_context(self) - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> bool | None: - self._depth -= 1 - exit_result: bool | None = None - if self._depth == 0: - exit_result = self._close_with_exception_info(exc_type, exc_value, tb) - pop_context() - - return exit_result - - @contextmanager - def scope(self, cleanup: bool = True) -> cabc.Iterator[Context]: - """This helper method can be used with the context object to promote - it to the current thread local (see :func:`get_current_context`). - The default behavior of this is to invoke the cleanup functions which - can be disabled by setting `cleanup` to `False`. The cleanup - functions are typically used for things such as closing file handles. - - If the cleanup is intended the context object can also be directly - used as a context manager. - - Example usage:: - - with ctx.scope(): - assert get_current_context() is ctx - - This is equivalent:: - - with ctx: - assert get_current_context() is ctx - - .. versionadded:: 5.0 - - :param cleanup: controls if the cleanup functions should be run or - not. The default is to run these functions. In - some situations the context only wants to be - temporarily pushed in which case this can be disabled. - Nested pushes automatically defer the cleanup. - """ - if not cleanup: - self._depth += 1 - try: - with self as rv: - yield rv - finally: - if not cleanup: - self._depth -= 1 - - @property - def meta(self) -> dict[str, t.Any]: - """This is a dictionary which is shared with all the contexts - that are nested. It exists so that click utilities can store some - state here if they need to. It is however the responsibility of - that code to manage this dictionary well. - - The keys are supposed to be unique dotted strings. For instance - module paths are a good choice for it. What is stored in there is - irrelevant for the operation of click. However what is important is - that code that places data here adheres to the general semantics of - the system. - - Example usage:: - - LANG_KEY = f'{__name__}.lang' - - def set_language(value): - ctx = get_current_context() - ctx.meta[LANG_KEY] = value - - def get_language(): - return get_current_context().meta.get(LANG_KEY, 'en_US') - - .. versionadded:: 5.0 - """ - return self._meta - - def make_formatter(self) -> HelpFormatter: - """Creates the :class:`~click.HelpFormatter` for the help and - usage output. - - To quickly customize the formatter class used without overriding - this method, set the :attr:`formatter_class` attribute. - - .. versionchanged:: 8.0 - Added the :attr:`formatter_class` attribute. - """ - return self.formatter_class( - width=self.terminal_width, max_width=self.max_content_width - ) - - def with_resource(self, context_manager: AbstractContextManager[V]) -> V: - """Register a resource as if it were used in a ``with`` - statement. The resource will be cleaned up when the context is - popped. - - Uses :meth:`contextlib.ExitStack.enter_context`. It calls the - resource's ``__enter__()`` method and returns the result. When - the context is popped, it closes the stack, which calls the - resource's ``__exit__()`` method. - - To register a cleanup function for something that isn't a - context manager, use :meth:`call_on_close`. Or use something - from :mod:`contextlib` to turn it into a context manager first. - - .. code-block:: python - - @click.group() - @click.option("--name") - @click.pass_context - def cli(ctx): - ctx.obj = ctx.with_resource(connect_db(name)) - - :param context_manager: The context manager to enter. - :return: Whatever ``context_manager.__enter__()`` returns. - - .. versionadded:: 8.0 - """ - return self._exit_stack.enter_context(context_manager) - - def call_on_close(self, f: t.Callable[..., t.Any]) -> t.Callable[..., t.Any]: - """Register a function to be called when the context tears down. - - This can be used to close resources opened during the script - execution. Resources that support Python's context manager - protocol which would be used in a ``with`` statement should be - registered with :meth:`with_resource` instead. - - :param f: The function to execute on teardown. - """ - return self._exit_stack.callback(f) - - def close(self) -> None: - """Invoke all close callbacks registered with - :meth:`call_on_close`, and exit all context managers entered - with :meth:`with_resource`. - """ - self._close_with_exception_info(None, None, None) - - def _close_with_exception_info( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> bool | None: - """Unwind the exit stack by calling its :meth:`__exit__` providing the exception - information to allow for exception handling by the various resources registered - using :meth;`with_resource` - - :return: Whatever ``exit_stack.__exit__()`` returns. - """ - exit_result = self._exit_stack.__exit__(exc_type, exc_value, tb) - # In case the context is reused, create a new exit stack. - self._exit_stack = ExitStack() - - return exit_result - - @property - def command_path(self) -> str: - """The computed command path. This is used for the ``usage`` - information on the help page. It's automatically created by - combining the info names of the chain of contexts to the root. - """ - rv = "" - if self.info_name is not None: - rv = self.info_name - if self.parent is not None: - parent_command_path = [self.parent.command_path] - - if isinstance(self.parent.command, Command): - for param in self.parent.command.get_params(self): - parent_command_path.extend(param.get_usage_pieces(self)) - - rv = f"{' '.join(parent_command_path)} {rv}" - return rv.lstrip() - - def find_root(self) -> Context: - """Finds the outermost context.""" - node = self - while node.parent is not None: - node = node.parent - return node - - def find_object(self, object_type: type[V]) -> V | None: - """Finds the closest object of a given type.""" - node: Context | None = self - - while node is not None: - if isinstance(node.obj, object_type): - return node.obj - - node = node.parent - - return None - - def ensure_object(self, object_type: type[V]) -> V: - """Like :meth:`find_object` but sets the innermost object to a - new instance of `object_type` if it does not exist. - """ - rv = self.find_object(object_type) - if rv is None: - self.obj = rv = object_type() - return rv - - def _default_map_has(self, name: str | None) -> bool: - """Check if :attr:`default_map` contains a real value for ``name``. - - Returns ``False`` when the key is absent, the map is ``None``, - ``name`` is ``None``, or the stored value is the internal - :data:`UNSET` sentinel. - """ - return ( - name is not None - and self.default_map is not None - and name in self.default_map - and self.default_map[name] is not UNSET - ) - - @t.overload - def lookup_default( - self, name: str, call: t.Literal[True] = True - ) -> t.Any | None: ... - - @t.overload - def lookup_default( - self, name: str, call: t.Literal[False] = ... - ) -> t.Any | t.Callable[[], t.Any] | None: ... - - def lookup_default(self, name: str, call: bool = True) -> t.Any | None: - """Get the default for a parameter from :attr:`default_map`. - - :param name: Name of the parameter. - :param call: If the default is a callable, call it. Disable to - return the callable instead. - - .. versionchanged:: 8.0 - Added the ``call`` parameter. - """ - if not self._default_map_has(name): - return None - - # Assert to make the type checker happy. - assert self.default_map is not None - value = self.default_map[name] - - if call and callable(value): - return value() - - return value - - def fail(self, message: str) -> t.NoReturn: - """Aborts the execution of the program with a specific error - message. - - :param message: the error message to fail with. - """ - raise UsageError(message, self) - - def abort(self) -> t.NoReturn: - """Aborts the script.""" - raise Abort() - - def exit(self, code: int = 0) -> t.NoReturn: - """Exits the application with a given exit code. - - .. versionchanged:: 8.2 - Callbacks and context managers registered with :meth:`call_on_close` - and :meth:`with_resource` are closed before exiting. - """ - self.close() - raise Exit(code) - - def get_usage(self) -> str: - """Helper method to get formatted usage string for the current - context and command. - """ - return self.command.get_usage(self) - - def get_help(self) -> str: - """Helper method to get formatted help page for the current - context and command. - """ - return self.command.get_help(self) - - def _make_sub_context(self, command: Command) -> Context: - """Create a new context of the same type as this context, but - for a new command. - - :meta private: - """ - return type(self)(command, info_name=command.name, parent=self) - - @t.overload - def invoke( - self, callback: t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any - ) -> V: ... - - @t.overload - def invoke(self, callback: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: ... - - def invoke( - self, callback: Command | t.Callable[..., V], /, *args: t.Any, **kwargs: t.Any - ) -> t.Any | V: - """Invokes a command callback in exactly the way it expects. There - are two ways to invoke this method: - - 1. the first argument can be a callback and all other arguments and - keyword arguments are forwarded directly to the function. - 2. the first argument is a click command object. In that case all - arguments are forwarded as well but proper click parameters - (options and click arguments) must be keyword arguments and Click - will fill in defaults. - - .. versionchanged:: 8.0 - All ``kwargs`` are tracked in :attr:`params` so they will be - passed if :meth:`forward` is called at multiple levels. - - .. versionchanged:: 3.2 - A new context is created, and missing arguments use default values. - """ - if isinstance(callback, Command): - other_cmd = callback - - if other_cmd.callback is None: - raise TypeError( - "The given command does not have a callback that can be invoked." - ) - else: - callback = t.cast("t.Callable[..., V]", other_cmd.callback) - - ctx = self._make_sub_context(other_cmd) - - for param in other_cmd.params: - if param.name not in kwargs and param.expose_value: - default_value = param.get_default(ctx) - # We explicitly hide the :attr:`UNSET` value to the user, as we - # choose to make it an implementation detail. And because ``invoke`` - # has been designed as part of Click public API, we return ``None`` - # instead. Refs: - # https://github.com/pallets/click/issues/3066 - # https://github.com/pallets/click/issues/3065 - # https://github.com/pallets/click/pull/3068 - if default_value is UNSET: - default_value = None - kwargs[param.name] = param.type_cast_value(ctx, default_value) - - # Track all kwargs as params, so that forward() will pass - # them on in subsequent calls. - ctx.params.update(kwargs) - else: - ctx = self - - with augment_usage_errors(self): - with ctx: - return callback(*args, **kwargs) - - def forward(self, cmd: Command, /, *args: t.Any, **kwargs: t.Any) -> t.Any: - """Similar to :meth:`invoke` but fills in default keyword - arguments from the current context if the other command expects - it. This cannot invoke callbacks directly, only other commands. - - .. versionchanged:: 8.0 - All ``kwargs`` are tracked in :attr:`params` so they will be - passed if ``forward`` is called at multiple levels. - """ - # Can only forward to other commands, not direct callbacks. - if not isinstance(cmd, Command): - raise TypeError("Callback is not a command.") - - for param in self.params: - if param not in kwargs: - kwargs[param] = self.params[param] - - return self.invoke(cmd, *args, **kwargs) - - def set_parameter_source(self, name: str, source: ParameterSource) -> None: - """Set the source of a parameter. This indicates the location - from which the value of the parameter was obtained. - - :param name: The name of the parameter. - :param source: A member of :class:`~click.core.ParameterSource`. - """ - self._parameter_source[name] = source - - def get_parameter_source(self, name: str) -> ParameterSource | None: - """Get the source of a parameter. This indicates the location - from which the value of the parameter was obtained. - - This can be useful for determining when a user specified a value - on the command line that is the same as the default value. It - will be :attr:`~click.core.ParameterSource.DEFAULT` only if the - value was actually taken from the default. - - :param name: The name of the parameter. - :rtype: ParameterSource - - .. versionchanged:: 8.0 - Returns ``None`` if the parameter was not provided from any - source. - """ - return self._parameter_source.get(name) - - -class Command: - """Commands are the basic building block of command line interfaces in - Click. A basic command handles command line parsing and might dispatch - more parsing to commands nested below it. - - :param name: the name of the command to use unless a group overrides it. - :param context_settings: an optional dictionary with defaults that are - passed to the context object. - :param callback: the callback to invoke. This is optional. - :param params: the parameters to register with this command. This can - be either :class:`Option` or :class:`Argument` objects. - :param help: the help string to use for this command. - :param epilog: like the help string but it's printed at the end of the - help page after everything else. - :param short_help: the short help to use for this command. This is - shown on the command listing of the parent command. - :param add_help_option: by default each command registers a ``--help`` - option. This can be disabled by this parameter. - :param no_args_is_help: this controls what happens if no arguments are - provided. This option is disabled by default. - If enabled this will add ``--help`` as argument - if no arguments are passed - :param hidden: hide this command from help outputs. - :param deprecated: If ``True`` or non-empty string, issues a message - indicating that the command is deprecated and highlights - its deprecation in --help. The message can be customized - by using a string as the value. - - .. versionchanged:: 8.2 - This is the base class for all commands, not ``BaseCommand``. - ``deprecated`` can be set to a string as well to customize the - deprecation message. - - .. versionchanged:: 8.1 - ``help``, ``epilog``, and ``short_help`` are stored unprocessed, - all formatting is done when outputting help text, not at init, - and is done even if not using the ``@command`` decorator. - - .. versionchanged:: 8.0 - Added a ``repr`` showing the command name. - - .. versionchanged:: 7.1 - Added the ``no_args_is_help`` parameter. - - .. versionchanged:: 2.0 - Added the ``context_settings`` parameter. - """ - - #: The context class to create with :meth:`make_context`. - #: - #: .. versionadded:: 8.0 - context_class: type[Context] = Context - - #: the default for the :attr:`Context.allow_extra_args` flag. - allow_extra_args = False - - #: the default for the :attr:`Context.allow_interspersed_args` flag. - allow_interspersed_args = True - - #: the default for the :attr:`Context.ignore_unknown_options` flag. - ignore_unknown_options = False - - def __init__( - self, - name: str | None, - context_settings: cabc.MutableMapping[str, t.Any] | None = None, - callback: t.Callable[..., t.Any] | None = None, - params: list[Parameter] | None = None, - help: str | None = None, - epilog: str | None = None, - short_help: str | None = None, - options_metavar: str | None = "[OPTIONS]", - add_help_option: bool = True, - no_args_is_help: bool = False, - hidden: bool = False, - deprecated: bool | str = False, - ) -> None: - #: the name the command thinks it has. Upon registering a command - #: on a :class:`Group` the group will default the command name - #: with this information. You should instead use the - #: :class:`Context`\'s :attr:`~Context.info_name` attribute. - self.name = name - - if context_settings is None: - context_settings = {} - - #: an optional dictionary with defaults passed to the context. - self.context_settings: cabc.MutableMapping[str, t.Any] = context_settings - - #: the callback to execute when the command fires. This might be - #: `None` in which case nothing happens. - self.callback = callback - #: the list of parameters for this command in the order they - #: should show up in the help page and execute. Eager parameters - #: will automatically be handled before non eager ones. - self.params: list[Parameter] = params or [] - self.help = help - self.epilog = epilog - self.options_metavar = options_metavar - self.short_help = short_help - self.add_help_option = add_help_option - self._help_option = None - self.no_args_is_help = no_args_is_help - self.hidden = hidden - self.deprecated = deprecated - - def to_info_dict(self, ctx: Context) -> dict[str, t.Any]: - return { - "name": self.name, - "params": [param.to_info_dict() for param in self.get_params(ctx)], - "help": self.help, - "epilog": self.epilog, - "short_help": self.short_help, - "hidden": self.hidden, - "deprecated": self.deprecated, - } - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self.name}>" - - def get_usage(self, ctx: Context) -> str: - """Formats the usage line into a string and returns it. - - Calls :meth:`format_usage` internally. - """ - formatter = ctx.make_formatter() - self.format_usage(ctx, formatter) - return formatter.getvalue().rstrip("\n") - - def get_params(self, ctx: Context) -> list[Parameter]: - params = self.params - help_option = self.get_help_option(ctx) - - if help_option is not None: - params = [*params, help_option] - - if __debug__: - import warnings - - opts = [opt for param in params for opt in param.opts] - opts_counter = Counter(opts) - duplicate_opts = (opt for opt, count in opts_counter.items() if count > 1) - - for duplicate_opt in duplicate_opts: - warnings.warn( - ( - f"The parameter {duplicate_opt} is used more than once. " - "Remove its duplicate as parameters should be unique." - ), - stacklevel=3, - ) - - return params - - def format_usage(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the usage line into the formatter. - - This is a low-level method called by :meth:`get_usage`. - """ - pieces = self.collect_usage_pieces(ctx) - formatter.write_usage(ctx.command_path, " ".join(pieces)) - - def collect_usage_pieces(self, ctx: Context) -> list[str]: - """Returns all the pieces that go into the usage line and returns - it as a list of strings. - """ - rv = [self.options_metavar] if self.options_metavar else [] - - for param in self.get_params(ctx): - rv.extend(param.get_usage_pieces(ctx)) - - return rv - - def get_help_option_names(self, ctx: Context) -> list[str]: - """Returns the names for the help option.""" - all_names = set(ctx.help_option_names) - for param in self.params: - all_names.difference_update(param.opts) - all_names.difference_update(param.secondary_opts) - return list(all_names) - - def get_help_option(self, ctx: Context) -> Option | None: - """Returns the help option object. - - Skipped if :attr:`add_help_option` is ``False``. - - .. versionchanged:: 8.1.8 - The help option is now cached to avoid creating it multiple times. - """ - help_option_names = self.get_help_option_names(ctx) - - if not help_option_names or not self.add_help_option: - return None - - # Cache the help option object in private _help_option attribute to - # avoid creating it multiple times. Not doing this will break the - # callback ordering by iter_params_for_processing(), which relies on - # object comparison. - if self._help_option is None: - # Avoid circular import. - from .decorators import help_option - - # Apply help_option decorator and pop resulting option - help_option(*help_option_names)(self) - self._help_option = self.params.pop() # type: ignore[assignment] - - return self._help_option - - def make_parser(self, ctx: Context) -> _OptionParser: - """Creates the underlying option parser for this command.""" - parser = _OptionParser(ctx) - for param in self.get_params(ctx): - param.add_to_parser(parser, ctx) - return parser - - def get_help(self, ctx: Context) -> str: - """Formats the help into a string and returns it. - - Calls :meth:`format_help` internally. - """ - formatter = ctx.make_formatter() - self.format_help(ctx, formatter) - return formatter.getvalue().rstrip("\n") - - def get_short_help_str(self, limit: int = 45) -> str: - """Gets short help for the command or makes it by shortening the - long help string. - """ - if self.short_help: - text = inspect.cleandoc(self.short_help) - elif self.help: - text = make_default_short_help(self.help, limit) - else: - text = "" - - if self.deprecated: - text = f"{_(text)} {_format_deprecated_label(self.deprecated)}" - - return text.strip() - - def format_help(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the help into the formatter if it exists. - - This is a low-level method called by :meth:`get_help`. - - This calls the following methods: - - - :meth:`format_usage` - - :meth:`format_help_text` - - :meth:`format_options` - - :meth:`format_epilog` - """ - self.format_usage(ctx, formatter) - self.format_help_text(ctx, formatter) - self.format_options(ctx, formatter) - self.format_epilog(ctx, formatter) - - def format_help_text(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the help text to the formatter if it exists.""" - if self.help is not None: - # truncate the help text to the first form feed - text = inspect.cleandoc(self.help).partition("\f")[0] - else: - text = "" - - if self.deprecated: - text = f"{_(text)} {_format_deprecated_label(self.deprecated)}" - - if text: - formatter.write_paragraph() - - with formatter.indentation(): - formatter.write_text(text) - - def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes all the options into the formatter if they exist.""" - opts = [] - for param in self.get_params(ctx): - rv = param.get_help_record(ctx) - if rv is not None: - opts.append(rv) - - if opts: - with formatter.section(_("Options")): - formatter.write_dl(opts) - - def format_epilog(self, ctx: Context, formatter: HelpFormatter) -> None: - """Writes the epilog into the formatter if it exists.""" - if self.epilog: - epilog = inspect.cleandoc(self.epilog) - formatter.write_paragraph() - - with formatter.indentation(): - formatter.write_text(epilog) - - def make_context( - self, - info_name: str | None, - args: list[str], - parent: Context | None = None, - **extra: t.Any, - ) -> Context: - """This function when given an info name and arguments will kick - off the parsing and create a new :class:`Context`. It does not - invoke the actual command callback though. - - To quickly customize the context class used without overriding - this method, set the :attr:`context_class` attribute. - - :param info_name: the info name for this invocation. Generally this - is the most descriptive name for the script or - command. For the toplevel script it's usually - the name of the script, for commands below it's - the name of the command. - :param args: the arguments to parse as list of strings. - :param parent: the parent context if available. - :param extra: extra keyword arguments forwarded to the context - constructor. - - .. versionchanged:: 8.0 - Added the :attr:`context_class` attribute. - """ - for key, value in self.context_settings.items(): - if key not in extra: - extra[key] = value - - ctx = self.context_class(self, info_name=info_name, parent=parent, **extra) - - with ctx.scope(cleanup=False): - self.parse_args(ctx, args) - return ctx - - def parse_args(self, ctx: Context, args: list[str]) -> list[str]: - if not args and self.no_args_is_help and not ctx.resilient_parsing: - raise NoArgsIsHelpError(ctx) - - parser = self.make_parser(ctx) - opts, args, param_order = parser.parse_args(args=args) - - for param in iter_params_for_processing(param_order, self.get_params(ctx)): - _, args = param.handle_parse_result(ctx, opts, args) - - # We now have all parameters' values into `ctx.params`, but the data may contain - # the `UNSET` sentinel. - # Convert `UNSET` to `None` to ensure that the user doesn't see `UNSET`. - # - # Waiting until after the initial parse to convert allows us to treat `UNSET` - # more like a missing value when multiple params use the same name. - # Refs: - # https://github.com/pallets/click/issues/3071 - # https://github.com/pallets/click/pull/3079 - for name, value in ctx.params.items(): - if value is UNSET: - ctx.params[name] = None - - if args and not ctx.allow_extra_args and not ctx.resilient_parsing: - ctx.fail( - ngettext( - "Got unexpected extra argument ({args})", - "Got unexpected extra arguments ({args})", - len(args), - ).format(args=" ".join(map(str, args))) - ) - - ctx.args = args - ctx._opt_prefixes.update(parser._opt_prefixes) - return args - - def invoke(self, ctx: Context) -> t.Any: - """Given a context, this invokes the attached callback (if it exists) - in the right way. - """ - if self.deprecated: - message = _( - "DeprecationWarning: The command {name!r} is deprecated.{extra_message}" - ).format( - name=self.name, - extra_message=_format_deprecated_suffix(self.deprecated), - ) - echo(style(message, fg="red"), err=True) - - if self.callback is not None: - return ctx.invoke(self.callback, **ctx.params) - - def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: - """Return a list of completions for the incomplete value. Looks - at the names of options and chained multi-commands. - - Any command could be part of a chained multi-command, so sibling - commands are valid at any point during command completion. - - :param ctx: Invocation context for this command. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - results: list[CompletionItem] = [] - - if incomplete and not incomplete[0].isalnum(): - for param in self.get_params(ctx): - if ( - not isinstance(param, Option) - or param.hidden - or ( - not param.multiple - and ctx.get_parameter_source(param.name) - is ParameterSource.COMMANDLINE - ) - ): - continue - - results.extend( - CompletionItem(name, help=param.help) - for name in [*param.opts, *param.secondary_opts] - if name.startswith(incomplete) - ) - - while ctx.parent is not None: - ctx = ctx.parent - - if isinstance(ctx.command, Group) and ctx.command.chain: - results.extend( - CompletionItem(name, help=command.get_short_help_str()) - for name, command in _complete_visible_commands(ctx, incomplete) - if name not in ctx._protected_args - ) - - return results - - @t.overload - def main( - self, - args: cabc.Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: t.Literal[True] = True, - **extra: t.Any, - ) -> t.NoReturn: ... - - @t.overload - def main( - self, - args: cabc.Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: bool = ..., - **extra: t.Any, - ) -> t.Any: ... - - def main( - self, - args: cabc.Sequence[str] | None = None, - prog_name: str | None = None, - complete_var: str | None = None, - standalone_mode: bool = True, - windows_expand_args: bool = True, - **extra: t.Any, - ) -> t.Any: - """This is the way to invoke a script with all the bells and - whistles as a command line application. This will always terminate - the application after a call. If this is not wanted, ``SystemExit`` - needs to be caught. - - This method is also available by directly calling the instance of - a :class:`Command`. - - :param args: the arguments that should be used for parsing. If not - provided, ``sys.argv[1:]`` is used. - :param prog_name: the program name that should be used. By default - the program name is constructed by taking the file - name from ``sys.argv[0]``. - :param complete_var: the environment variable that controls the - bash completion support. The default is - ``"__COMPLETE"`` with prog_name in - uppercase. - :param standalone_mode: the default behavior is to invoke the script - in standalone mode. Click will then - handle exceptions and convert them into - error messages and the function will never - return but shut down the interpreter. If - this is set to `False` they will be - propagated to the caller and the return - value of this function is the return value - of :meth:`invoke`. - :param windows_expand_args: Expand glob patterns, user dir, and - env vars in command line args on Windows. - :param extra: extra keyword arguments are forwarded to the context - constructor. See :class:`Context` for more information. - - .. versionchanged:: 8.0.1 - Added the ``windows_expand_args`` parameter to allow - disabling command line arg expansion on Windows. - - .. versionchanged:: 8.0 - When taking arguments from ``sys.argv`` on Windows, glob - patterns, user dir, and env vars are expanded. - - .. versionchanged:: 3.0 - Added the ``standalone_mode`` parameter. - """ - if args is None: - args = sys.argv[1:] - - if os.name == "nt" and windows_expand_args: - args = _expand_args(args) - else: - args = list(args) - - if prog_name is None: - prog_name = _detect_program_name() - - # Process shell completion requests and exit early. - self._main_shell_completion(extra, prog_name, complete_var) - - try: - try: - with self.make_context(prog_name, args, **extra) as ctx: - rv = self.invoke(ctx) - if not standalone_mode: - return rv - # it's not safe to `ctx.exit(rv)` here! - # note that `rv` may actually contain data like "1" which - # has obvious effects - # more subtle case: `rv=[None, None]` can come out of - # chained commands which all returned `None` -- so it's not - # even always obvious that `rv` indicates success/failure - # by its truthiness/falsiness - ctx.exit() - except (EOFError, KeyboardInterrupt) as e: - echo(file=sys.stderr) - raise Abort() from e - except ClickException as e: - if not standalone_mode: - raise - e.show() - sys.exit(e.exit_code) - except OSError as e: - if e.errno == errno.EPIPE: - sys.stdout = t.cast(t.TextIO, PacifyFlushWrapper(sys.stdout)) - sys.stderr = t.cast(t.TextIO, PacifyFlushWrapper(sys.stderr)) - sys.exit(1) - else: - raise - except Exit as e: - if standalone_mode: - sys.exit(e.exit_code) - else: - # in non-standalone mode, return the exit code - # note that this is only reached if `self.invoke` above raises - # an Exit explicitly -- thus bypassing the check there which - # would return its result - # the results of non-standalone execution may therefore be - # somewhat ambiguous: if there are codepaths which lead to - # `ctx.exit(1)` and to `return 1`, the caller won't be able to - # tell the difference between the two - return e.exit_code - except Abort: - if not standalone_mode: - raise - echo(_("Aborted!"), file=sys.stderr) - sys.exit(1) - - def _main_shell_completion( - self, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - complete_var: str | None = None, - ) -> None: - """Check if the shell is asking for tab completion, process - that, then exit early. Called from :meth:`main` before the - program is invoked. - - :param prog_name: Name of the executable in the shell. - :param complete_var: Name of the environment variable that holds - the completion instruction. Defaults to - ``_{PROG_NAME}_COMPLETE``. - - .. versionchanged:: 8.2.0 - Dots (``.``) in ``prog_name`` are replaced with underscores (``_``). - """ - if complete_var is None: - complete_name = prog_name.replace("-", "_").replace(".", "_") - complete_var = f"_{complete_name}_COMPLETE".upper() - - instruction = os.environ.get(complete_var) - - if not instruction: - return - - from .shell_completion import shell_complete - - rv = shell_complete(self, ctx_args, prog_name, complete_var, instruction) - sys.exit(rv) - - def __call__(self, *args: t.Any, **kwargs: t.Any) -> t.Any: - """Alias for :meth:`main`.""" - return self.main(*args, **kwargs) - - -class _FakeSubclassCheck(type): - def __subclasscheck__(cls, subclass: type) -> bool: - return issubclass(subclass, cls.__bases__[0]) - - def __instancecheck__(cls, instance: t.Any) -> bool: - return isinstance(instance, cls.__bases__[0]) - - -class _BaseCommand(Command, metaclass=_FakeSubclassCheck): - """ - .. deprecated:: 8.2 - Will be removed in Click 9.0. Use ``Command`` instead. - """ - - -class Group(Command): - """A group is a command that nests other commands (or more groups). - - :param name: The name of the group command. - :param commands: Map names to :class:`Command` objects. Can be a list, which - will use :attr:`Command.name` as the keys. - :param invoke_without_command: Invoke the group's callback even if a - subcommand is not given. - :param no_args_is_help: If no arguments are given, show the group's help and - exit. Defaults to the opposite of ``invoke_without_command``. - :param subcommand_metavar: How to represent the subcommand argument in help. - The default will represent whether ``chain`` is set or not. - :param chain: Allow passing more than one subcommand argument. After parsing - a command's arguments, if any arguments remain another command will be - matched, and so on. - :param result_callback: A function to call after the group's and - subcommand's callbacks. The value returned by the subcommand is passed. - If ``chain`` is enabled, the value will be a list of values returned by - all the commands. If ``invoke_without_command`` is enabled, the value - will be the value returned by the group's callback, or an empty list if - ``chain`` is enabled. - :param kwargs: Other arguments passed to :class:`Command`. - - .. versionchanged:: 8.0 - The ``commands`` argument can be a list of command objects. - - .. versionchanged:: 8.2 - Merged with and replaces the ``MultiCommand`` base class. - """ - - allow_extra_args = True - allow_interspersed_args = False - - #: If set, this is used by the group's :meth:`command` decorator - #: as the default :class:`Command` class. This is useful to make all - #: subcommands use a custom command class. - #: - #: .. versionadded:: 8.0 - command_class: type[Command] | None = None - - #: If set, this is used by the group's :meth:`group` decorator - #: as the default :class:`Group` class. This is useful to make all - #: subgroups use a custom group class. - #: - #: If set to the special value :class:`type` (literally - #: ``group_class = type``), this group's class will be used as the - #: default class. This makes a custom group class continue to make - #: custom groups. - #: - #: .. versionadded:: 8.0 - group_class: type[Group] | type[type] | None = None - # Literal[type] isn't valid, so use Type[type] - - def __init__( - self, - name: str | None = None, - commands: cabc.MutableMapping[str, Command] - | cabc.Sequence[Command] - | None = None, - invoke_without_command: bool = False, - no_args_is_help: bool | None = None, - subcommand_metavar: str | None = None, - chain: bool = False, - result_callback: t.Callable[..., t.Any] | None = None, - **kwargs: t.Any, - ) -> None: - super().__init__(name, **kwargs) - - if commands is None: - commands = {} - elif isinstance(commands, abc.Sequence): - commands = {c.name: c for c in commands if c.name is not None} - - #: The registered subcommands by their exported names. - self.commands: cabc.MutableMapping[str, Command] = commands - - if no_args_is_help is None: - no_args_is_help = not invoke_without_command - - self.no_args_is_help = no_args_is_help - self.invoke_without_command = invoke_without_command - - if subcommand_metavar is None: - if chain: - subcommand_metavar = "COMMAND1 [ARGS]... [COMMAND2 [ARGS]...]..." - else: - subcommand_metavar = "COMMAND [ARGS]..." - - self.subcommand_metavar = subcommand_metavar - self.chain = chain - # The result callback that is stored. This can be set or - # overridden with the :func:`result_callback` decorator. - self._result_callback = result_callback - - if self.chain: - for param in self.params: - if isinstance(param, Argument) and not param.required: - raise RuntimeError( - "A group in chain mode cannot have optional arguments." - ) - - def to_info_dict(self, ctx: Context) -> dict[str, t.Any]: - info_dict = super().to_info_dict(ctx) - commands = {} - - for name in self.list_commands(ctx): - command = self.get_command(ctx, name) - - if command is None: - continue - - sub_ctx = ctx._make_sub_context(command) - - with sub_ctx.scope(cleanup=False): - commands[name] = command.to_info_dict(sub_ctx) - - info_dict.update(commands=commands, chain=self.chain) - return info_dict - - def add_command(self, cmd: Command, name: str | None = None) -> None: - """Registers another :class:`Command` with this group. If the name - is not provided, the name of the command is used. - """ - name = name or cmd.name - if name is None: - raise TypeError("Command has no name.") - _check_nested_chain(self, name, cmd, register=True) - self.commands[name] = cmd - - @t.overload - def command(self, __func: t.Callable[..., t.Any]) -> Command: ... - - @t.overload - def command( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Command]: ... - - def command( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Command] | Command: - """A shortcut decorator for declaring and attaching a command to - the group. This takes the same arguments as :func:`command` and - immediately registers the created command with this group by - calling :meth:`add_command`. - - To customize the command class used, set the - :attr:`command_class` attribute. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - - .. versionchanged:: 8.0 - Added the :attr:`command_class` attribute. - """ - from .decorators import command - - func: t.Callable[..., t.Any] | None = None - - if args and callable(args[0]): - assert len(args) == 1 and not kwargs, ( - "Use 'command(**kwargs)(callable)' to provide arguments." - ) - (func,) = args - args = () - - if self.command_class and kwargs.get("cls") is None: - kwargs["cls"] = self.command_class - - def decorator(f: t.Callable[..., t.Any]) -> Command: - cmd: Command = command(*args, **kwargs)(f) - self.add_command(cmd) - return cmd - - if func is not None: - return decorator(func) - - return decorator - - @t.overload - def group(self, __func: t.Callable[..., t.Any]) -> Group: ... - - @t.overload - def group( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Group]: ... - - def group( - self, *args: t.Any, **kwargs: t.Any - ) -> t.Callable[[t.Callable[..., t.Any]], Group] | Group: - """A shortcut decorator for declaring and attaching a group to - the group. This takes the same arguments as :func:`group` and - immediately registers the created group with this group by - calling :meth:`add_command`. - - To customize the group class used, set the :attr:`group_class` - attribute. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - - .. versionchanged:: 8.0 - Added the :attr:`group_class` attribute. - """ - from .decorators import group - - func: t.Callable[..., t.Any] | None = None - - if args and callable(args[0]): - assert len(args) == 1 and not kwargs, ( - "Use 'group(**kwargs)(callable)' to provide arguments." - ) - (func,) = args - args = () - - if self.group_class is not None and kwargs.get("cls") is None: - if self.group_class is type: - kwargs["cls"] = type(self) - else: - kwargs["cls"] = self.group_class - - def decorator(f: t.Callable[..., t.Any]) -> Group: - cmd: Group = group(*args, **kwargs)(f) - self.add_command(cmd) - return cmd - - if func is not None: - return decorator(func) - - return decorator - - def result_callback(self, replace: bool = False) -> t.Callable[[F], F]: - """Adds a result callback to the command. By default if a - result callback is already registered this will chain them but - this can be disabled with the `replace` parameter. The result - callback is invoked with the return value of the subcommand - (or the list of return values from all subcommands if chaining - is enabled) as well as the parameters as they would be passed - to the main callback. - - Example:: - - @click.group() - @click.option('-i', '--input', default=23) - def cli(input): - return 42 - - @cli.result_callback() - def process_result(result, input): - return result + input - - :param replace: if set to `True` an already existing result - callback will be removed. - - .. versionchanged:: 8.0 - Renamed from ``resultcallback``. - - .. versionadded:: 3.0 - """ - - def decorator(f: F) -> F: - old_callback = self._result_callback - - if old_callback is None or replace: - self._result_callback = f - return f - - def function(value: t.Any, /, *args: t.Any, **kwargs: t.Any) -> t.Any: - inner = old_callback(value, *args, **kwargs) - return f(inner, *args, **kwargs) - - self._result_callback = rv = update_wrapper(t.cast(F, function), f) - return rv # type: ignore[return-value] - - return decorator - - def get_command(self, ctx: Context, cmd_name: str) -> Command | None: - """Given a context and a command name, this returns a :class:`Command` - object if it exists or returns ``None``. - """ - return self.commands.get(cmd_name) - - def list_commands(self, ctx: Context) -> list[str]: - """Returns a list of subcommand names in the order they should appear.""" - return sorted(self.commands) - - def collect_usage_pieces(self, ctx: Context) -> list[str]: - rv = super().collect_usage_pieces(ctx) - rv.append(self.subcommand_metavar) - return rv - - def format_options(self, ctx: Context, formatter: HelpFormatter) -> None: - super().format_options(ctx, formatter) - self.format_commands(ctx, formatter) - - def format_commands(self, ctx: Context, formatter: HelpFormatter) -> None: - """Extra format methods for multi methods that adds all the commands - after the options. - """ - commands = [] - for subcommand in self.list_commands(ctx): - cmd = self.get_command(ctx, subcommand) - # What is this, the tool lied about a command. Ignore it - if cmd is None: - continue - if cmd.hidden: - continue - - commands.append((subcommand, cmd)) - - # allow for 3 times the default spacing - if len(commands): - limit = formatter.width - 6 - max(len(cmd[0]) for cmd in commands) - - rows = [] - for subcommand, cmd in commands: - help = cmd.get_short_help_str(limit) - rows.append((subcommand, help)) - - if rows: - with formatter.section(_("Commands")): - formatter.write_dl(rows) - - def parse_args(self, ctx: Context, args: list[str]) -> list[str]: - if not args and self.no_args_is_help and not ctx.resilient_parsing: - raise NoArgsIsHelpError(ctx) - - rest = super().parse_args(ctx, args) - - if self.chain: - ctx._protected_args = rest - ctx.args = [] - elif rest: - ctx._protected_args, ctx.args = rest[:1], rest[1:] - - return ctx.args - - def invoke(self, ctx: Context) -> t.Any: - def _process_result(value: t.Any) -> t.Any: - if self._result_callback is not None: - value = ctx.invoke(self._result_callback, value, **ctx.params) - return value - - if not ctx._protected_args: - if self.invoke_without_command: - # No subcommand was invoked, so the result callback is - # invoked with the group return value for regular - # groups, or an empty list for chained groups. - with ctx: - rv = super().invoke(ctx) - return _process_result([] if self.chain else rv) - ctx.fail(_("Missing command.")) - - # Fetch args back out - args = [*ctx._protected_args, *ctx.args] - ctx.args = [] - ctx._protected_args = [] - - # If we're not in chain mode, we only allow the invocation of a - # single command but we also inform the current context about the - # name of the command to invoke. - if not self.chain: - # Make sure the context is entered so we do not clean up - # resources until the result processor has worked. - with ctx: - cmd_name, cmd, args = self.resolve_command(ctx, args) - assert cmd is not None - ctx.invoked_subcommand = cmd_name - super().invoke(ctx) - sub_ctx = cmd.make_context(cmd_name, args, parent=ctx) - with sub_ctx: - return _process_result(sub_ctx.command.invoke(sub_ctx)) - - # In chain mode we create the contexts step by step, but after the - # base command has been invoked. Because at that point we do not - # know the subcommands yet, the invoked subcommand attribute is - # set to ``*`` to inform the command that subcommands are executed - # but nothing else. - with ctx: - ctx.invoked_subcommand = "*" if args else None - super().invoke(ctx) - - # Otherwise we make every single context and invoke them in a - # chain. In that case the return value to the result processor - # is the list of all invoked subcommand's results. - contexts = [] - while args: - cmd_name, cmd, args = self.resolve_command(ctx, args) - assert cmd is not None - sub_ctx = cmd.make_context( - cmd_name, - args, - parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False, - ) - contexts.append(sub_ctx) - args, sub_ctx.args = sub_ctx.args, [] - - rv = [] - for sub_ctx in contexts: - with sub_ctx: - rv.append(sub_ctx.command.invoke(sub_ctx)) - return _process_result(rv) - - def resolve_command( - self, ctx: Context, args: list[str] - ) -> tuple[str | None, Command | None, list[str]]: - cmd_name = make_str(args[0]) - - # Get the command - cmd = self.get_command(ctx, cmd_name) - - # If we can't find the command but there is a normalization - # function available, we try with that one. - if cmd is None and ctx.token_normalize_func is not None: - cmd_name = ctx.token_normalize_func(cmd_name) - cmd = self.get_command(ctx, cmd_name) - - # If we don't find the command we want to show an error message - # to the user that it was not provided. However, there is - # something else we should do: if the first argument looks like - # an option we want to kick off parsing again for arguments to - # resolve things like --help which now should go to the main - # place. - if cmd is None and not ctx.resilient_parsing: - if _split_opt(cmd_name)[0]: - self.parse_args(ctx, args) - raise NoSuchCommand(cmd_name, possibilities=self.commands, ctx=ctx) - return cmd_name if cmd else None, cmd, args[1:] - - def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: - """Return a list of completions for the incomplete value. Looks - at the names of options, subcommands, and chained - multi-commands. - - :param ctx: Invocation context for this command. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - results = [ - CompletionItem(name, help=command.get_short_help_str()) - for name, command in _complete_visible_commands(ctx, incomplete) - ] - results.extend(super().shell_complete(ctx, incomplete)) - return results - - -class _MultiCommand(Group, metaclass=_FakeSubclassCheck): - """ - .. deprecated:: 8.2 - Will be removed in Click 9.0. Use ``Group`` instead. - """ - - -class CommandCollection(Group): - """A :class:`Group` that looks up subcommands on other groups. If a command - is not found on this group, each registered source is checked in order. - Parameters on a source are not added to this group, and a source's callback - is not invoked when invoking its commands. In other words, this "flattens" - commands in many groups into this one group. - - :param name: The name of the group command. - :param sources: A list of :class:`Group` objects to look up commands from. - :param kwargs: Other arguments passed to :class:`Group`. - - .. versionchanged:: 8.2 - This is a subclass of ``Group``. Commands are looked up first on this - group, then each of its sources. - """ - - def __init__( - self, - name: str | None = None, - sources: list[Group] | None = None, - **kwargs: t.Any, - ) -> None: - super().__init__(name, **kwargs) - #: The list of registered groups. - self.sources: list[Group] = sources or [] - - def add_source(self, group: Group) -> None: - """Add a group as a source of commands.""" - self.sources.append(group) - - def get_command(self, ctx: Context, cmd_name: str) -> Command | None: - rv = super().get_command(ctx, cmd_name) - - if rv is not None: - return rv - - for source in self.sources: - rv = source.get_command(ctx, cmd_name) - - if rv is not None: - if self.chain: - _check_nested_chain(self, cmd_name, rv) - - return rv - - return None - - def list_commands(self, ctx: Context) -> list[str]: - rv: set[str] = set(super().list_commands(ctx)) - - for source in self.sources: - rv.update(source.list_commands(ctx)) - - return sorted(rv) - - -def _check_iter(value: t.Any) -> cabc.Iterator[t.Any]: - """Check if the value is iterable but not a string. Raises a type - error, or return an iterator over the value. - """ - if isinstance(value, str): - raise TypeError - - return iter(value) - - -class Parameter(ABC): - r"""A parameter to a command comes in two versions: they are either - :class:`Option`\s or :class:`Argument`\s. Other subclasses are currently - not supported by design as some of the internals for parsing are - intentionally not finalized. - - Some settings are supported by both options and arguments. - - :param param_decls: the parameter declarations for this option or - argument. This is a list of flags or argument - names. - :param type: the type that should be used. Either a :class:`ParamType` - or a Python type. The latter is converted into the former - automatically if supported. - :param required: controls if this is optional or not. - :param default: the default value if omitted. This can also be a callable, - in which case it's invoked when the default is needed - without any arguments. - :param callback: A function to further process or validate the value - after type conversion. It is called as ``f(ctx, param, value)`` - and must return the value. It is called for all sources, - including prompts. - :param nargs: the number of arguments to match. If not ``1`` the return - value is a tuple instead of single value. The default for - nargs is ``1`` (except if the type is a tuple, then it's - the arity of the tuple). If ``nargs=-1``, all remaining - parameters are collected. - :param metavar: how the value is represented in the help page. - :param expose_value: if this is `True` then the value is passed onwards - to the command callback and stored on the context, - otherwise it's skipped. - :param is_eager: eager values are processed before non eager ones. This - should not be set for arguments or it will inverse the - order of processing. - :param envvar: environment variable(s) that are used to provide a default value for - this parameter. This can be a string or a sequence of strings. If a sequence is - given, only the first non-empty environment variable is used for the parameter. - :param shell_complete: A function that returns custom shell - completions. Used instead of the param's type completion if - given. Takes ``ctx, param, incomplete`` and must return a list - of :class:`~click.shell_completion.CompletionItem` or a list of - strings. - :param deprecated: If ``True`` or non-empty string, issues a message - indicating that the argument is deprecated and highlights - its deprecation in --help. The message can be customized - by using a string as the value. A deprecated parameter - cannot be required, a ValueError will be raised otherwise. - - .. versionchanged:: 8.2.0 - Introduction of ``deprecated``. - - .. versionchanged:: 8.2 - Adding duplicate parameter names to a :class:`~click.core.Command` will - result in a ``UserWarning`` being shown. - - .. versionchanged:: 8.2 - Adding duplicate parameter names to a :class:`~click.core.Command` will - result in a ``UserWarning`` being shown. - - .. versionchanged:: 8.0 - ``process_value`` validates required parameters and bounded - ``nargs``, and invokes the parameter callback before returning - the value. This allows the callback to validate prompts. - ``full_process_value`` is removed. - - .. versionchanged:: 8.0 - ``autocompletion`` is renamed to ``shell_complete`` and has new - semantics described above. The old name is deprecated and will - be removed in 8.1, until then it will be wrapped to match the - new requirements. - - .. versionchanged:: 8.0 - For ``multiple=True, nargs>1``, the default must be a list of - tuples. - - .. versionchanged:: 8.0 - Setting a default is no longer required for ``nargs>1``, it will - default to ``None``. ``multiple=True`` or ``nargs=-1`` will - default to ``()``. - - .. versionchanged:: 7.1 - Empty environment variables are ignored rather than taking the - empty string value. This makes it possible for scripts to clear - variables if they can't unset them. - - .. versionchanged:: 2.0 - Changed signature for parameter callback to also be passed the - parameter. The old callback format will still work, but it will - raise a warning to give you a chance to migrate the code easier. - """ - - param_type_name = "parameter" - - def __init__( - self, - param_decls: cabc.Sequence[str] | None = None, - type: types.ParamType[t.Any] | t.Any | None = None, - required: bool = False, - # XXX The default historically embed two concepts: - # - the declaration of a Parameter object carrying the default (handy to - # arbitrage the default value of coupled Parameters sharing the same - # self.name, like flag options), - # - and the actual value of the default. - # It is confusing and is the source of many issues discussed in: - # https://github.com/pallets/click/pull/3030 - # In the future, we might think of splitting it in two, not unlike - # Option.is_flag and Option.flag_value: we could have something like - # Parameter.is_default and Parameter.default_value. - default: t.Any | t.Callable[[], t.Any] | None = UNSET, - callback: t.Callable[[Context, Parameter, t.Any], t.Any] | None = None, - nargs: int | None = None, - multiple: bool = False, - metavar: str | None = None, - expose_value: bool = True, - is_eager: bool = False, - envvar: str | cabc.Sequence[str] | None = None, - shell_complete: t.Callable[ - [Context, Parameter, str], list[CompletionItem] | list[str] - ] - | None = None, - deprecated: bool | str = False, - ) -> None: - self.name: str - self.opts: list[str] - self.secondary_opts: list[str] - self.name, self.opts, self.secondary_opts = self._parse_decls( - param_decls or (), expose_value - ) - self.type: types.ParamType[t.Any] = types.convert_type(type, default) - - # Default nargs to what the type tells us if we have that - # information available. - if nargs is None: - if self.type.is_composite: - nargs = self.type.arity - else: - nargs = 1 - - self.required = required - self.callback = callback - self.nargs = nargs - self.multiple = multiple - self.expose_value = expose_value - self.default: t.Any | t.Callable[[], t.Any] | None = default - # Whether the user passed ``default`` explicitly to the constructor. - # Captured before any auto-derived default (like ``False`` for boolean - # flags in :class:`Option`) replaces the :data:`UNSET` sentinel, so it - # remains ``False`` when the default was inferred rather than chosen. - # Refs: https://github.com/pallets/click/issues/3403 - self._default_explicit: bool = default is not UNSET - self.is_eager = is_eager - self.metavar = metavar - self.envvar = envvar - self._custom_shell_complete = shell_complete - self.deprecated = deprecated - - if __debug__: - if self.type.is_composite and nargs != self.type.arity: - raise ValueError( - f"'nargs' must be {self.type.arity} (or None) for" - f" type {self.type!r}, but it was {nargs}." - ) - - if required and deprecated: - raise ValueError( - f"The {self.param_type_name} '{self.human_readable_name}' " - "is deprecated and still required. A deprecated " - f"{self.param_type_name} cannot be required." - ) - - def to_info_dict(self) -> dict[str, t.Any]: - """Gather information that could be useful for a tool generating - user-facing documentation. - - Use :meth:`click.Context.to_info_dict` to traverse the entire - CLI structure. - - .. versionchanged:: 8.3.0 - Returns ``None`` for the :attr:`default` if it was not set. - - .. versionadded:: 8.0 - """ - return { - "name": self.name, - "param_type_name": self.param_type_name, - "opts": self.opts, - "secondary_opts": self.secondary_opts, - "type": self.type.to_info_dict(), - "required": self.required, - "nargs": self.nargs, - "multiple": self.multiple, - # We explicitly hide the :attr:`UNSET` value to the user, as we choose to - # make it an implementation detail. And because ``to_info_dict`` has been - # designed for documentation purposes, we return ``None`` instead. - "default": self.default if self.default is not UNSET else None, - "envvar": self.envvar, - } - - def __repr__(self) -> str: - return f"<{self.__class__.__name__} {self.name}>" - - @abstractmethod - def _parse_decls( - self, decls: cabc.Sequence[str], expose_value: bool - ) -> tuple[str, list[str], list[str]]: ... - - @property - def human_readable_name(self) -> str: - """Returns the human readable name of this parameter. This is the - same as the name for options, but the metavar for arguments. - """ - return self.name - - def make_metavar(self, ctx: Context) -> str: - if self.metavar is not None: - return self.metavar - - metavar = self.type.get_metavar(param=self, ctx=ctx) - - if metavar is None: - metavar = self.type.name.upper() - - if self.nargs != 1: - metavar += "..." - - return metavar - - @t.overload - def get_default( - self, ctx: Context, call: t.Literal[True] = True - ) -> t.Any | None: ... - - @t.overload - def get_default( - self, ctx: Context, call: bool = ... - ) -> t.Any | t.Callable[[], t.Any] | None: ... - - def get_default( - self, ctx: Context, call: bool = True - ) -> t.Any | t.Callable[[], t.Any] | None: - """Get the default for the parameter. Tries - :meth:`Context.lookup_default` first, then the local default. - - :param ctx: Current context. - :param call: If the default is a callable, call it. Disable to - return the callable instead. - - .. versionchanged:: 8.0.2 - Type casting is no longer performed when getting a default. - - .. versionchanged:: 8.0.1 - Type casting can fail in resilient parsing mode. Invalid - defaults will not prevent showing help text. - - .. versionchanged:: 8.0 - Looks at ``ctx.default_map`` first. - - .. versionchanged:: 8.0 - Added the ``call`` parameter. - """ - value = ctx.lookup_default(self.name, call=False) - - if value is None and not ctx._default_map_has(self.name): - value = self.default - - if call and callable(value): - value = value() - - return value - - @abstractmethod - def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: ... - - def consume_value( - self, ctx: Context, opts: cabc.Mapping[str, t.Any] - ) -> tuple[t.Any, ParameterSource]: - """Returns the parameter value produced by the parser. - - If the parser did not produce a value from user input, the value is either - sourced from the environment variable, the default map, or the parameter's - default value. In that order of precedence. - - If no value is found, an internal sentinel value is returned. - - :meta private: - """ - # Collect from the parse the value passed by the user to the CLI. - value = opts.get(self.name, UNSET) - # If the value is set, it means it was sourced from the command line by the - # parser, otherwise it left unset by default. - source = ( - ParameterSource.COMMANDLINE - if value is not UNSET - else ParameterSource.DEFAULT - ) - - if value is UNSET: - envvar_value = self.value_from_envvar(ctx) - if envvar_value is not None: - value = envvar_value - source = ParameterSource.ENVIRONMENT - - if value is UNSET: - default_map_value = ctx.lookup_default(self.name) - if default_map_value is not None or ctx._default_map_has(self.name): - value = default_map_value - source = ParameterSource.DEFAULT_MAP - - # A string from default_map must be split for multi-value - # parameters, matching value_from_envvar behavior. - if isinstance(value, str) and self.nargs != 1: - value = self.type.split_envvar_value(value) - - if value is UNSET: - default_value = self.get_default(ctx) - if default_value is not UNSET: - value = default_value - source = ParameterSource.DEFAULT - - return value, source - - def type_cast_value(self, ctx: Context, value: t.Any) -> t.Any: - """Convert and validate a value against the parameter's - :attr:`type`, :attr:`multiple`, and :attr:`nargs`. - """ - if value is None: - if self.multiple or self.nargs == -1: - return () - else: - return value - - def check_iter(value: t.Any) -> cabc.Iterator[t.Any]: - try: - return _check_iter(value) - except TypeError: - # This should only happen when passing in args manually, - # the parser should construct an iterable when parsing - # the command line. - raise BadParameter( - _("Value must be an iterable."), ctx=ctx, param=self - ) from None - - # Define the conversion function based on nargs and type. - - if self.nargs == 1 or self.type.is_composite: - - def convert(value: t.Any) -> t.Any: - return self.type(value, param=self, ctx=ctx) - - elif self.nargs == -1: - - def convert(value: t.Any) -> t.Any: # tuple[t.Any, ...] - return tuple(self.type(x, self, ctx) for x in check_iter(value)) - - else: # nargs > 1 - - def convert(value: t.Any) -> t.Any: # tuple[t.Any, ...] - value = tuple(check_iter(value)) - - if len(value) != self.nargs: - raise BadParameter( - ngettext( - "Takes {nargs} values but 1 was given.", - "Takes {nargs} values but {len} were given.", - len(value), - ).format(nargs=self.nargs, len=len(value)), - ctx=ctx, - param=self, - ) - - return tuple(self.type(x, self, ctx) for x in value) - - if self.multiple: - return tuple(convert(x) for x in check_iter(value)) - - return convert(value) - - def value_is_missing(self, value: t.Any) -> bool: - """A value is considered missing if: - - - it is :attr:`UNSET`, - - or if it is an empty sequence while the parameter is suppose to have - non-single value (i.e. :attr:`nargs` is not ``1`` or :attr:`multiple` is - set). - - :meta private: - """ - if value is UNSET: - return True - - if (self.nargs != 1 or self.multiple) and value == (): - return True - - return False - - def process_value(self, ctx: Context, value: t.Any) -> t.Any: - """Process the value of this parameter: - - 1. Type cast the value using :meth:`type_cast_value`. - 2. Check if the value is missing (see: :meth:`value_is_missing`), and raise - :exc:`MissingParameter` if it is required. - 3. If a :attr:`callback` is set, call it to have the value replaced by the - result of the callback. If the value was not set, the callback receive - ``None``. This keep the legacy behavior as it was before the introduction of - the :attr:`UNSET` sentinel. - - :meta private: - """ - # shelter `type_cast_value` from ever seeing an `UNSET` value by handling the - # cases in which `UNSET` gets special treatment explicitly at this layer - # - # Refs: - # https://github.com/pallets/click/issues/3069 - if value is UNSET: - if self.multiple or self.nargs == -1: - value = () - else: - value = self.type_cast_value(ctx, value) - - if self.required and self.value_is_missing(value): - raise MissingParameter(ctx=ctx, param=self) - - if self.callback is not None: - # Legacy case: UNSET is not exposed directly to the callback, but converted - # to None. - if value is UNSET: - value = None - - # Search for parameters with UNSET values in the context. - unset_keys = {k: None for k, v in ctx.params.items() if v is UNSET} - # No UNSET values, call the callback as usual. - if not unset_keys: - value = self.callback(ctx, self, value) - - # Legacy case: provide a temporarily manipulated context to the callback - # to hide UNSET values as None. - # - # Refs: - # https://github.com/pallets/click/issues/3136 - # https://github.com/pallets/click/pull/3137 - else: - # Add another layer to the context stack to clearly hint that the - # context is temporarily modified. - with ctx: - # Update the context parameters to replace UNSET with None. - ctx.params.update(unset_keys) - # Feed these fake context parameters to the callback. - value = self.callback(ctx, self, value) - # Restore the UNSET values in the context parameters. - ctx.params.update( - { - k: UNSET - for k in unset_keys - # Only restore keys that are present and still None, in case - # the callback modified other parameters. - if k in ctx.params and ctx.params[k] is None - } - ) - - return value - - def resolve_envvar_value(self, ctx: Context) -> str | None: - """Returns the value found in the environment variable(s) attached to this - parameter. - - Environment variables values are `always returned as strings - `_. - - This method returns ``None`` if: - - - the :attr:`envvar` property is not set on the :class:`Parameter`, - - the environment variable is not found in the environment, - - the variable is found in the environment but its value is empty (i.e. the - environment variable is present but has an empty string). - - If :attr:`envvar` is setup with multiple environment variables, - then only the first non-empty value is returned. - - .. caution:: - - The raw value extracted from the environment is not normalized and is - returned as-is. Any normalization or reconciliation is performed later by - the :class:`Parameter`'s :attr:`type`. - - :meta private: - """ - if not self.envvar: - return None - - if isinstance(self.envvar, str): - rv = os.environ.get(self.envvar) - - if rv: - return rv - else: - for envvar in self.envvar: - rv = os.environ.get(envvar) - - # Return the first non-empty value of the list of environment variables. - if rv: - return rv - # Else, absence of value is interpreted as an environment variable that - # is not set, so proceed to the next one. - - return None - - def value_from_envvar(self, ctx: Context) -> str | cabc.Sequence[str] | None: - """Process the raw environment variable string for this parameter. - - Returns the string as-is or splits it into a sequence of strings if the - parameter is expecting multiple values (i.e. its :attr:`nargs` property is set - to a value other than ``1``). - - :meta private: - """ - rv = self.resolve_envvar_value(ctx) - - if rv is not None and self.nargs != 1: - return self.type.split_envvar_value(rv) - - return rv - - def handle_parse_result( - self, ctx: Context, opts: cabc.Mapping[str, t.Any], args: list[str] - ) -> tuple[t.Any, list[str]]: - """Process the value produced by the parser from user input. - - Always process the value through the Parameter's :attr:`type`, wherever it - comes from. - - If the parameter is deprecated, this method warn the user about it. But only if - the value has been explicitly set by the user (and as such, is not coming from - a default). - - :meta private: - """ - # Capture the slot's existing state before we mutate - # ``_parameter_source`` so the write decision below can compare our - # incoming source against the source of the option that already wrote - # the slot (if any). - existing_value = ctx.params.get(self.name, UNSET) - existing_source = ctx.get_parameter_source(self.name) - existing_default_explicit = ctx._param_default_explicit.get(self.name, False) - - with augment_usage_errors(ctx, param=self): - value, source = self.consume_value(ctx, opts) - - # Record the source before processing so eager callbacks and type - # conversion can inspect it. Restored after arbitration if this - # option loses a feature-switch group. - ctx.set_parameter_source(self.name, source) - - # Display a deprecation warning if necessary. - if ( - self.deprecated - and value is not UNSET - and source < ParameterSource.DEFAULT_MAP - ): - message = _( - "DeprecationWarning: The {param_type} {name!r} is deprecated." - "{extra_message}" - ).format( - param_type=self.param_type_name, - name=self.human_readable_name, - extra_message=_format_deprecated_suffix(self.deprecated), - ) - echo(style(message, fg="red"), err=True) - - # Process the value through the parameter's type. - try: - value = self.process_value(ctx, value) - except Exception: - if not ctx.resilient_parsing: - raise - # In resilient parsing mode, we do not want to fail the command if the - # value is incompatible with the parameter type, so we reset the value - # to UNSET, which will be interpreted as a missing value. - value = UNSET - - # Arbitrate the slot when several parameters target the same variable - # name (feature-switch groups). See: https://github.com/pallets/click/issues/3403 - slot_empty = existing_value is UNSET - more_explicit = existing_source is not None and source < existing_source - same_source = existing_source is not None and source == existing_source - auto_would_downgrade_explicit = ( - same_source - and source == ParameterSource.DEFAULT - and existing_default_explicit - and not self._default_explicit - ) - is_winner = ( - slot_empty - or more_explicit - or (same_source and not auto_would_downgrade_explicit) - ) - - if is_winner: - if self.expose_value: - ctx.params[self.name] = value - ctx._param_default_explicit[self.name] = self._default_explicit - elif existing_source is not None: - # Lost arbitration; restore the winning option's source. - ctx.set_parameter_source(self.name, existing_source) - # else: ctx.params[self.name] was populated by code that bypassed - # handle_parse_result (from another option's callback for example). Keep - # the provisional source recorded before process_value so downstream - # lookups don't return ``None``. - - return value, args - - def get_help_record(self, ctx: Context) -> tuple[str, str] | None: - return None - - def get_usage_pieces(self, ctx: Context) -> list[str]: - return [] - - def get_error_hint(self, ctx: Context | None) -> str: - """Get a stringified version of the param for use in error messages to - indicate which param caused the error. - - .. versionchanged:: 8.4.0 - ``ctx`` can be ``None``. - """ - hint_list = self.opts or [self.human_readable_name] - return " / ".join(f"'{x}'" for x in hint_list) - - def shell_complete(self, ctx: Context, incomplete: str) -> list[CompletionItem]: - """Return a list of completions for the incomplete value. If a - ``shell_complete`` function was given during init, it is used. - Otherwise, the :attr:`type` - :meth:`~click.types.ParamType[t.Any].shell_complete` function is used. - - :param ctx: Invocation context for this command. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - if self._custom_shell_complete is not None: - results = self._custom_shell_complete(ctx, self, incomplete) - - if results and isinstance(results[0], str): - from click.shell_completion import CompletionItem - - results = [CompletionItem(c) for c in results] - - return t.cast("list[CompletionItem]", results) - - return self.type.shell_complete(ctx, self, incomplete) - - -class Option(Parameter): - """Options are usually optional values on the command line and - have some extra features that arguments don't have. - - All other parameters are passed onwards to the parameter constructor. - - :param show_default: Show the default value for this option in its - help text. Values are not shown by default, unless - :attr:`Context.show_default` is ``True``. If this value is a - string, it shows that string in parentheses instead of the - actual value. This is particularly useful for dynamic options. - For single option boolean flags, the default remains hidden if - its value is ``False``. - :param show_envvar: Controls if an environment variable should be - shown on the help page and error messages. - Normally, environment variables are not shown. - :param prompt: If set to ``True`` or a non empty string then the - user will be prompted for input. If set to ``True`` the prompt - will be the option name capitalized. A deprecated option cannot be - prompted. - :param confirmation_prompt: Prompt a second time to confirm the - value if it was prompted for. Can be set to a string instead of - ``True`` to customize the message. - :param prompt_required: If set to ``False``, the user will be - prompted for input only when the option was specified as a flag - without a value. - :param hide_input: If this is ``True`` then the input on the prompt - will be hidden from the user. This is useful for password input. - :param is_flag: forces this option to act as a flag. The default is - auto detection. - :param flag_value: which value should be used for this flag if it's - enabled. This is set to a boolean automatically if - the option string contains a slash to mark two options. - :param multiple: if this is set to `True` then the argument is accepted - multiple times and recorded. This is similar to ``nargs`` - in how it works but supports arbitrary number of - arguments. - :param count: this flag makes an option increment an integer. - :param allow_from_autoenv: if this is enabled then the value of this - parameter will be pulled from an environment - variable in case a prefix is defined on the - context. - :param help: the help string. - :param hidden: hide this option from help outputs. - :param attrs: Other command arguments described in :class:`Parameter`. - - .. versionchanged:: 8.4.0 - Non-basic ``flag_value`` types (not ``str``, ``int``, ``float``, or - ``bool``) are passed through unchanged instead of being stringified. - Previously, ``type=click.UNPROCESSED`` was required to preserve them. - - .. versionchanged:: 8.2 - ``envvar`` used with ``flag_value`` will always use the ``flag_value``, - previously it would use the value of the environment variable. - - .. versionchanged:: 8.1 - Help text indentation is cleaned here instead of only in the - ``@option`` decorator. - - .. versionchanged:: 8.1 - The ``show_default`` parameter overrides - ``Context.show_default``. - - .. versionchanged:: 8.1 - The default of a single option boolean flag is not shown if the - default value is ``False``. - - .. versionchanged:: 8.0.1 - ``type`` is detected from ``flag_value`` if given, for basic Python - types (``str``, ``int``, ``float``, ``bool``). - """ - - param_type_name = "option" - - def __init__( - self, - param_decls: cabc.Sequence[str] | None = None, - show_default: bool | str | None = None, - prompt: bool | str = False, - confirmation_prompt: bool | str = False, - prompt_required: bool = True, - hide_input: bool = False, - is_flag: bool | None = None, - flag_value: t.Any = UNSET, - multiple: bool = False, - count: bool = False, - allow_from_autoenv: bool = True, - type: types.ParamType[t.Any] | t.Any | None = None, - help: str | None = None, - hidden: bool = False, - show_choices: bool = True, - show_envvar: bool = False, - deprecated: bool | str = False, - **attrs: t.Any, - ) -> None: - if help: - help = inspect.cleandoc(help) - - super().__init__( - param_decls, type=type, multiple=multiple, deprecated=deprecated, **attrs - ) - - if prompt is True: - if not self.name: - raise TypeError("'name' is required with 'prompt=True'.") - - prompt_text: str | None = self.name.replace("_", " ").capitalize() - elif prompt is False: - prompt_text = None - else: - prompt_text = prompt - - if deprecated: - label = _format_deprecated_label(deprecated) - help = f"{help} {label}" if help is not None else label - - self.prompt = prompt_text - self.confirmation_prompt = confirmation_prompt - self.prompt_required = prompt_required - self.hide_input = hide_input - self.hidden = hidden - - # The _flag_needs_value property tells the parser that this option is a flag - # that cannot be used standalone and needs a value. With this information, the - # parser can determine whether to consider the next user-provided argument in - # the CLI as a value for this flag or as a new option. - # If prompt is enabled but not required, then it opens the possibility for the - # option to gets its value from the user. - self._flag_needs_value = self.prompt is not None and not self.prompt_required - - # Auto-detect if this is a flag or not. - if is_flag is None: - # Implicitly a flag because flag_value was set. - if flag_value is not UNSET: - is_flag = True - # Not a flag, but when used as a flag it shows a prompt. - elif self._flag_needs_value: - is_flag = False - # Implicitly a flag because secondary options names were given. - elif self.secondary_opts: - is_flag = True - - # The option is explicitly not a flag, but to determine whether or not it needs - # value, we need to check if `flag_value` or `default` was set. Either one is - # sufficient. - # Ref: https://github.com/pallets/click/issues/3084 - elif is_flag is False and not self._flag_needs_value: - self._flag_needs_value = flag_value is not UNSET or self.default is UNSET - - if is_flag: - # Set missing default for flags if not explicitly required or prompted. - if self.default is UNSET and not self.required and not self.prompt: - if multiple: - self.default = () - - # Auto-detect the type of the flag based on the flag_value. - if type is None: - # A flag without a flag_value is a boolean flag. - if flag_value is UNSET: - self.type: types.ParamType[t.Any] = types.BoolParamType() - # If the flag value is a boolean, use BoolParamType. - elif isinstance(flag_value, bool): - self.type = types.BoolParamType() - # Otherwise, guess the type from the flag value. - else: - guessed = types.convert_type(None, flag_value) - if ( - isinstance(guessed, types.StringParamType) - and not isinstance(flag_value, str) - and flag_value is not None - ): - # The flag_value type couldn't be auto-detected - # (not str, int, float, or bool). Since flag_value - # is a programmer-provided Python object, not CLI - # input, pass it through unchanged instead of - # stringifying it. - self.type = types.UNPROCESSED - else: - self.type = guessed - - self.is_flag: bool = bool(is_flag) - self.is_bool_flag: bool = bool( - is_flag and isinstance(self.type, types.BoolParamType) - ) - self.flag_value: t.Any = flag_value - - # Set boolean flag default to False if unset and not required. - if self.is_bool_flag: - if self.default is UNSET and not self.required: - self.default = False - - # The alignment of default to the flag_value is resolved lazily in - # get_default() to prevent callable flag_values (like classes) from - # being instantiated. Refs: - # https://github.com/pallets/click/issues/3121 - # https://github.com/pallets/click/issues/3024#issuecomment-3146199461 - # https://github.com/pallets/click/pull/3030/commits/06847da - - # Set the default flag_value if it is not set. - if self.flag_value is UNSET: - if self.is_flag: - self.flag_value = True - else: - self.flag_value = None - - # Counting. - self.count = count - if count: - if type is None: - self.type = types.IntRange(min=0) - if self.default is UNSET: - self.default = 0 - - self.allow_from_autoenv = allow_from_autoenv - self.help = help - self.show_default = show_default - self.show_choices = show_choices - self.show_envvar = show_envvar - - if __debug__: - if deprecated and prompt: - raise ValueError("`deprecated` options cannot use `prompt`.") - - if self.nargs == -1: - raise TypeError("nargs=-1 is not supported for options.") - - if not self.is_bool_flag and self.secondary_opts: - raise TypeError("Secondary flag is not valid for non-boolean flag.") - - if self.is_bool_flag and self.hide_input and self.prompt is not None: - raise TypeError( - "'prompt' with 'hide_input' is not valid for boolean flag." - ) - - if self.count: - if self.multiple: - raise TypeError("'count' is not valid with 'multiple'.") - - if self.is_flag: - raise TypeError("'count' is not valid with 'is_flag'.") - - def to_info_dict(self) -> dict[str, t.Any]: - """ - .. versionchanged:: 8.3.0 - Returns ``None`` for the :attr:`flag_value` if it was not set. - """ - info_dict = super().to_info_dict() - info_dict.update( - help=self.help, - prompt=self.prompt, - is_flag=self.is_flag, - # We explicitly hide the :attr:`UNSET` value to the user, as we choose to - # make it an implementation detail. And because ``to_info_dict`` has been - # designed for documentation purposes, we return ``None`` instead. - flag_value=self.flag_value if self.flag_value is not UNSET else None, - count=self.count, - hidden=self.hidden, - ) - return info_dict - - def get_default( - self, ctx: Context, call: bool = True - ) -> t.Any | t.Callable[[], t.Any] | None: - """Return the default value for this option. - - For non-boolean flag options, ``default=True`` is treated as a sentinel - meaning "activate this flag by default" and is resolved to - :attr:`flag_value`. For example, with ``--upper/--lower`` feature - switches where ``flag_value="upper"`` and ``default=True``, the default - resolves to ``"upper"``. - - .. caution:: - This substitution only applies to non-boolean flags - (:attr:`is_bool_flag` is ``False``). For boolean flags, ``True`` is - a legitimate Python value and ``default=True`` is returned as-is. - - .. versionchanged:: 8.3.3 - ``default=True`` is no longer substituted with ``flag_value`` for - boolean flags, fixing negative boolean flags like - ``flag_value=False, default=True``. - """ - value = super().get_default(ctx, call=False) - - # Resolve default=True to flag_value lazily (here instead of - # __init__) to prevent callable flag_values (like classes) from - # being instantiated by the callable check below. - if value is True and self.is_flag and not self.is_bool_flag: - value = self.flag_value - elif call and callable(value): - value = value() - - return value - - def get_error_hint(self, ctx: Context | None) -> str: - result = super().get_error_hint(ctx) - if self.show_envvar and self.envvar is not None: - result += f" (env var: '{self.envvar}')" - return result - - def _parse_decls( - self, decls: cabc.Sequence[str], expose_value: bool - ) -> tuple[str, list[str], list[str]]: - opts = [] - secondary_opts = [] - name = None - possible_names = [] - - for decl in decls: - if decl.isidentifier(): - if name is not None: - raise TypeError(_("Name '{name}' defined twice").format(name=name)) - name = decl - else: - split_char = ";" if decl[:1] == "/" else "/" - if split_char in decl: - first, second = decl.split(split_char, 1) - first = first.rstrip() - if first: - possible_names.append(_split_opt(first)) - opts.append(first) - second = second.lstrip() - if second: - secondary_opts.append(second.lstrip()) - if first == second: - raise ValueError( - _( - "Boolean option {decl!r} cannot use the" - " same flag for true/false." - ).format(decl=decl) - ) - else: - possible_names.append(_split_opt(decl)) - opts.append(decl) - - if name is None and possible_names: - possible_names.sort(key=lambda x: -len(x[0])) # group long options first - name = possible_names[0][1].replace("-", "_").lower() - if not name.isidentifier(): - name = None - - if name is None: - if not expose_value: - return "", opts, secondary_opts - raise TypeError( - _( - "Could not determine name for option with declarations {decls!r}" - ).format(decls=decls) - ) - - if not opts and not secondary_opts: - raise TypeError( - _( - "No options defined but a name was passed ({name})." - " Did you mean to declare an argument instead? Did" - " you mean to pass '--{name}'?" - ).format(name=name) - ) - - return name, opts, secondary_opts - - def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: - if self.multiple: - action = "append" - elif self.count: - action = "count" - else: - action = "store" - - if self.is_flag: - action = f"{action}_const" - - if self.is_bool_flag and self.secondary_opts: - parser.add_option( - obj=self, opts=self.opts, dest=self.name, action=action, const=True - ) - parser.add_option( - obj=self, - opts=self.secondary_opts, - dest=self.name, - action=action, - const=False, - ) - else: - parser.add_option( - obj=self, - opts=self.opts, - dest=self.name, - action=action, - const=self.flag_value, - ) - else: - parser.add_option( - obj=self, - opts=self.opts, - dest=self.name, - action=action, - nargs=self.nargs, - ) - - def get_help_record(self, ctx: Context) -> tuple[str, str] | None: - if self.hidden: - return None - - any_prefix_is_slash = False - - def _write_opts(opts: cabc.Sequence[str]) -> str: - nonlocal any_prefix_is_slash - - rv, any_slashes = join_options(opts) - - if any_slashes: - any_prefix_is_slash = True - - if not self.is_flag and not self.count: - rv += f" {self.make_metavar(ctx=ctx)}" - - return rv - - rv = [_write_opts(self.opts)] - - if self.secondary_opts: - rv.append(_write_opts(self.secondary_opts)) - - help = self.help or "" - - extra = self.get_help_extra(ctx) - extra_items = [] - if "envvars" in extra: - extra_items.append( - _("env var: {var}").format(var=", ".join(extra["envvars"])) - ) - if "default" in extra: - extra_items.append(_("default: {default}").format(default=extra["default"])) - if "range" in extra: - extra_items.append(extra["range"]) - if "required" in extra: - extra_items.append(_(extra["required"])) - - if extra_items: - extra_str = "; ".join(extra_items) - help = f"{help} [{extra_str}]" if help else f"[{extra_str}]" - - return ("; " if any_prefix_is_slash else " / ").join(rv), help - - def get_help_extra(self, ctx: Context) -> types.OptionHelpExtra: - extra: types.OptionHelpExtra = {} - - if self.show_envvar: - envvar = self.envvar - - if envvar is None: - if ( - self.allow_from_autoenv - and ctx.auto_envvar_prefix is not None - and self.name - ): - envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" - - if envvar is not None: - if isinstance(envvar, str): - extra["envvars"] = (envvar,) - else: - extra["envvars"] = tuple(str(d) for d in envvar) - - # Temporarily enable resilient parsing to avoid type casting - # failing for the default. Might be possible to extend this to - # help formatting in general. - resilient = ctx.resilient_parsing - ctx.resilient_parsing = True - - try: - default_value = self.get_default(ctx, call=False) - finally: - ctx.resilient_parsing = resilient - - show_default = False - show_default_is_str = False - - if self.show_default is not None: - if isinstance(self.show_default, str): - show_default_is_str = show_default = True - else: - show_default = self.show_default - elif ctx.show_default is not None: - show_default = ctx.show_default - - if show_default_is_str or ( - show_default and (default_value not in (None, UNSET)) - ): - if show_default_is_str: - default_string = f"({self.show_default})" - elif isinstance(default_value, (list, tuple)): - default_string = ", ".join(str(d) for d in default_value) - elif isinstance(default_value, enum.Enum): - default_string = default_value.name - elif inspect.isfunction(default_value): - default_string = _("(dynamic)") - elif self.is_bool_flag and self.secondary_opts: - # For boolean flags that have distinct True/False opts, - # use the opt without prefix instead of the value. - default_string = _split_opt( - (self.opts if default_value else self.secondary_opts)[0] - )[1] - elif self.is_bool_flag and not self.secondary_opts and not default_value: - default_string = "" - elif isinstance(default_value, str) and default_value == "": - default_string = '""' - else: - default_string = str(default_value) - - if default_string: - extra["default"] = default_string - - if ( - isinstance(self.type, types._NumberRangeBase) - # skip count with default range type - and not (self.count and self.type.min == 0 and self.type.max is None) - ): - range_str = self.type._describe_range() - - if range_str: - extra["range"] = range_str - - if self.required: - extra["required"] = "required" - - return extra - - def prompt_for_value(self, ctx: Context) -> t.Any: - """This is an alternative flow that can be activated in the full - value processing if a value does not exist. It will prompt the - user until a valid value exists and then returns the processed - value as result. - """ - assert self.prompt is not None - - # Calculate the default before prompting anything to lock in the value before - # attempting any user interaction. - default = self.get_default(ctx) - - # A boolean flag can use a simplified [y/n] confirmation prompt. - if self.is_bool_flag: - # If we have no boolean default, we force the user to explicitly provide - # one. - if default in (UNSET, None): - default = None - # Nothing prevent you to declare an option that is simultaneously: - # 1) auto-detected as a boolean flag, - # 2) allowed to prompt, and - # 3) still declare a non-boolean default. - # This forced casting into a boolean is necessary to align any non-boolean - # default to the prompt, which is going to be a [y/n]-style confirmation - # because the option is still a boolean flag. That way, instead of [y/n], - # we get [Y/n] or [y/N] depending on the truthy value of the default. - # Refs: https://github.com/pallets/click/pull/3030#discussion_r2289180249 - else: - default = bool(default) - return confirm(self.prompt, default) - - # If show_default is given, provide this to `prompt` as well, - # otherwise we use `prompt`'s default behavior - prompt_kwargs: t.Any = {} - if self.show_default is not None: - prompt_kwargs["show_default"] = self.show_default - - return prompt( - self.prompt, - # Use ``None`` to inform the prompt() function to reiterate until a valid - # value is provided by the user if we have no default. - default=None if default is UNSET else default, - type=self.type, - hide_input=self.hide_input, - show_choices=self.show_choices, - confirmation_prompt=self.confirmation_prompt, - value_proc=lambda x: self.process_value(ctx, x), - **prompt_kwargs, - ) - - def resolve_envvar_value(self, ctx: Context) -> str | None: - """:class:`Option` resolves its environment variable the same way as - :func:`Parameter.resolve_envvar_value`, but it also supports - :attr:`Context.auto_envvar_prefix`. If we could not find an environment from - the :attr:`envvar` property, we fallback on :attr:`Context.auto_envvar_prefix` - to build dynamiccaly the environment variable name using the - :python:`{ctx.auto_envvar_prefix}_{self.name.upper()}` template. - - :meta private: - """ - rv = super().resolve_envvar_value(ctx) - - if rv is not None: - return rv - - if self.allow_from_autoenv and ctx.auto_envvar_prefix is not None and self.name: - envvar = f"{ctx.auto_envvar_prefix}_{self.name.upper()}" - rv = os.environ.get(envvar) - - if rv: - return rv - - return None - - def value_from_envvar(self, ctx: Context) -> t.Any: - """For :class:`Option`, this method processes the raw environment variable - string the same way as :func:`Parameter.value_from_envvar` does. - - But in the case of non-boolean flags, the value is analyzed to determine if the - flag is activated or not, and returns a boolean of its activation, or the - :attr:`flag_value` if the latter is set. - - This method also takes care of repeated options (i.e. options with - :attr:`multiple` set to ``True``). - - :meta private: - """ - rv = self.resolve_envvar_value(ctx) - - # Absent environment variable or an empty string is interpreted as unset. - if rv is None: - return None - - # Non-boolean flags are more liberal in what they accept. But a flag being a - # flag, its envvar value still needs to be analyzed to determine if the flag is - # activated or not. - if self.is_flag and not self.is_bool_flag: - # If the flag_value is set and match the envvar value, return it - # directly. - if self.flag_value is not UNSET and rv == self.flag_value: - return self.flag_value - # Analyze the envvar value as a boolean to know if the flag is - # activated or not. - return types.BoolParamType.str_to_bool(rv) - - # Split the envvar value if it is allowed to be repeated. - value_depth = (self.nargs != 1) + bool(self.multiple) - if value_depth > 0: - multi_rv = self.type.split_envvar_value(rv) - if self.multiple and self.nargs != 1: - multi_rv = batch(multi_rv, self.nargs) # type: ignore[assignment] - - return multi_rv - - return rv - - def consume_value( - self, ctx: Context, opts: cabc.Mapping[str, Parameter] - ) -> tuple[t.Any, ParameterSource]: - """For :class:`Option`, the value can be collected from an interactive prompt - if the option is a flag that needs a value (and the :attr:`prompt` property is - set). - - Additionally, this method handles flag option that are activated without a - value, in which case the :attr:`flag_value` is returned. - - :meta private: - """ - value, source = super().consume_value(ctx, opts) - - # The parser will emit a sentinel value if the option is allowed to as a flag - # without a value. - if value is FLAG_NEEDS_VALUE: - # If the option allows for a prompt, we start an interaction with the user. - if self.prompt is not None and not ctx.resilient_parsing: - value = self.prompt_for_value(ctx) - source = ParameterSource.PROMPT - # Else the flag takes its flag_value as value. - else: - value = self.flag_value - source = ParameterSource.COMMANDLINE - - # A flag which is activated always returns the flag value, unless the value - # comes from the explicitly sets default. - elif ( - self.is_flag - and value is True - and not self.is_bool_flag - and source < ParameterSource.DEFAULT_MAP - ): - value = self.flag_value - - # Re-interpret a multiple option which has been sent as-is by the parser. - # Here we replace each occurrence of value-less flags (marked by the - # FLAG_NEEDS_VALUE sentinel) with the flag_value. - elif ( - self.multiple - and value is not UNSET - and isinstance(value, cabc.Iterable) - and source < ParameterSource.DEFAULT_MAP - and any(v is FLAG_NEEDS_VALUE for v in value) - ): - value = [self.flag_value if v is FLAG_NEEDS_VALUE else v for v in value] - source = ParameterSource.COMMANDLINE - - # The value wasn't set, or used the param's default, prompt for one to the user - # if prompting is enabled. - elif ( - (value is UNSET or source >= ParameterSource.DEFAULT_MAP) - and self.prompt is not None - and (self.required or self.prompt_required) - and not ctx.resilient_parsing - ): - value = self.prompt_for_value(ctx) - source = ParameterSource.PROMPT - - return value, source - - def process_value(self, ctx: Context, value: t.Any) -> t.Any: - # process_value has to be overridden on Options in order to capture - # `value == UNSET` cases before `type_cast_value()` gets called. - # - # Refs: - # https://github.com/pallets/click/issues/3069 - if self.is_flag and not self.required and self.is_bool_flag and value is UNSET: - value = False - - if self.callback is not None: - value = self.callback(ctx, self, value) - - return value - - # in the normal case, rely on Parameter.process_value - return super().process_value(ctx, value) - - -class Argument(Parameter): - """Arguments are positional parameters to a command. They generally - provide fewer features than options but can have infinite ``nargs`` - and are required by default. - - All parameters are passed onwards to the constructor of :class:`Parameter`. - """ - - param_type_name = "argument" - - def __init__( - self, - param_decls: cabc.Sequence[str], - required: bool | None = None, - **attrs: t.Any, - ) -> None: - # Auto-detect the requirement status of the argument if not explicitly set. - if required is None: - # The argument gets automatically required if it has no explicit default - # value set and is setup to match at least one value. - if attrs.get("default", UNSET) is UNSET: - required = attrs.get("nargs", 1) > 0 - # If the argument has a default value, it is not required. - else: - required = False - - if "multiple" in attrs: - raise TypeError("__init__() got an unexpected keyword argument 'multiple'.") - - super().__init__(param_decls, required=required, **attrs) - - @property - def human_readable_name(self) -> str: - if self.metavar is not None: - return self.metavar - return self.name.upper() - - def make_metavar(self, ctx: Context) -> str: - if self.metavar is not None: - return self.metavar - var = self.type.get_metavar(param=self, ctx=ctx) - if not var: - var = self.name.upper() - if self.deprecated: - var += "!" - if not self.required: - var = f"[{var}]" - if self.nargs != 1: - var += "..." - return var - - def _parse_decls( - self, decls: cabc.Sequence[str], expose_value: bool - ) -> tuple[str, list[str], list[str]]: - if not decls: - if not expose_value: - return "", [], [] - raise TypeError("Argument is marked as exposed, but does not have a name.") - if len(decls) == 1: - name = arg = decls[0] - name = name.replace("-", "_").lower() - else: - raise TypeError( - _( - "Arguments take exactly one parameter declaration, got" - " {length}: {decls}." - ).format(length=len(decls), decls=decls) - ) - return name, [arg], [] - - def get_usage_pieces(self, ctx: Context) -> list[str]: - return [self.make_metavar(ctx)] - - def get_error_hint(self, ctx: Context | None) -> str: - if ctx is not None: - return f"'{self.make_metavar(ctx)}'" - return f"'{self.human_readable_name}'" - - def add_to_parser(self, parser: _OptionParser, ctx: Context) -> None: - parser.add_argument(dest=self.name, nargs=self.nargs, obj=self) - - -def __getattr__(name: str) -> object: - import warnings - - if name == "BaseCommand": - warnings.warn( - "'BaseCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Command' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _BaseCommand - - if name == "MultiCommand": - warnings.warn( - "'MultiCommand' is deprecated and will be removed in Click 9.0. Use" - " 'Group' instead.", - DeprecationWarning, - stacklevel=2, - ) - return _MultiCommand - - raise AttributeError(name) diff --git a/functions/python/data-processor/click/decorators.py b/functions/python/data-processor/click/decorators.py deleted file mode 100644 index 14aee42ea..000000000 --- a/functions/python/data-processor/click/decorators.py +++ /dev/null @@ -1,551 +0,0 @@ -from __future__ import annotations - -import inspect -import typing as t -from functools import update_wrapper -from gettext import gettext as _ - -from .core import Argument -from .core import Command -from .core import Context -from .core import Group -from .core import Option -from .core import Parameter -from .globals import get_current_context -from .utils import echo - -if t.TYPE_CHECKING: - import typing_extensions as te - - P = te.ParamSpec("P") - -R = t.TypeVar("R") -T = t.TypeVar("T") -_AnyCallable = t.Callable[..., t.Any] -FC = t.TypeVar("FC", bound="_AnyCallable | Command") - - -def pass_context(f: t.Callable[te.Concatenate[Context, P], R]) -> t.Callable[P, R]: - """Marks a callback as wanting to receive the current context - object as first argument. - """ - - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - return f(get_current_context(), *args, **kwargs) - - return update_wrapper(new_func, f) - - -def pass_obj(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: - """Similar to :func:`pass_context`, but only pass the object on the - context onwards (:attr:`Context.obj`). This is useful if that object - represents the state of a nested system. - """ - - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - return f(get_current_context().obj, *args, **kwargs) - - return update_wrapper(new_func, f) - - -def make_pass_decorator( - object_type: type[T], ensure: bool = False -) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]: - """Given an object type this creates a decorator that will work - similar to :func:`pass_obj` but instead of passing the object of the - current context, it will find the innermost context of type - :func:`object_type`. - - This generates a decorator that works roughly like this:: - - from functools import update_wrapper - - def decorator(f): - @pass_context - def new_func(ctx, *args, **kwargs): - obj = ctx.find_object(object_type) - return ctx.invoke(f, obj, *args, **kwargs) - return update_wrapper(new_func, f) - return decorator - - :param object_type: the type of the object to pass. - :param ensure: if set to `True`, a new object will be created and - remembered on the context if it's not there yet. - """ - - def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - ctx = get_current_context() - - obj: T | None - if ensure: - obj = ctx.ensure_object(object_type) - else: - obj = ctx.find_object(object_type) - - if obj is None: - raise RuntimeError( - "Managed to invoke callback without a context" - f" object of type {object_type.__name__!r}" - " existing." - ) - - return ctx.invoke(f, obj, *args, **kwargs) - - return update_wrapper(new_func, f) - - return decorator - - -def pass_meta_key( - key: str, *, doc_description: str | None = None -) -> t.Callable[[t.Callable[te.Concatenate[T, P], R]], t.Callable[P, R]]: - """Create a decorator that passes a key from - :attr:`click.Context.meta` as the first argument to the decorated - function. - - :param key: Key in ``Context.meta`` to pass. - :param doc_description: Description of the object being passed, - inserted into the decorator's docstring. Defaults to "the 'key' - key from Context.meta". - - .. versionadded:: 8.0 - """ - - def decorator(f: t.Callable[te.Concatenate[T, P], R]) -> t.Callable[P, R]: - def new_func(*args: P.args, **kwargs: P.kwargs) -> R: - ctx = get_current_context() - obj = ctx.meta[key] - return ctx.invoke(f, obj, *args, **kwargs) - - return update_wrapper(new_func, f) - - if doc_description is None: - doc_description = f"the {key!r} key from :attr:`click.Context.meta`" - - decorator.__doc__ = ( - f"Decorator that passes {doc_description} as the first argument" - " to the decorated function." - ) - return decorator - - -CmdType = t.TypeVar("CmdType", bound=Command) - - -# variant: no call, directly as decorator for a function. -@t.overload -def command(name: _AnyCallable) -> Command: ... - - -# variant: with positional name and with positional or keyword cls argument: -# @command(namearg, CommandCls, ...) or @command(namearg, cls=CommandCls, ...) -@t.overload -def command( - name: str | None, - cls: type[CmdType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], CmdType]: ... - - -# variant: name omitted, cls _must_ be a keyword argument, @command(cls=CommandCls, ...) -@t.overload -def command( - name: None = None, - *, - cls: type[CmdType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], CmdType]: ... - - -# variant: with optional string name, no cls argument provided. -@t.overload -def command( - name: str | None = ..., cls: None = None, **attrs: t.Any -) -> t.Callable[[_AnyCallable], Command]: ... - - -def command( - name: str | _AnyCallable | None = None, - cls: type[CmdType] | None = None, - **attrs: t.Any, -) -> Command | t.Callable[[_AnyCallable], Command | CmdType]: - r"""Creates a new :class:`Command` and uses the decorated function as - callback. This will also automatically attach all decorated - :func:`option`\s and :func:`argument`\s as parameters to the command. - - The name of the command defaults to the name of the function, converted to - lowercase, with underscores ``_`` replaced by dashes ``-``, and the suffixes - ``_command``, ``_cmd``, ``_group``, and ``_grp`` are removed. For example, - ``init_data_command`` becomes ``init-data``. - - All keyword arguments are forwarded to the underlying command class. - For the ``params`` argument, any decorated params are appended to - the end of the list. - - Once decorated the function turns into a :class:`Command` instance - that can be invoked as a command line utility or be attached to a - command :class:`Group`. - - :param name: The name of the command. Defaults to modifying the function's - name as described above. - :param cls: The command class to create. Defaults to :class:`Command`. - - .. versionchanged:: 8.2 - The suffixes ``_command``, ``_cmd``, ``_group``, and ``_grp`` are - removed when generating the name. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - - .. versionchanged:: 8.1 - The ``params`` argument can be used. Decorated params are - appended to the end of the list. - """ - - func: t.Callable[[_AnyCallable], t.Any] | None = None - - if callable(name): - func = name - name = None - assert cls is None, "Use 'command(cls=cls)(callable)' to specify a class." - assert not attrs, "Use 'command(**kwargs)(callable)' to provide arguments." - - if cls is None: - cls = t.cast("type[CmdType]", Command) - - def decorator(f: _AnyCallable) -> CmdType: - if isinstance(f, Command): - raise TypeError("Attempted to convert a callback into a command twice.") - - attr_params = attrs.pop("params", None) - params = attr_params if attr_params is not None else [] - - try: - decorator_params = f.__click_params__ # type: ignore - except AttributeError: - pass - else: - del f.__click_params__ # type: ignore - params.extend(reversed(decorator_params)) - - if attrs.get("help") is None: - attrs["help"] = f.__doc__ - - if t.TYPE_CHECKING: - assert cls is not None - assert not callable(name) - - if name is not None: - cmd_name = name - else: - cmd_name = f.__name__.lower().replace("_", "-") - cmd_left, sep, suffix = cmd_name.rpartition("-") - - if sep and suffix in {"command", "cmd", "group", "grp"}: - cmd_name = cmd_left - - cmd = cls(name=cmd_name, callback=f, params=params, **attrs) - cmd.__doc__ = f.__doc__ - return cmd - - if func is not None: - return decorator(func) - - return decorator - - -GrpType = t.TypeVar("GrpType", bound=Group) - - -# variant: no call, directly as decorator for a function. -@t.overload -def group(name: _AnyCallable) -> Group: ... - - -# variant: with positional name and with positional or keyword cls argument: -# @group(namearg, GroupCls, ...) or @group(namearg, cls=GroupCls, ...) -@t.overload -def group( - name: str | None, - cls: type[GrpType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], GrpType]: ... - - -# variant: name omitted, cls _must_ be a keyword argument, @group(cmd=GroupCls, ...) -@t.overload -def group( - name: None = None, - *, - cls: type[GrpType], - **attrs: t.Any, -) -> t.Callable[[_AnyCallable], GrpType]: ... - - -# variant: with optional string name, no cls argument provided. -@t.overload -def group( - name: str | None = ..., cls: None = None, **attrs: t.Any -) -> t.Callable[[_AnyCallable], Group]: ... - - -def group( - name: str | _AnyCallable | None = None, - cls: type[GrpType] | None = None, - **attrs: t.Any, -) -> Group | t.Callable[[_AnyCallable], Group | GrpType]: - """Creates a new :class:`Group` with a function as callback. This - works otherwise the same as :func:`command` just that the `cls` - parameter is set to :class:`Group`. - - .. versionchanged:: 8.1 - This decorator can be applied without parentheses. - """ - if cls is None: - cls = t.cast("type[GrpType]", Group) - - if callable(name): - return command(cls=cls, **attrs)(name) - - return command(name, cls, **attrs) - - -def _param_memo(f: t.Callable[..., t.Any], param: Parameter) -> None: - if isinstance(f, Command): - f.params.append(param) - else: - if not hasattr(f, "__click_params__"): - f.__click_params__ = [] # type: ignore - - f.__click_params__.append(param) # type: ignore - - -def argument( - *param_decls: str, cls: type[Argument] | None = None, **attrs: t.Any -) -> t.Callable[[FC], FC]: - """Attaches an argument to the command. All positional arguments are - passed as parameter declarations to :class:`Argument`; all keyword - arguments are forwarded unchanged (except ``cls``). - This is equivalent to creating an :class:`Argument` instance manually - and attaching it to the :attr:`Command.params` list. - - For the default argument class, refer to :class:`Argument` and - :class:`Parameter` for descriptions of parameters. - - :param cls: the argument class to instantiate. This defaults to - :class:`Argument`. - :param param_decls: Passed as positional arguments to the constructor of - ``cls``. - :param attrs: Passed as keyword arguments to the constructor of ``cls``. - """ - if cls is None: - cls = Argument - - def decorator(f: FC) -> FC: - _param_memo(f, cls(param_decls, **attrs)) - return f - - return decorator - - -def option( - *param_decls: str, cls: type[Option] | None = None, **attrs: t.Any -) -> t.Callable[[FC], FC]: - """Attaches an option to the command. All positional arguments are - passed as parameter declarations to :class:`Option`; all keyword - arguments are forwarded unchanged (except ``cls``). - This is equivalent to creating an :class:`Option` instance manually - and attaching it to the :attr:`Command.params` list. - - For the default option class, refer to :class:`Option` and - :class:`Parameter` for descriptions of parameters. - - :param cls: the option class to instantiate. This defaults to - :class:`Option`. - :param param_decls: Passed as positional arguments to the constructor of - ``cls``. - :param attrs: Passed as keyword arguments to the constructor of ``cls``. - """ - if cls is None: - cls = Option - - def decorator(f: FC) -> FC: - _param_memo(f, cls(param_decls, **attrs)) - return f - - return decorator - - -def confirmation_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: - """Add a ``--yes`` option which shows a prompt before continuing if - not passed. If the prompt is declined, the program will exit. - - :param param_decls: One or more option names. Defaults to the single - value ``"--yes"``. - :param kwargs: Extra arguments are passed to :func:`option`. - """ - - def callback(ctx: Context, param: Parameter, value: bool) -> None: - if not value: - ctx.abort() - - if not param_decls: - param_decls = ("--yes",) - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("callback", callback) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("prompt", _("Do you want to continue?")) - kwargs.setdefault("help", _("Confirm the action without prompting.")) - return option(*param_decls, **kwargs) - - -def password_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: - """Add a ``--password`` option which prompts for a password, hiding - input and asking to enter the value again for confirmation. - - :param param_decls: One or more option names. Defaults to the single - value ``"--password"``. - :param kwargs: Extra arguments are passed to :func:`option`. - """ - if not param_decls: - param_decls = ("--password",) - - kwargs.setdefault("prompt", True) - kwargs.setdefault("confirmation_prompt", True) - kwargs.setdefault("hide_input", True) - return option(*param_decls, **kwargs) - - -def version_option( - version: str | None = None, - *param_decls: str, - package_name: str | None = None, - prog_name: str | None = None, - message: str | None = None, - **kwargs: t.Any, -) -> t.Callable[[FC], FC]: - """Add a ``--version`` option which immediately prints the version - number and exits the program. - - If ``version`` is not provided, Click will try to detect it using - :func:`importlib.metadata.version` to get the version for the - ``package_name``. - - If ``package_name`` is not provided, Click will try to detect it by - inspecting the stack frames. This will be used to detect the - version, so it must match the name of the installed package. - - :param version: The version number to show. If not provided, Click - will try to detect it. - :param param_decls: One or more option names. Defaults to the single - value ``"--version"``. - :param package_name: The package name to detect the version from. If - not provided, Click will try to detect it. - :param prog_name: The name of the CLI to show in the message. If not - provided, it will be detected from the command. - :param message: The message to show. The values ``%(prog)s``, - ``%(package)s``, and ``%(version)s`` are available. Defaults to - ``"%(prog)s, version %(version)s"``. - :param kwargs: Extra arguments are passed to :func:`option`. - :raise RuntimeError: ``version`` could not be detected. - - .. versionchanged:: 8.0 - Add the ``package_name`` parameter, and the ``%(package)s`` - value for messages. - - .. versionchanged:: 8.0 - Use :mod:`importlib.metadata` instead of ``pkg_resources``. The - version is detected based on the package name, not the entry - point name. The Python package name must match the installed - package name, or be passed with ``package_name=``. - """ - if message is None: - message = _("%(prog)s, version %(version)s") - - if version is None and package_name is None: - frame = inspect.currentframe() - f_back = frame.f_back if frame is not None else None - f_globals = f_back.f_globals if f_back is not None else None - # break reference cycle - # https://docs.python.org/3/library/inspect.html#the-interpreter-stack - del frame - - if f_globals is not None: - package_name = f_globals.get("__name__") - - if package_name == "__main__": - package_name = f_globals.get("__package__") - - if package_name: - package_name = package_name.partition(".")[0] - - def callback(ctx: Context, param: Parameter, value: bool) -> None: - if not value or ctx.resilient_parsing: - return - - nonlocal prog_name - nonlocal version - - if prog_name is None: - prog_name = ctx.find_root().info_name - - if version is None and package_name is not None: - import importlib.metadata - - try: - version = importlib.metadata.version(package_name) - except importlib.metadata.PackageNotFoundError: - raise RuntimeError( - f"{package_name!r} is not installed. Try passing" - " 'package_name' instead." - ) from None - - if version is None: - raise RuntimeError( - f"Could not determine the version for {package_name!r} automatically." - ) - - echo( - message % {"prog": prog_name, "package": package_name, "version": version}, - color=ctx.color, - ) - ctx.exit() - - if not param_decls: - param_decls = ("--version",) - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("is_eager", True) - kwargs.setdefault("help", _("Show the version and exit.")) - kwargs["callback"] = callback - return option(*param_decls, **kwargs) - - -def help_option(*param_decls: str, **kwargs: t.Any) -> t.Callable[[FC], FC]: - """Pre-configured ``--help`` option which immediately prints the help page - and exits the program. - - :param param_decls: One or more option names. Defaults to the single - value ``"--help"``. - :param kwargs: Extra arguments are passed to :func:`option`. - """ - - def show_help(ctx: Context, param: Parameter, value: bool) -> None: - """Callback that print the help page on ```` and exits.""" - if value and not ctx.resilient_parsing: - echo(ctx.get_help(), color=ctx.color) - ctx.exit() - - if not param_decls: - param_decls = ("--help",) - - kwargs.setdefault("is_flag", True) - kwargs.setdefault("expose_value", False) - kwargs.setdefault("is_eager", True) - kwargs.setdefault("help", _("Show this message and exit.")) - kwargs.setdefault("callback", show_help) - - return option(*param_decls, **kwargs) diff --git a/functions/python/data-processor/click/exceptions.py b/functions/python/data-processor/click/exceptions.py deleted file mode 100644 index 68c280603..000000000 --- a/functions/python/data-processor/click/exceptions.py +++ /dev/null @@ -1,347 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import typing as t -from gettext import gettext as _ -from gettext import ngettext - -from ._compat import get_text_stderr -from .globals import resolve_color_default -from .utils import echo -from .utils import format_filename - -if t.TYPE_CHECKING: - from .core import Command - from .core import Context - from .core import Parameter - - -def _join_param_hints(param_hint: cabc.Sequence[str] | str | None) -> str | None: - if param_hint is not None and not isinstance(param_hint, str): - return " / ".join(repr(x) for x in param_hint) - - return param_hint - - -def _format_possibilities(possibilities: list[str]) -> str: - possibility_str = ", ".join(repr(p) for p in sorted(possibilities)) - return ngettext( - "Did you mean {possibility}?", - "(Did you mean one of: {possibilities}?)", - len(possibilities), - ).format(possibility=possibility_str, possibilities=possibility_str) - - -class ClickException(Exception): - """An exception that Click can handle and show to the user.""" - - #: The exit code for this exception. - exit_code = 1 - - def __init__(self, message: str) -> None: - super().__init__(message) - # The context will be removed by the time we print the message, so cache - # the color settings here to be used later on (in `show`) - self.show_color: bool | None = resolve_color_default() - self.message = message - - def format_message(self) -> str: - return self.message - - def __str__(self) -> str: - return self.message - - def show(self, file: t.IO[t.Any] | None = None) -> None: - if file is None: - file = get_text_stderr() - - echo( - _("Error: {message}").format(message=self.format_message()), - file=file, - color=self.show_color, - ) - - -class UsageError(ClickException): - """An internal exception that signals a usage error. This typically - aborts any further handling. - - :param message: the error message to display. - :param ctx: optionally the context that caused this error. Click will - fill in the context automatically in some situations. - """ - - exit_code = 2 - - def __init__(self, message: str, ctx: Context | None = None) -> None: - super().__init__(message) - self.ctx = ctx - self.cmd: Command | None = self.ctx.command if self.ctx else None - - def show(self, file: t.IO[t.Any] | None = None) -> None: - if file is None: - file = get_text_stderr() - color = None - hint = "" - if ( - self.ctx is not None - and self.ctx.command.get_help_option(self.ctx) is not None - ): - help_names = self.ctx.command.get_help_option_names(self.ctx) - # Pick the longest name (like ``--help`` over ``-h``) for - # readability in error messages. - hint = _("Try '{command} {option}' for help.").format( - command=self.ctx.command_path, - option=max(help_names, key=len), - ) - hint = f"{hint}\n" - if self.ctx is not None: - color = self.ctx.color - echo(f"{self.ctx.get_usage()}\n{hint}", file=file, color=color) - echo( - _("Error: {message}").format(message=self.format_message()), - file=file, - color=color, - ) - - -class BadParameter(UsageError): - """An exception that formats out a standardized error message for a - bad parameter. This is useful when thrown from a callback or type as - Click will attach contextual information to it (for instance, which - parameter it is). - - .. versionadded:: 2.0 - - :param param: the parameter object that caused this error. This can - be left out, and Click will attach this info itself - if possible. - :param param_hint: a string that shows up as parameter name. This - can be used as alternative to `param` in cases - where custom validation should happen. If it is - a string it's used as such, if it's a list then - each item is quoted and separated. - """ - - def __init__( - self, - message: str, - ctx: Context | None = None, - param: Parameter | None = None, - param_hint: cabc.Sequence[str] | str | None = None, - ) -> None: - super().__init__(message, ctx) - self.param = param - self.param_hint = param_hint - - def format_message(self) -> str: - if self.param_hint is not None: - param_hint = self.param_hint - elif self.param is not None: - param_hint = self.param.get_error_hint(self.ctx) - else: - return _("Invalid value: {message}").format(message=self.message) - - return _("Invalid value for {param_hint}: {message}").format( - param_hint=_join_param_hints(param_hint), message=self.message - ) - - -class MissingParameter(BadParameter): - """Raised if click required an option or argument but it was not - provided when invoking the script. - - .. versionadded:: 4.0 - - :param param_type: a string that indicates the type of the parameter. - The default is to inherit the parameter type from - the given `param`. Valid values are ``'parameter'``, - ``'option'`` or ``'argument'``. - """ - - def __init__( - self, - message: str | None = None, - ctx: Context | None = None, - param: Parameter | None = None, - param_hint: cabc.Sequence[str] | str | None = None, - param_type: str | None = None, - ) -> None: - super().__init__(message or "", ctx, param, param_hint) - self.param_type = param_type - - def format_message(self) -> str: - if self.param_hint is not None: - param_hint: cabc.Sequence[str] | str | None = self.param_hint - elif self.param is not None: - param_hint = self.param.get_error_hint(self.ctx) - else: - param_hint = None - - param_hint = _join_param_hints(param_hint) - param_hint = f" {param_hint}" if param_hint else "" - - param_type = self.param_type - if param_type is None and self.param is not None: - param_type = self.param.param_type_name - - msg = self.message - if self.param is not None: - msg_extra = self.param.type.get_missing_message( - param=self.param, ctx=self.ctx - ) - if msg_extra: - if msg: - msg += f". {msg_extra}" - else: - msg = msg_extra - - msg = f" {msg}" if msg else "" - - # Translate param_type for known types. - if param_type == "argument": - missing = _("Missing argument") - elif param_type == "option": - missing = _("Missing option") - elif param_type == "parameter": - missing = _("Missing parameter") - else: - missing = _("Missing {param_type}").format(param_type=param_type) - - return f"{missing}{param_hint}.{msg}" - - def __str__(self) -> str: - if not self.message: - param_name = self.param.name if self.param else None - return _("Missing parameter: {param_name}").format(param_name=param_name) - else: - return self.message - - -class NoSuchOption(UsageError): - """Raised if Click attempted to handle an option that does not exist. - - .. versionadded:: 4.0 - """ - - def __init__( - self, - option_name: str, - message: str | None = None, - possibilities: cabc.Iterable[str] | None = None, - ctx: Context | None = None, - ) -> None: - if message is None: - message = _("No such option {name!r}.").format(name=option_name) - - super().__init__(message, ctx) - self.option_name = option_name - self.possibilities: list[str] | None = None - if possibilities: - from difflib import get_close_matches - - self.possibilities = get_close_matches(option_name, possibilities) - - def format_message(self) -> str: - if not self.possibilities: - return self.message - return f"{self.message} {_format_possibilities(self.possibilities)}" - - -class NoSuchCommand(UsageError): - """Raised if Click attempted to handle a command that does not exist. - - .. versionadded:: 8.4.0 - """ - - def __init__( - self, - command_name: str, - message: str | None = None, - possibilities: cabc.Iterable[str] | None = None, - ctx: Context | None = None, - ) -> None: - if message is None: - message = _("No such command {name!r}.").format(name=command_name) - - super().__init__(message, ctx) - self.command_name = command_name - self.possibilities: list[str] | None = None - if possibilities: - from difflib import get_close_matches - - self.possibilities = get_close_matches(command_name, possibilities) - - def format_message(self) -> str: - if not self.possibilities: - return self.message - return f"{self.message} {_format_possibilities(self.possibilities)}" - - -class BadOptionUsage(UsageError): - """Raised if an option is generally supplied but the use of the option - was incorrect. This is for instance raised if the number of arguments - for an option is not correct. - - .. versionadded:: 4.0 - - :param option_name: the name of the option being used incorrectly. - """ - - def __init__( - self, option_name: str, message: str, ctx: Context | None = None - ) -> None: - super().__init__(message, ctx) - self.option_name = option_name - - -class BadArgumentUsage(UsageError): - """Raised if an argument is generally supplied but the use of the argument - was incorrect. This is for instance raised if the number of values - for an argument is not correct. - - .. versionadded:: 6.0 - """ - - -class NoArgsIsHelpError(UsageError): - def __init__(self, ctx: Context) -> None: - self.ctx: Context - super().__init__(ctx.get_help(), ctx=ctx) - - def show(self, file: t.IO[t.Any] | None = None) -> None: - echo(self.format_message(), file=file, err=True, color=self.ctx.color) - - -class FileError(ClickException): - """Raised if a file cannot be opened.""" - - def __init__(self, filename: str, hint: str | None = None) -> None: - if hint is None: - hint = _("unknown error") - - super().__init__(hint) - self.ui_filename: str = format_filename(filename) - self.filename = filename - - def format_message(self) -> str: - return _("Could not open file {filename!r}: {message}").format( - filename=self.ui_filename, message=self.message - ) - - -class Abort(RuntimeError): - """An internal signalling exception that signals Click to abort.""" - - -class Exit(RuntimeError): - """An exception that indicates that the application should exit with some - status code. - - :param code: the status code to exit with. - """ - - __slots__ = ("exit_code",) - - def __init__(self, code: int = 0) -> None: - self.exit_code: int = code diff --git a/functions/python/data-processor/click/formatting.py b/functions/python/data-processor/click/formatting.py deleted file mode 100644 index 886a4ada7..000000000 --- a/functions/python/data-processor/click/formatting.py +++ /dev/null @@ -1,315 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -from contextlib import contextmanager -from gettext import gettext as _ - -from ._compat import term_len -from .parser import _split_opt - -# Can force a width. This is used by the test system -FORCED_WIDTH: int | None = None - - -def measure_table(rows: cabc.Iterable[tuple[str, str]]) -> tuple[int, ...]: - widths: dict[int, int] = {} - - for row in rows: - for idx, col in enumerate(row): - widths[idx] = max(widths.get(idx, 0), term_len(col)) - - return tuple(y for x, y in sorted(widths.items())) - - -def iter_rows( - rows: cabc.Iterable[tuple[str, str]], col_count: int -) -> cabc.Iterator[tuple[str, ...]]: - for row in rows: - yield row + ("",) * (col_count - len(row)) - - -def wrap_text( - text: str, - width: int = 78, - initial_indent: str = "", - subsequent_indent: str = "", - preserve_paragraphs: bool = False, -) -> str: - """A helper function that intelligently wraps text. By default, it - assumes that it operates on a single paragraph of text but if the - `preserve_paragraphs` parameter is provided it will intelligently - handle paragraphs (defined by two empty lines). - - If paragraphs are handled, a paragraph can be prefixed with an empty - line containing the ``\\b`` character (``\\x08``) to indicate that - no rewrapping should happen in that block. - - :param text: the text that should be rewrapped. - :param width: the maximum width for the text. - :param initial_indent: the initial indent that should be placed on the - first line as a string. - :param subsequent_indent: the indent string that should be placed on - each consecutive line. - :param preserve_paragraphs: if this flag is set then the wrapping will - intelligently handle paragraphs. - - .. versionchanged:: 8.4.0 - Width is measured in visible characters. ANSI escape sequences in - ``text``, ``initial_indent``, or ``subsequent_indent`` no longer - count toward the width budget, so styled input wraps based on what - the user sees instead of raw byte length. - """ - from ._textwrap import TextWrapper - - text = text.expandtabs() - wrapper = TextWrapper( - width, - initial_indent=initial_indent, - subsequent_indent=subsequent_indent, - replace_whitespace=False, - ) - if not preserve_paragraphs: - return wrapper.fill(text) - - p: list[tuple[int, bool, str]] = [] - buf: list[str] = [] - indent = None - - def _flush_par() -> None: - if not buf: - return - if buf[0].strip() == "\b": - p.append((indent or 0, True, "\n".join(buf[1:]))) - else: - p.append((indent or 0, False, " ".join(buf))) - del buf[:] - - for line in text.splitlines(): - if not line: - _flush_par() - indent = None - else: - if indent is None: - orig_len = term_len(line) - line = line.lstrip() - indent = orig_len - term_len(line) - buf.append(line) - _flush_par() - - rv = [] - for indent, raw, text in p: - with wrapper.extra_indent(" " * indent): - if raw: - rv.append(wrapper.indent_only(text)) - else: - rv.append(wrapper.fill(text)) - - return "\n\n".join(rv) - - -class HelpFormatter: - """This class helps with formatting text-based help pages. It's - usually just needed for very special internal cases, but it's also - exposed so that developers can write their own fancy outputs. - - At present, it always writes into memory. - - :param indent_increment: the additional increment for each level. - :param width: the width for the text. This defaults to the terminal - width clamped to a maximum of 78. - """ - - def __init__( - self, - indent_increment: int = 2, - width: int | None = None, - max_width: int | None = None, - ) -> None: - self.indent_increment = indent_increment - if max_width is None: - max_width = 80 - if width is None: - import shutil - - width = FORCED_WIDTH - if width is None: - width = max(min(shutil.get_terminal_size().columns, max_width) - 2, 50) - self.width = width - self.current_indent: int = 0 - self.buffer: list[str] = [] - - def write(self, string: str) -> None: - """Writes a unicode string into the internal buffer.""" - self.buffer.append(string) - - def indent(self) -> None: - """Increases the indentation.""" - self.current_indent += self.indent_increment - - def dedent(self) -> None: - """Decreases the indentation.""" - self.current_indent -= self.indent_increment - - def write_usage(self, prog: str, args: str = "", prefix: str | None = None) -> None: - """Writes a usage line into the buffer. - - :param prog: the program name. - :param args: whitespace separated list of arguments. - :param prefix: The prefix for the first line. Defaults to - ``"Usage: "``. - """ - if prefix is None: - prefix = "{usage} ".format(usage=_("Usage:")) - - usage_prefix = f"{prefix:>{self.current_indent}}{prog} " - text_width = self.width - self.current_indent - - if not args: - # Without args, the prefix's trailing space and the wrap_text - # call that would normally place args on the line are both - # unnecessary. Emit just the prefix line. - self.write(usage_prefix.rstrip(" ")) - self.write("\n") - return - - if text_width >= (term_len(usage_prefix) + 20): - # The arguments will fit to the right of the prefix. - indent = " " * term_len(usage_prefix) - self.write( - wrap_text( - args, - text_width, - initial_indent=usage_prefix, - subsequent_indent=indent, - ) - ) - else: - # The prefix is too long, put the arguments on the next line. - self.write(usage_prefix) - self.write("\n") - indent = " " * (max(self.current_indent, term_len(prefix)) + 4) - self.write( - wrap_text( - args, text_width, initial_indent=indent, subsequent_indent=indent - ) - ) - - self.write("\n") - - def write_heading(self, heading: str) -> None: - """Writes a heading into the buffer.""" - self.write(f"{'':>{self.current_indent}}{heading}:\n") - - def write_paragraph(self) -> None: - """Writes a paragraph into the buffer.""" - if self.buffer: - self.write("\n") - - def write_text(self, text: str) -> None: - """Writes re-indented text into the buffer. This rewraps and - preserves paragraphs. - """ - indent = " " * self.current_indent - self.write( - wrap_text( - text, - self.width, - initial_indent=indent, - subsequent_indent=indent, - preserve_paragraphs=True, - ) - ) - self.write("\n") - - def write_dl( - self, - rows: cabc.Sequence[tuple[str, str]], - col_max: int = 30, - col_spacing: int = 2, - ) -> None: - """Writes a definition list into the buffer. This is how options - and commands are usually formatted. - - :param rows: a list of two item tuples for the terms and values. - :param col_max: the maximum width of the first column. - :param col_spacing: the number of spaces between the first and - second column. - """ - rows = list(rows) - widths = measure_table(rows) - if len(widths) != 2: - raise TypeError("Expected two columns for definition list") - - first_col = min(widths[0], col_max) + col_spacing - - for first, second in iter_rows(rows, len(widths)): - self.write(f"{'':>{self.current_indent}}{first}") - if not second: - self.write("\n") - continue - if term_len(first) <= first_col - col_spacing: - self.write(" " * (first_col - term_len(first))) - else: - self.write("\n") - self.write(" " * (first_col + self.current_indent)) - - text_width = max(self.width - first_col - 2, 10) - wrapped_text = wrap_text(second, text_width, preserve_paragraphs=True) - lines = wrapped_text.splitlines() - - if lines: - self.write(f"{lines[0]}\n") - - for line in lines[1:]: - self.write(f"{'':>{first_col + self.current_indent}}{line}\n") - else: - self.write("\n") - - @contextmanager - def section(self, name: str) -> cabc.Iterator[None]: - """Helpful context manager that writes a paragraph, a heading, - and the indents. - - :param name: the section name that is written as heading. - """ - self.write_paragraph() - self.write_heading(name) - self.indent() - try: - yield - finally: - self.dedent() - - @contextmanager - def indentation(self) -> cabc.Iterator[None]: - """A context manager that increases the indentation.""" - self.indent() - try: - yield - finally: - self.dedent() - - def getvalue(self) -> str: - """Returns the buffer contents.""" - return "".join(self.buffer) - - -def join_options(options: cabc.Sequence[str]) -> tuple[str, bool]: - """Given a list of option strings this joins them in the most appropriate - way and returns them in the form ``(formatted_string, - any_prefix_is_slash)`` where the second item in the tuple is a flag that - indicates if any of the option prefixes was a slash. - """ - rv = [] - any_prefix_is_slash = False - - for opt in options: - prefix = _split_opt(opt)[0] - - if prefix == "/": - any_prefix_is_slash = True - - rv.append((len(prefix), opt)) - - rv.sort(key=lambda x: x[0]) - return ", ".join(x[1] for x in rv), any_prefix_is_slash diff --git a/functions/python/data-processor/click/globals.py b/functions/python/data-processor/click/globals.py deleted file mode 100644 index a2f91723d..000000000 --- a/functions/python/data-processor/click/globals.py +++ /dev/null @@ -1,67 +0,0 @@ -from __future__ import annotations - -import typing as t -from threading import local - -if t.TYPE_CHECKING: - from .core import Context - -_local = local() - - -@t.overload -def get_current_context(silent: t.Literal[False] = False) -> Context: ... - - -@t.overload -def get_current_context(silent: bool = ...) -> Context | None: ... - - -def get_current_context(silent: bool = False) -> Context | None: - """Returns the current click context. This can be used as a way to - access the current context object from anywhere. This is a more implicit - alternative to the :func:`pass_context` decorator. This function is - primarily useful for helpers such as :func:`echo` which might be - interested in changing its behavior based on the current context. - - To push the current context, :meth:`Context.scope` can be used. - - .. versionadded:: 5.0 - - :param silent: if set to `True` the return value is `None` if no context - is available. The default behavior is to raise a - :exc:`RuntimeError`. - """ - try: - return t.cast("Context", _local.stack[-1]) - except (AttributeError, IndexError) as e: - if not silent: - raise RuntimeError("There is no active click context.") from e - - return None - - -def push_context(ctx: Context) -> None: - """Pushes a new context to the current stack.""" - _local.__dict__.setdefault("stack", []).append(ctx) - - -def pop_context() -> None: - """Removes the top level from the stack.""" - _local.stack.pop() - - -def resolve_color_default(color: bool | None = None) -> bool | None: - """Internal helper to get the default value of the color flag. If a - value is passed it's returned unchanged, otherwise it's looked up from - the current context. - """ - if color is not None: - return color - - ctx = get_current_context(silent=True) - - if ctx is not None: - return ctx.color - - return None diff --git a/functions/python/data-processor/click/parser.py b/functions/python/data-processor/click/parser.py deleted file mode 100644 index 4fcbf7caa..000000000 --- a/functions/python/data-processor/click/parser.py +++ /dev/null @@ -1,533 +0,0 @@ -""" -This module started out as largely a copy paste from the stdlib's -optparse module with the features removed that we do not need from -optparse because we implement them in Click on a higher level (for -instance type handling, help formatting and a lot more). - -The plan is to remove more and more from here over time. - -The reason this is a different module and not optparse from the stdlib -is that there are differences in 2.x and 3.x about the error messages -generated and optparse in the stdlib uses gettext for no good reason -and might cause us issues. - -Click uses parts of optparse written by Gregory P. Ward and maintained -by the Python Software Foundation. This is limited to code in parser.py. - -Copyright 2001-2006 Gregory P. Ward. All rights reserved. -Copyright 2002-2006 Python Software Foundation. All rights reserved. -""" - -# This code uses parts of optparse written by Gregory P. Ward and -# maintained by the Python Software Foundation. -# Copyright 2001-2006 Gregory P. Ward -# Copyright 2002-2006 Python Software Foundation -from __future__ import annotations - -import collections.abc as cabc -import typing as t -from collections import deque -from gettext import gettext as _ -from gettext import ngettext - -from ._utils import FLAG_NEEDS_VALUE -from ._utils import UNSET -from .exceptions import BadArgumentUsage -from .exceptions import BadOptionUsage -from .exceptions import NoSuchOption -from .exceptions import UsageError - -if t.TYPE_CHECKING: - from ._utils import T_FLAG_NEEDS_VALUE - from ._utils import T_UNSET - from .core import Argument as CoreArgument - from .core import Context - from .core import Option as CoreOption - from .core import Parameter as CoreParameter - -V = t.TypeVar("V") - - -def _unpack_args( - args: cabc.Sequence[str], nargs_spec: cabc.Sequence[int] -) -> tuple[cabc.Sequence[str | cabc.Sequence[str | T_UNSET] | T_UNSET], list[str]]: - """Given an iterable of arguments and an iterable of nargs specifications, - it returns a tuple with all the unpacked arguments at the first index - and all remaining arguments as the second. - - The nargs specification is the number of arguments that should be consumed - or `-1` to indicate that this position should eat up all the remainders. - - Missing items are filled with ``UNSET``. - """ - args = deque(args) - nargs_spec = deque(nargs_spec) - rv: list[str | tuple[str | T_UNSET, ...] | T_UNSET] = [] - spos: int | None = None - - def _fetch(c: deque[str]) -> str | T_UNSET: - try: - if spos is None: - return c.popleft() - else: - return c.pop() - except IndexError: - return UNSET - - while nargs_spec: - if spos is None: - nargs = nargs_spec.popleft() - else: - nargs = nargs_spec.pop() - - if nargs == 1: - rv.append(_fetch(args)) - elif nargs > 1: - x: list[str | T_UNSET] = [_fetch(args) for _ in range(nargs)] - - # If we're reversed, we're pulling in the arguments in reverse, - # so we need to turn them around. - if spos is not None: - x.reverse() - - rv.append(tuple(x)) - elif nargs < 0: - if spos is not None: - raise TypeError("Cannot have two nargs < 0") - - spos = len(rv) - rv.append(UNSET) - - # spos is the position of the wildcard (star). If it's not `None`, - # we fill it with the remainder. - if spos is not None: - rv[spos] = tuple(args) - args = [] - rv[spos + 1 :] = reversed(rv[spos + 1 :]) - - return tuple(rv), list(args) - - -def _split_opt(opt: str) -> tuple[str, str]: - first = opt[:1] - if first.isalnum(): - return "", opt - if opt[1:2] == first: - return opt[:2], opt[2:] - return first, opt[1:] - - -def _normalize_opt(opt: str, ctx: Context | None) -> str: - if ctx is None or ctx.token_normalize_func is None: - return opt - prefix, opt = _split_opt(opt) - return f"{prefix}{ctx.token_normalize_func(opt)}" - - -class _Option: - def __init__( - self, - obj: CoreOption, - opts: cabc.Sequence[str], - dest: str | None, - action: str | None = None, - nargs: int = 1, - const: t.Any | None = None, - ): - self._short_opts = [] - self._long_opts = [] - self.prefixes: set[str] = set() - - for opt in opts: - prefix, value = _split_opt(opt) - if not prefix: - raise ValueError( - _("Invalid start character for option ({option})").format( - option=opt - ) - ) - self.prefixes.add(prefix[0]) - if len(prefix) == 1 and len(value) == 1: - self._short_opts.append(opt) - else: - self._long_opts.append(opt) - self.prefixes.add(prefix) - - if action is None: - action = "store" - - self.dest = dest - self.action = action - self.nargs = nargs - self.const = const - self.obj = obj - - @property - def takes_value(self) -> bool: - return self.action in ("store", "append") - - def process(self, value: t.Any, state: _ParsingState) -> None: - if self.action == "store": - state.opts[self.dest] = value # type: ignore - elif self.action == "store_const": - state.opts[self.dest] = self.const # type: ignore - elif self.action == "append": - state.opts.setdefault(self.dest, []).append(value) # type: ignore - elif self.action == "append_const": - state.opts.setdefault(self.dest, []).append(self.const) # type: ignore - elif self.action == "count": - state.opts[self.dest] = state.opts.get(self.dest, 0) + 1 # type: ignore - else: - raise ValueError(f"unknown action '{self.action}'") - state.order.append(self.obj) - - -class _Argument: - def __init__(self, obj: CoreArgument, dest: str | None, nargs: int = 1): - self.dest = dest - self.nargs = nargs - self.obj = obj - - def process( - self, - value: str | cabc.Sequence[str | T_UNSET] | T_UNSET, - state: _ParsingState, - ) -> None: - if self.nargs > 1: - assert isinstance(value, cabc.Sequence) - holes = sum(x is UNSET for x in value) - if holes == len(value): - value = UNSET - elif holes != 0: - raise BadArgumentUsage( - _("Argument {name!r} takes {nargs} values.").format( - name=self.dest, nargs=self.nargs - ) - ) - - # We failed to collect any argument value so we consider the argument as unset. - if value == (): - value = UNSET - - state.opts[self.dest] = value # type: ignore - state.order.append(self.obj) - - -class _ParsingState: - def __init__(self, rargs: list[str]) -> None: - self.opts: dict[str, t.Any] = {} - self.largs: list[str] = [] - self.rargs = rargs - self.order: list[CoreParameter] = [] - - -class _OptionParser: - """The option parser is an internal class that is ultimately used to - parse options and arguments. It's modelled after optparse and brings - a similar but vastly simplified API. It should generally not be used - directly as the high level Click classes wrap it for you. - - It's not nearly as extensible as optparse or argparse as it does not - implement features that are implemented on a higher level (such as - types or defaults). - - :param ctx: optionally the :class:`~click.Context` where this parser - should go with. - - .. deprecated:: 8.2 - Will be removed in Click 9.0. - """ - - def __init__(self, ctx: Context | None = None) -> None: - #: The :class:`~click.Context` for this parser. This might be - #: `None` for some advanced use cases. - self.ctx = ctx - #: This controls how the parser deals with interspersed arguments. - #: If this is set to `False`, the parser will stop on the first - #: non-option. Click uses this to implement nested subcommands - #: safely. - self.allow_interspersed_args: bool = True - #: This tells the parser how to deal with unknown options. By - #: default it will error out (which is sensible), but there is a - #: second mode where it will ignore it and continue processing - #: after shifting all the unknown options into the resulting args. - self.ignore_unknown_options: bool = False - - if ctx is not None: - self.allow_interspersed_args = ctx.allow_interspersed_args - self.ignore_unknown_options = ctx.ignore_unknown_options - - self._short_opt: dict[str, _Option] = {} - self._long_opt: dict[str, _Option] = {} - self._opt_prefixes = {"-", "--"} - self._args: list[_Argument] = [] - - def add_option( - self, - obj: CoreOption, - opts: cabc.Sequence[str], - dest: str | None, - action: str | None = None, - nargs: int = 1, - const: t.Any | None = None, - ) -> None: - """Adds a new option named `dest` to the parser. The destination - is not inferred (unlike with optparse) and needs to be explicitly - provided. Action can be any of ``store``, ``store_const``, - ``append``, ``append_const`` or ``count``. - - The `obj` can be used to identify the option in the order list - that is returned from the parser. - """ - opts = [_normalize_opt(opt, self.ctx) for opt in opts] - option = _Option(obj, opts, dest, action=action, nargs=nargs, const=const) - self._opt_prefixes.update(option.prefixes) - for opt in option._short_opts: - self._short_opt[opt] = option - for opt in option._long_opts: - self._long_opt[opt] = option - - def add_argument(self, obj: CoreArgument, dest: str | None, nargs: int = 1) -> None: - """Adds a positional argument named `dest` to the parser. - - The `obj` can be used to identify the option in the order list - that is returned from the parser. - """ - self._args.append(_Argument(obj, dest=dest, nargs=nargs)) - - def parse_args( - self, args: list[str] - ) -> tuple[dict[str, t.Any], list[str], list[CoreParameter]]: - """Parses positional arguments and returns ``(values, args, order)`` - for the parsed options and arguments as well as the leftover - arguments if there are any. The order is a list of objects as they - appear on the command line. If arguments appear multiple times they - will be memorized multiple times as well. - """ - state = _ParsingState(args) - try: - self._process_args_for_options(state) - self._process_args_for_args(state) - except UsageError: - if self.ctx is None or not self.ctx.resilient_parsing: - raise - return state.opts, state.largs, state.order - - def _process_args_for_args(self, state: _ParsingState) -> None: - pargs, args = _unpack_args( - state.largs + state.rargs, [x.nargs for x in self._args] - ) - - for idx, arg in enumerate(self._args): - arg.process(pargs[idx], state) - - state.largs = args - state.rargs = [] - - def _process_args_for_options(self, state: _ParsingState) -> None: - while state.rargs: - arg = state.rargs.pop(0) - arglen = len(arg) - # Double dashes always handled explicitly regardless of what - # prefixes are valid. - if arg == "--": - return - elif arg[:1] in self._opt_prefixes and arglen > 1: - self._process_opts(arg, state) - elif self.allow_interspersed_args: - state.largs.append(arg) - else: - state.rargs.insert(0, arg) - return - - # Say this is the original argument list: - # [arg0, arg1, ..., arg(i-1), arg(i), arg(i+1), ..., arg(N-1)] - # ^ - # (we are about to process arg(i)). - # - # Then rargs is [arg(i), ..., arg(N-1)] and largs is a *subset* of - # [arg0, ..., arg(i-1)] (any options and their arguments will have - # been removed from largs). - # - # The while loop will usually consume 1 or more arguments per pass. - # If it consumes 1 (eg. arg is an option that takes no arguments), - # then after _process_arg() is done the situation is: - # - # largs = subset of [arg0, ..., arg(i)] - # rargs = [arg(i+1), ..., arg(N-1)] - # - # If allow_interspersed_args is false, largs will always be - # *empty* -- still a subset of [arg0, ..., arg(i-1)], but - # not a very interesting subset! - - def _match_long_opt( - self, opt: str, explicit_value: str | None, state: _ParsingState - ) -> None: - if opt not in self._long_opt: - raise NoSuchOption(opt, possibilities=self._long_opt, ctx=self.ctx) - - option = self._long_opt[opt] - if option.takes_value: - # At this point it's safe to modify rargs by injecting the - # explicit value, because no exception is raised in this - # branch. This means that the inserted value will be fully - # consumed. - if explicit_value is not None: - state.rargs.insert(0, explicit_value) - - value = self._get_value_from_state(opt, option, state) - - elif explicit_value is not None: - raise BadOptionUsage( - opt, _("Option {name!r} does not take a value.").format(name=opt) - ) - - else: - value = UNSET - - option.process(value, state) - - def _match_short_opt(self, arg: str, state: _ParsingState) -> None: - stop = False - i = 1 - prefix = arg[0] - unknown_options = [] - - for ch in arg[1:]: - opt = _normalize_opt(f"{prefix}{ch}", self.ctx) - option = self._short_opt.get(opt) - i += 1 - - if not option: - if self.ignore_unknown_options: - unknown_options.append(ch) - continue - raise NoSuchOption(opt, ctx=self.ctx) - if option.takes_value: - # Any characters left in arg? Pretend they're the - # next arg, and stop consuming characters of arg. - if i < len(arg): - state.rargs.insert(0, arg[i:]) - stop = True - - value = self._get_value_from_state(opt, option, state) - - else: - value = UNSET - - option.process(value, state) - - if stop: - break - - # If we got any unknown options we recombine the string of the - # remaining options and re-attach the prefix, then report that - # to the state as new large. This way there is basic combinatorics - # that can be achieved while still ignoring unknown arguments. - if self.ignore_unknown_options and unknown_options: - state.largs.append(f"{prefix}{''.join(unknown_options)}") - - def _get_value_from_state( - self, option_name: str, option: _Option, state: _ParsingState - ) -> str | cabc.Sequence[str] | T_UNSET | T_FLAG_NEEDS_VALUE: - nargs = option.nargs - - value: str | cabc.Sequence[str] | T_UNSET | T_FLAG_NEEDS_VALUE - - if len(state.rargs) < nargs: - if option.obj._flag_needs_value: - # Option allows omitting the value. - value = FLAG_NEEDS_VALUE - else: - raise BadOptionUsage( - option_name, - ngettext( - "Option {name!r} requires an argument.", - "Option {name!r} requires {nargs} arguments.", - nargs, - ).format(name=option_name, nargs=nargs), - ) - elif nargs == 1: - next_rarg = state.rargs[0] - - if ( - option.obj._flag_needs_value - and isinstance(next_rarg, str) - and next_rarg[:1] in self._opt_prefixes - and len(next_rarg) > 1 - ): - # The next arg looks like the start of an option, don't - # use it as the value if omitting the value is allowed. - value = FLAG_NEEDS_VALUE - else: - value = state.rargs.pop(0) - else: - value = tuple(state.rargs[:nargs]) - del state.rargs[:nargs] - - return value - - def _process_opts(self, arg: str, state: _ParsingState) -> None: - explicit_value = None - # Long option handling happens in two parts. The first part is - # supporting explicitly attached values. In any case, we will try - # to long match the option first. - if "=" in arg: - long_opt, explicit_value = arg.split("=", 1) - else: - long_opt = arg - norm_long_opt = _normalize_opt(long_opt, self.ctx) - - # At this point we will match the (assumed) long option through - # the long option matching code. Note that this allows options - # like "-foo" to be matched as long options. - try: - self._match_long_opt(norm_long_opt, explicit_value, state) - except NoSuchOption: - # At this point the long option matching failed, and we need - # to try with short options. However there is a special rule - # which says, that if we have a two character options prefix - # (applies to "--foo" for instance), we do not dispatch to the - # short option code and will instead raise the no option - # error. - if arg[:2] not in self._opt_prefixes: - self._match_short_opt(arg, state) - return - - if not self.ignore_unknown_options: - raise - - state.largs.append(arg) - - -def __getattr__(name: str) -> object: - import warnings - - if name in { - "OptionParser", - "Argument", - "Option", - "split_opt", - "normalize_opt", - "ParsingState", - }: - warnings.warn( - f"'parser.{name}' is deprecated and will be removed in Click 9.0." - " The old parser is available in 'optparse'.", - DeprecationWarning, - stacklevel=2, - ) - return globals()[f"_{name}"] - - if name == "split_arg_string": - from .shell_completion import split_arg_string - - warnings.warn( - "Importing 'parser.split_arg_string' is deprecated, it will only be" - " available in 'shell_completion' in Click 9.0.", - DeprecationWarning, - stacklevel=2, - ) - return split_arg_string - - raise AttributeError(name) diff --git a/functions/python/data-processor/click/py.typed b/functions/python/data-processor/click/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/click/shell_completion.py b/functions/python/data-processor/click/shell_completion.py deleted file mode 100644 index b01e290a9..000000000 --- a/functions/python/data-processor/click/shell_completion.py +++ /dev/null @@ -1,680 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import os -import re -import typing as t -from gettext import gettext as _ - -from .core import Argument -from .core import Command -from .core import Context -from .core import Group -from .core import Option -from .core import Parameter -from .core import ParameterSource -from .utils import echo - - -def shell_complete( - cli: Command, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - complete_var: str, - instruction: str, -) -> int: - """Perform shell completion for the given CLI program. - - :param cli: Command being called. - :param ctx_args: Extra arguments to pass to - ``cli.make_context``. - :param prog_name: Name of the executable in the shell. - :param complete_var: Name of the environment variable that holds - the completion instruction. - :param instruction: Value of ``complete_var`` with the completion - instruction and shell, in the form ``instruction_shell``. - :return: Status code to exit with. - """ - shell, _, instruction = instruction.partition("_") - comp_cls = get_completion_class(shell) - - if comp_cls is None: - return 1 - - comp = comp_cls(cli, ctx_args, prog_name, complete_var) - - # Write bytes, otherwise Windows text stdout translates LF to CRLF and breaks. - if instruction == "source": - echo(comp.source().encode(), nl=False) - return 0 - - if instruction == "complete": - echo(comp.complete().encode()) - return 0 - - return 1 - - -class CompletionItem: - """Represents a completion value and metadata about the value. The - default metadata is ``type`` to indicate special shell handling, - and ``help`` if a shell supports showing a help string next to the - value. - - Arbitrary parameters can be passed when creating the object, and - accessed using ``item.attr``. If an attribute wasn't passed, - accessing it returns ``None``. - - :param value: The completion suggestion. - :param type: Tells the shell script to provide special completion - support for the type. Click uses ``"dir"`` and ``"file"``. - :param help: String shown next to the value if supported. - :param kwargs: Arbitrary metadata. The built-in implementations - don't use this, but custom type completions paired with custom - shell support could use it. - """ - - __slots__ = ("value", "type", "help", "_info") - - def __init__( - self, - value: t.Any, - type: str = "plain", - help: str | None = None, - **kwargs: t.Any, - ) -> None: - self.value: t.Any = value - self.type: str = type - self.help: str | None = help - self._info = kwargs - - def __getattr__(self, name: str) -> t.Any: - return self._info.get(name) - - -# Only Bash >= 4.4 has the nosort option. -_SOURCE_BASH = """\ -%(complete_func)s() { - local IFS=$'\\n' - local response - - response=$(env COMP_WORDS="${COMP_WORDS[*]}" COMP_CWORD=$COMP_CWORD \ -%(complete_var)s=bash_complete $1) - - for completion in $response; do - IFS=',' read type value <<< "$completion" - - if [[ $type == 'dir' ]]; then - COMPREPLY=() - compopt -o dirnames - elif [[ $type == 'file' ]]; then - COMPREPLY=() - compopt -o default - elif [[ $type == 'plain' ]]; then - COMPREPLY+=($value) - fi - done - - return 0 -} - -%(complete_func)s_setup() { - complete -o nosort -F %(complete_func)s %(prog_name)s -} - -%(complete_func)s_setup; -""" - -# See ZshComplete.format_completion below, and issue #2703, before -# changing this script. -# -# (TL;DR: _describe is picky about the format, but this Zsh script snippet -# is already widely deployed. So freeze this script, and use clever-ish -# handling of colons in ZshComplet.format_completion.) -_SOURCE_ZSH = """\ -#compdef %(prog_name)s - -%(complete_func)s() { - local -a completions - local -a completions_with_descriptions - local -a response - (( ! $+commands[%(prog_name)s] )) && return 1 - - response=("${(@f)$(env COMP_WORDS="${words[*]}" COMP_CWORD=$((CURRENT-1)) \ -%(complete_var)s=zsh_complete %(prog_name)s)}") - - for type key descr in ${response}; do - if [[ "$type" == "plain" ]]; then - if [[ "$descr" == "_" ]]; then - completions+=("$key") - else - completions_with_descriptions+=("$key":"$descr") - fi - elif [[ "$type" == "dir" ]]; then - _path_files -/ - elif [[ "$type" == "file" ]]; then - _path_files -f - fi - done - - if [ -n "$completions_with_descriptions" ]; then - _describe -V unsorted completions_with_descriptions -U - fi - - if [ -n "$completions" ]; then - compadd -U -V unsorted -a completions - fi -} - -if [[ $zsh_eval_context[-1] == loadautofunc ]]; then - # autoload from fpath, call function directly - %(complete_func)s "$@" -else - # eval/source/. command, register function for later - compdef %(complete_func)s %(prog_name)s -fi -""" - -_SOURCE_FISH = """\ -function %(complete_func)s; - set -l response (env %(complete_var)s=fish_complete COMP_WORDS=(commandline -cp) \ -COMP_CWORD=(commandline -t) %(prog_name)s); - - for completion in $response; - set -l metadata (string split \n $completion); - - if test $metadata[1] = "dir"; - __fish_complete_directories $metadata[2]; - else if test $metadata[1] = "file"; - __fish_complete_path $metadata[2]; - else if test $metadata[1] = "plain"; - if test $metadata[3] != "_"; - echo $metadata[2]\t$metadata[3]; - else; - echo $metadata[2]; - end; - end; - end; -end; - -complete --no-files --command %(prog_name)s --arguments \ -"(%(complete_func)s)"; -""" - - -class ShellComplete: - """Base class for providing shell completion support. A subclass for - a given shell will override attributes and methods to implement the - completion instructions (``source`` and ``complete``). - - :param cli: Command being called. - :param prog_name: Name of the executable in the shell. - :param complete_var: Name of the environment variable that holds - the completion instruction. - - .. versionadded:: 8.0 - """ - - name: t.ClassVar[str] - """Name to register the shell as with :func:`add_completion_class`. - This is used in completion instructions (``{name}_source`` and - ``{name}_complete``). - """ - - source_template: t.ClassVar[str] - """Completion script template formatted by :meth:`source`. This must - be provided by subclasses. - """ - - def __init__( - self, - cli: Command, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - complete_var: str, - ) -> None: - self.cli = cli - self.ctx_args = ctx_args - self.prog_name = prog_name - self.complete_var = complete_var - - @property - def func_name(self) -> str: - """The name of the shell function defined by the completion - script. - """ - safe_name = re.sub(r"\W*", "", self.prog_name.replace("-", "_"), flags=re.ASCII) - return f"_{safe_name}_completion" - - def source_vars(self) -> dict[str, t.Any]: - """Vars for formatting :attr:`source_template`. - - By default this provides ``complete_func``, ``complete_var``, - and ``prog_name``. - """ - return { - "complete_func": self.func_name, - "complete_var": self.complete_var, - "prog_name": self.prog_name, - } - - def source(self) -> str: - """Produce the shell script that defines the completion - function. By default this ``%``-style formats - :attr:`source_template` with the dict returned by - :meth:`source_vars`. - """ - return self.source_template % self.source_vars() - - def get_completion_args(self) -> tuple[list[str], str]: - """Use the env vars defined by the shell script to return a - tuple of ``args, incomplete``. This must be implemented by - subclasses. - """ - raise NotImplementedError - - def get_completions(self, args: list[str], incomplete: str) -> list[CompletionItem]: - """Determine the context and last complete command or parameter - from the complete args. Call that object's ``shell_complete`` - method to get the completions for the incomplete value. - - :param args: List of complete args before the incomplete value. - :param incomplete: Value being completed. May be empty. - """ - ctx = _resolve_context(self.cli, self.ctx_args, self.prog_name, args) - obj, incomplete = _resolve_incomplete(ctx, args, incomplete) - return obj.shell_complete(ctx, incomplete) - - def format_completion(self, item: CompletionItem) -> str: - """Format a completion item into the form recognized by the - shell script. This must be implemented by subclasses. - - :param item: Completion item to format. - """ - raise NotImplementedError - - def complete(self) -> str: - """Produce the completion data to send back to the shell. - - By default this calls :meth:`get_completion_args`, gets the - completions, then calls :meth:`format_completion` for each - completion. - """ - args, incomplete = self.get_completion_args() - completions = self.get_completions(args, incomplete) - out = [self.format_completion(item) for item in completions] - return "\n".join(out) - - -class BashComplete(ShellComplete): - """Shell completion for Bash.""" - - name = "bash" - source_template = _SOURCE_BASH - - @staticmethod - def _check_version() -> None: - import shutil - import subprocess - - bash_exe = shutil.which("bash") - - if bash_exe is None: - match = None - else: - output = subprocess.run( - [bash_exe, "--norc", "-c", 'echo "${BASH_VERSION}"'], - stdout=subprocess.PIPE, - ) - match = re.search(r"^(\d+)\.(\d+)\.\d+", output.stdout.decode()) - - if match is not None: - major, minor = match.groups() - - if major < "4" or major == "4" and minor < "4": - echo( - _( - "Shell completion is not supported for Bash" - " versions older than 4.4." - ), - err=True, - ) - else: - echo( - _("Couldn't detect Bash version, shell completion is not supported."), - err=True, - ) - - def source(self) -> str: - self._check_version() - return super().source() - - def get_completion_args(self) -> tuple[list[str], str]: - cwords = split_arg_string(os.environ["COMP_WORDS"]) - cword = int(os.environ["COMP_CWORD"]) - args = cwords[1:cword] - - try: - incomplete = cwords[cword] - except IndexError: - incomplete = "" - - return args, incomplete - - def format_completion(self, item: CompletionItem) -> str: - return f"{item.type},{item.value}" - - -class ZshComplete(ShellComplete): - """Shell completion for Zsh.""" - - name = "zsh" - source_template = _SOURCE_ZSH - - def get_completion_args(self) -> tuple[list[str], str]: - cwords = split_arg_string(os.environ["COMP_WORDS"]) - cword = int(os.environ["COMP_CWORD"]) - args = cwords[1:cword] - - try: - incomplete = cwords[cword] - except IndexError: - incomplete = "" - - return args, incomplete - - def format_completion(self, item: CompletionItem) -> str: - help_ = item.help or "_" - # The zsh completion script uses `_describe` on items with help - # texts (which splits the item help from the item value at the - # first unescaped colon) and `compadd` on items without help - # text (which uses the item value as-is and does not support - # colon escaping). So escape colons in the item value if and - # only if the item help is not the sentinel "_" value, as used - # by the completion script. - # - # (The zsh completion script is potentially widely deployed, and - # thus harder to fix than this method.) - # - # See issue #1812 and issue #2703 for further context. - value = item.value.replace(":", r"\:") if help_ != "_" else item.value - return f"{item.type}\n{value}\n{help_}" - - -class FishComplete(ShellComplete): - """Shell completion for Fish.""" - - name = "fish" - source_template = _SOURCE_FISH - - def get_completion_args(self) -> tuple[list[str], str]: - cwords = split_arg_string(os.environ["COMP_WORDS"]) - incomplete = os.environ["COMP_CWORD"] - if incomplete: - incomplete = split_arg_string(incomplete)[0] - args = cwords[1:] - - # Fish stores the partial word in both COMP_WORDS and - # COMP_CWORD, remove it from complete args. - if incomplete and args and args[-1] == incomplete: - args.pop() - - return args, incomplete - - def format_completion(self, item: CompletionItem) -> str: - """ - .. versionchanged:: 8.4.0 - Escape newlines in value and help to fix completion errors with - multi-line help strings. - """ - # The fish completion script splits each response line on literal - # newlines, so any newline in the value or help would corrupt the - # frame. Replace them with the two-character escape "\n" so the text - # round-trips through fish without breaking the format. The "_" - # sentinel for missing help mirrors :class:`ZshComplete`. - help_ = item.help or "_" - value = item.value.replace("\n", r"\n") - help_escaped = help_.replace("\n", r"\n") - return f"{item.type}\n{value}\n{help_escaped}" - - -ShellCompleteType = t.TypeVar("ShellCompleteType", bound="type[ShellComplete]") - - -_available_shells: dict[str, type[ShellComplete]] = { - "bash": BashComplete, - "fish": FishComplete, - "zsh": ZshComplete, -} - - -def add_completion_class( - cls: ShellCompleteType, name: str | None = None -) -> ShellCompleteType: - """Register a :class:`ShellComplete` subclass under the given name. - The name will be provided by the completion instruction environment - variable during completion. - - :param cls: The completion class that will handle completion for the - shell. - :param name: Name to register the class under. Defaults to the - class's ``name`` attribute. - """ - if name is None: - name = cls.name - - _available_shells[name] = cls - - return cls - - -def get_completion_class(shell: str) -> type[ShellComplete] | None: - """Look up a registered :class:`ShellComplete` subclass by the name - provided by the completion instruction environment variable. If the - name isn't registered, returns ``None``. - - :param shell: Name the class is registered under. - """ - return _available_shells.get(shell) - - -def split_arg_string(string: str) -> list[str]: - """Split an argument string as with :func:`shlex.split`, but don't - fail if the string is incomplete. Ignores a missing closing quote or - incomplete escape sequence and uses the partial token as-is. - - .. code-block:: python - - split_arg_string("example 'my file") - ["example", "my file"] - - split_arg_string("example my\\") - ["example", "my"] - - :param string: String to split. - - .. versionchanged:: 8.2 - Moved to ``shell_completion`` from ``parser``. - """ - import shlex - - lex = shlex.shlex(string, posix=True) - lex.whitespace_split = True - lex.commenters = "" - out = [] - - try: - for token in lex: - out.append(token) - except ValueError: - # Raised when end-of-string is reached in an invalid state. Use - # the partial token as-is. The quote or escape character is in - # lex.state, not lex.token. - out.append(lex.token) - - return out - - -def _is_incomplete_argument(ctx: Context, param: Parameter) -> bool: - """Determine if the given parameter is an argument that can still - accept values. - - :param ctx: Invocation context for the command represented by the - parsed complete args. - :param param: Argument object being checked. - """ - if not isinstance(param, Argument): - return False - - value = ctx.params.get(param.name) - return ( - param.nargs == -1 - or ctx.get_parameter_source(param.name) is not ParameterSource.COMMANDLINE - or ( - param.nargs > 1 - and isinstance(value, (tuple, list)) - and len(value) < param.nargs - ) - ) - - -def _start_of_option(ctx: Context, value: str) -> bool: - """Check if the value looks like the start of an option.""" - if not value: - return False - - c = value[0] - return c in ctx._opt_prefixes - - -def _is_incomplete_option(ctx: Context, args: list[str], param: Parameter) -> bool: - """Determine if the given parameter is an option that needs a value. - - :param args: List of complete args before the incomplete value. - :param param: Option object being checked. - """ - if not isinstance(param, Option): - return False - - if param.is_flag or param.count: - return False - - last_option = None - - for index, arg in enumerate(reversed(args)): - if index + 1 > param.nargs: - break - - if _start_of_option(ctx, arg): - last_option = arg - break - - return last_option is not None and last_option in param.opts - - -def _resolve_context( - cli: Command, - ctx_args: cabc.MutableMapping[str, t.Any], - prog_name: str, - args: list[str], -) -> Context: - """Produce the context hierarchy starting with the command and - traversing the complete arguments. This only follows the commands, - it doesn't trigger input prompts or callbacks. - - :param cli: Command being called. - :param prog_name: Name of the executable in the shell. - :param args: List of complete args before the incomplete value. - """ - ctx_args["resilient_parsing"] = True - with cli.make_context(prog_name, args.copy(), **ctx_args) as ctx: - args = ctx._protected_args + ctx.args - - while args: - command = ctx.command - - if isinstance(command, Group): - if not command.chain: - name, cmd, args = command.resolve_command(ctx, args) - - if cmd is None: - return ctx - - with cmd.make_context( - name, args, parent=ctx, resilient_parsing=True - ) as sub_ctx: - ctx = sub_ctx - args = ctx._protected_args + ctx.args - else: - sub_ctx = ctx - - while args: - name, cmd, args = command.resolve_command(ctx, args) - - if cmd is None: - return ctx - - with cmd.make_context( - name, - args, - parent=ctx, - allow_extra_args=True, - allow_interspersed_args=False, - resilient_parsing=True, - ) as sub_sub_ctx: - sub_ctx = sub_sub_ctx - args = sub_ctx.args - - ctx = sub_ctx - args = [*sub_ctx._protected_args, *sub_ctx.args] - else: - break - - return ctx - - -def _resolve_incomplete( - ctx: Context, args: list[str], incomplete: str -) -> tuple[Command | Parameter, str]: - """Find the Click object that will handle the completion of the - incomplete value. Return the object and the incomplete value. - - :param ctx: Invocation context for the command represented by - the parsed complete args. - :param args: List of complete args before the incomplete value. - :param incomplete: Value being completed. May be empty. - """ - # Different shells treat an "=" between a long option name and - # value differently. Might keep the value joined, return the "=" - # as a separate item, or return the split name and value. Always - # split and discard the "=" to make completion easier. - if incomplete == "=": - incomplete = "" - elif "=" in incomplete and _start_of_option(ctx, incomplete): - name, _, incomplete = incomplete.partition("=") - args.append(name) - - # The "--" marker tells Click to stop treating values as options - # even if they start with the option character. If it hasn't been - # given and the incomplete arg looks like an option, the current - # command will provide option name completions. - if "--" not in args and _start_of_option(ctx, incomplete): - return ctx.command, incomplete - - params = ctx.command.get_params(ctx) - - # If the last complete arg is an option name with an incomplete - # value, the option will provide value completions. - for param in params: - if _is_incomplete_option(ctx, args, param): - return param, incomplete - - # It's not an option name or value. The first argument without a - # parsed value will provide value completions. - for param in params: - if _is_incomplete_argument(ctx, param): - return param, incomplete - - # There were no unparsed arguments, the command may be a group that - # will provide command name completions. - return ctx.command, incomplete diff --git a/functions/python/data-processor/click/termui.py b/functions/python/data-processor/click/termui.py deleted file mode 100644 index 91ef6e19b..000000000 --- a/functions/python/data-processor/click/termui.py +++ /dev/null @@ -1,941 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import inspect -import io -import itertools -import re -import sys -import typing as t -from contextlib import AbstractContextManager -from contextlib import redirect_stdout -from gettext import gettext as _ - -from ._compat import isatty -from ._compat import strip_ansi -from ._compat import WIN -from .exceptions import Abort -from .exceptions import UsageError -from .globals import resolve_color_default -from .types import Choice -from .types import convert_type -from .types import ParamType -from .utils import echo -from .utils import LazyFile - -if t.TYPE_CHECKING: - from ._termui_impl import ProgressBar - -V = t.TypeVar("V") - -# The prompt functions to use. The doc tools currently override these -# functions to customize how they work. -visible_prompt_func: t.Callable[[str], str] = input - -_ansi_colors = { - "black": 30, - "red": 31, - "green": 32, - "yellow": 33, - "blue": 34, - "magenta": 35, - "cyan": 36, - "white": 37, - "reset": 39, - "bright_black": 90, - "bright_red": 91, - "bright_green": 92, - "bright_yellow": 93, - "bright_blue": 94, - "bright_magenta": 95, - "bright_cyan": 96, - "bright_white": 97, -} -_ansi_reset_all = "\033[0m" - - -_HIDDEN_INPUT_MASK = "'***'" - - -def _mask_hidden_input(message: str, value: str) -> str: - """Replace occurrences of ``value`` in ``message`` with a fixed mask. - - Both ``repr(value)`` (the form built-in :class:`ParamType` errors use - via ``{value!r}``) and the raw value are masked. The raw-value pass - uses word-boundary lookarounds so a substring like ``"1"`` does not - match inside ``"10"``, and ``"ent"`` does not match inside - ``"Authentication"``. The empty string is skipped to avoid matching - at every boundary. - """ - message = message.replace(repr(value), _HIDDEN_INPUT_MASK) - if value: - message = re.sub( - rf"(? str: - import getpass - - return getpass.getpass(prompt) - - -def _readline_prompt(func: t.Callable[[str], str], text: str, err: bool) -> str: - """Call a prompt function, passing the full prompt on non-Windows so - readline can handle line editing and cursor positioning correctly. - - On Windows the prompt is written separately via :func:`echo` for - colorama support, with only the last character passed to *func*. - """ - if WIN: - # Write the prompt separately so that we get nice coloring - # through colorama on Windows. - echo(text[:-1], nl=False, err=err) - # Echo the last character to stdout to work around an issue - # where readline causes backspace to clear the whole line. - return func(text[-1:]) - if err: - with redirect_stdout(sys.stderr): - return func(text) - return func(text) - - -def _build_prompt( - text: str, - suffix: str, - show_default: bool | str = False, - default: t.Any | None = None, - show_choices: bool = True, - type: ParamType[t.Any] | None = None, -) -> str: - prompt = text - if type is not None and show_choices and isinstance(type, Choice): - prompt += f" ({', '.join(map(str, type.choices))})" - if isinstance(show_default, str): - default = f"({show_default})" - if default is not None and show_default: - prompt = f"{prompt} [{_format_default(default)}]" - return f"{prompt}{suffix}" - - -def _format_default(default: t.Any) -> t.Any: - if isinstance(default, (io.IOBase, LazyFile)) and hasattr(default, "name"): - return default.name - - return default - - -def prompt( - text: str, - default: t.Any | None = None, - hide_input: bool = False, - confirmation_prompt: bool | str = False, - type: ParamType[t.Any] | t.Any | None = None, - value_proc: t.Callable[[str], t.Any] | None = None, - prompt_suffix: str = ": ", - show_default: bool | str = True, - err: bool = False, - show_choices: bool = True, -) -> t.Any: - """Prompts a user for input. This is a convenience function that can - be used to prompt a user for input later. - - If the user aborts the input by sending an interrupt signal, this - function will catch it and raise a :exc:`Abort` exception. - - :param text: the text to show for the prompt. - :param default: the default value to use if no input happens. If this - is not given it will prompt until it's aborted. - :param hide_input: if this is set to true then the input value will - be hidden. - :param confirmation_prompt: Prompt a second time to confirm the - value. Can be set to a string instead of ``True`` to customize - the message. - :param type: the type to use to check the value against. - :param value_proc: if this parameter is provided it's a function that - is invoked instead of the type conversion to - convert a value. - :param prompt_suffix: a suffix that should be added to the prompt. - :param show_default: shows or hides the default value in the prompt. - If this value is a string, it shows that string - in parentheses instead of the actual value. - :param err: if set to true the file defaults to ``stderr`` instead of - ``stdout``, the same as with echo. - :param show_choices: Show or hide choices if the passed type is a Choice. - For example if type is a Choice of either day or week, - show_choices is true and text is "Group by" then the - prompt will be "Group by (day, week): ". - - .. versionchanged:: 8.3.3 - ``show_default`` can be a string to show a custom value instead - of the actual default, matching the help text behavior. - - .. versionchanged:: 8.3.1 - A space is no longer appended to the prompt. - - .. versionadded:: 8.0 - ``confirmation_prompt`` can be a custom string. - - .. versionadded:: 7.0 - Added the ``show_choices`` parameter. - - .. versionadded:: 6.0 - Added unicode support for cmd.exe on Windows. - - .. versionadded:: 4.0 - Added the `err` parameter. - - """ - - def prompt_func(text: str) -> str: - f = hidden_prompt_func if hide_input else visible_prompt_func - try: - return _readline_prompt(f, text, err) - except (KeyboardInterrupt, EOFError): - # getpass doesn't print a newline if the user aborts input with ^C. - # Allegedly this behavior is inherited from getpass(3). - # A doc bug has been filed at https://bugs.python.org/issue24711 - if hide_input: - echo(None, err=err) - raise Abort() from None - - if value_proc is None: - value_proc = convert_type(type, default) - - prompt = _build_prompt( - text, prompt_suffix, show_default, default, show_choices, type - ) - - if confirmation_prompt: - if confirmation_prompt is True: - confirmation_prompt = _("Repeat for confirmation") - - confirmation_prompt = _build_prompt(confirmation_prompt, prompt_suffix) - - while True: - while True: - value = prompt_func(prompt) - if value: - break - elif default is not None: - value = default - break - try: - result = value_proc(value) - except UsageError as e: - message = _mask_hidden_input(e.message, value) if hide_input else e.message - echo(_("Error: {message}").format(message=message), err=err) - continue - if not confirmation_prompt: - return result - while True: - value2 = prompt_func(confirmation_prompt) - is_empty = not value and not value2 - if value2 or is_empty: - break - if value == value2: - return result - echo(_("Error: The two entered values do not match."), err=err) - - -def confirm( - text: str, - default: bool | None = False, - abort: bool = False, - prompt_suffix: str = ": ", - show_default: bool = True, - err: bool = False, -) -> bool: - """Prompts for confirmation (yes/no question). - - If the user aborts the input by sending a interrupt signal this - function will catch it and raise a :exc:`Abort` exception. - - :param text: the question to ask. - :param default: The default value to use when no input is given. If - ``None``, repeat until input is given. - :param abort: if this is set to `True` a negative answer aborts the - exception by raising :exc:`Abort`. - :param prompt_suffix: a suffix that should be added to the prompt. - :param show_default: shows or hides the default value in the prompt. - :param err: if set to true the file defaults to ``stderr`` instead of - ``stdout``, the same as with echo. - - .. versionchanged:: 8.3.1 - A space is no longer appended to the prompt. - - .. versionchanged:: 8.0 - Repeat until input is given if ``default`` is ``None``. - - .. versionadded:: 4.0 - Added the ``err`` parameter. - """ - prompt = _build_prompt( - text, - prompt_suffix, - show_default, - "y/n" if default is None else ("Y/n" if default else "y/N"), - ) - - while True: - try: - value = _readline_prompt(visible_prompt_func, prompt, err).lower().strip() - except (KeyboardInterrupt, EOFError): - raise Abort() from None - if value in ("y", "yes"): - rv = True - elif value in ("n", "no"): - rv = False - elif default is not None and value == "": - rv = default - else: - echo(_("Error: invalid input"), err=err) - continue - break - if abort and not rv: - raise Abort() - return rv - - -def get_pager_file( - color: bool | None = None, -) -> t.ContextManager[t.TextIO]: - """Context manager. - - Yields a writable file-like object which can be used as an output pager. - - .. versionadded:: 8.4.0 - - :param color: controls if the pager supports ANSI colors or not. The - default is autodetection. - """ - from ._termui_impl import get_pager_file - - color = resolve_color_default(color) - - return get_pager_file(color=color) - - -def echo_via_pager( - text_or_generator: cabc.Iterable[str] | t.Callable[[], cabc.Iterable[str]] | str, - color: bool | None = None, -) -> None: - """This function takes a text and shows it via an environment specific - pager on stdout. - - .. versionchanged:: 3.0 - Added the `color` flag. - - :param text_or_generator: the text to page, or alternatively, a - generator emitting the text to page. - :param color: controls if the pager supports ANSI colors or not. The - default is autodetection. - """ - - if inspect.isgeneratorfunction(text_or_generator): - i = t.cast("t.Callable[[], cabc.Iterable[str]]", text_or_generator)() - elif isinstance(text_or_generator, str): - i = [text_or_generator] - else: - i = iter(t.cast("cabc.Iterable[str]", text_or_generator)) - - # convert every element of i to a text type if necessary - text_generator = (el if isinstance(el, str) else str(el) for el in i) - - with get_pager_file(color=color) as pager: - for text in itertools.chain(text_generator, "\n"): - pager.write(text) - - -@t.overload -def progressbar( - *, - length: int, - label: str | None = None, - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - fill_char: str = "#", - empty_char: str = "-", - bar_template: str = "%(label)s [%(bar)s] %(info)s", - info_sep: str = " ", - width: int = 36, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, -) -> ProgressBar[int]: ... - - -@t.overload -def progressbar( - iterable: cabc.Iterable[V] | None = None, - length: int | None = None, - label: str | None = None, - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - item_show_func: t.Callable[[V | None], str | None] | None = None, - fill_char: str = "#", - empty_char: str = "-", - bar_template: str = "%(label)s [%(bar)s] %(info)s", - info_sep: str = " ", - width: int = 36, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, -) -> ProgressBar[V]: ... - - -def progressbar( - iterable: cabc.Iterable[V] | None = None, - length: int | None = None, - label: str | None = None, - hidden: bool = False, - show_eta: bool = True, - show_percent: bool | None = None, - show_pos: bool = False, - item_show_func: t.Callable[[V | None], str | None] | None = None, - fill_char: str = "#", - empty_char: str = "-", - bar_template: str = "%(label)s [%(bar)s] %(info)s", - info_sep: str = " ", - width: int = 36, - file: t.TextIO | None = None, - color: bool | None = None, - update_min_steps: int = 1, -) -> ProgressBar[V]: - """This function creates an iterable context manager that can be used - to iterate over something while showing a progress bar. It will - either iterate over the `iterable` or `length` items (that are counted - up). While iteration happens, this function will print a rendered - progress bar to the given `file` (defaults to stdout) and will attempt - to calculate remaining time and more. By default, this progress bar - will not be rendered if the file is not a terminal. - - The context manager creates the progress bar. When the context - manager is entered the progress bar is already created. With every - iteration over the progress bar, the iterable passed to the bar is - advanced and the bar is updated. When the context manager exits, - a newline is printed and the progress bar is finalized on screen. - - Note: The progress bar is currently designed for use cases where the - total progress can be expected to take at least several seconds. - Because of this, the ProgressBar class object won't display - progress that is considered too fast, and progress where the time - between steps is less than a second. - - No printing must happen or the progress bar will be unintentionally - destroyed. - - Example usage:: - - with progressbar(items) as bar: - for item in bar: - do_something_with(item) - - Alternatively, if no iterable is specified, one can manually update the - progress bar through the `update()` method instead of directly - iterating over the progress bar. The update method accepts the number - of steps to increment the bar with:: - - with progressbar(length=chunks.total_bytes) as bar: - for chunk in chunks: - process_chunk(chunk) - bar.update(chunks.bytes) - - The ``update()`` method also takes an optional value specifying the - ``current_item`` at the new position. This is useful when used - together with ``item_show_func`` to customize the output for each - manual step:: - - with click.progressbar( - length=total_size, - label='Unzipping archive', - item_show_func=lambda a: a.filename - ) as bar: - for archive in zip_file: - archive.extract() - bar.update(archive.size, archive) - - :param iterable: an iterable to iterate over. If not provided the length - is required. - :param length: the number of items to iterate over. By default the - progressbar will attempt to ask the iterator about its - length, which might or might not work. If an iterable is - also provided this parameter can be used to override the - length. If an iterable is not provided the progress bar - will iterate over a range of that length. - :param label: the label to show next to the progress bar. - :param hidden: hide the progressbar. Defaults to ``False``. When no tty is - detected, it will only print the progressbar label. Setting this to - ``False`` also disables that. - :param show_eta: enables or disables the estimated time display. This is - automatically disabled if the length cannot be - determined. - :param show_percent: enables or disables the percentage display. The - default is `True` if the iterable has a length or - `False` if not. - :param show_pos: enables or disables the absolute position display. The - default is `False`. - :param item_show_func: A function called with the current item which - can return a string to show next to the progress bar. If the - function returns ``None`` nothing is shown. The current item can - be ``None``, such as when entering and exiting the bar. - :param fill_char: the character to use to show the filled part of the - progress bar. - :param empty_char: the character to use to show the non-filled part of - the progress bar. - :param bar_template: the format string to use as template for the bar. - The parameters in it are ``label`` for the label, - ``bar`` for the progress bar and ``info`` for the - info section. - :param info_sep: the separator between multiple info items (eta etc.) - :param width: the width of the progress bar in characters, 0 means full - terminal width - :param file: The file to write to. If this is not a terminal then - only the label is printed. - :param color: controls if the terminal supports ANSI colors or not. The - default is autodetection. This is only needed if ANSI - codes are included anywhere in the progress bar output - which is not the case by default. - :param update_min_steps: Render only when this many updates have - completed. This allows tuning for very fast iterators. - - .. versionadded:: 8.2 - The ``hidden`` argument. - - .. versionchanged:: 8.0 - Output is shown even if execution time is less than 0.5 seconds. - - .. versionchanged:: 8.0 - ``item_show_func`` shows the current item, not the previous one. - - .. versionchanged:: 8.0 - Labels are echoed if the output is not a TTY. Reverts a change - in 7.0 that removed all output. - - .. versionadded:: 8.0 - The ``update_min_steps`` parameter. - - .. versionadded:: 4.0 - The ``color`` parameter and ``update`` method. - - .. versionadded:: 2.0 - """ - from ._termui_impl import ProgressBar - - color = resolve_color_default(color) - return ProgressBar( - iterable=iterable, - length=length, - hidden=hidden, - show_eta=show_eta, - show_percent=show_percent, - show_pos=show_pos, - item_show_func=item_show_func, - fill_char=fill_char, - empty_char=empty_char, - bar_template=bar_template, - info_sep=info_sep, - file=file, - label=label, - width=width, - color=color, - update_min_steps=update_min_steps, - ) - - -def clear() -> None: - """Clears the terminal screen. This will have the effect of clearing - the whole visible space of the terminal and moving the cursor to the - top left. This does not do anything if not connected to a terminal. - - .. versionadded:: 2.0 - """ - if not isatty(sys.stdout): - return - - # ANSI escape \033[2J clears the screen, \033[1;1H moves the cursor - echo("\033[2J\033[1;1H", nl=False) - - -def _interpret_color(color: int | tuple[int, int, int] | str, offset: int = 0) -> str: - if isinstance(color, int): - return f"{38 + offset};5;{color:d}" - - if isinstance(color, (tuple, list)): - r, g, b = color - return f"{38 + offset};2;{r:d};{g:d};{b:d}" - - return str(_ansi_colors[color] + offset) - - -def style( - text: t.Any, - fg: int | tuple[int, int, int] | str | None = None, - bg: int | tuple[int, int, int] | str | None = None, - bold: bool | None = None, - dim: bool | None = None, - underline: bool | None = None, - overline: bool | None = None, - italic: bool | None = None, - blink: bool | None = None, - reverse: bool | None = None, - strikethrough: bool | None = None, - reset: bool = True, -) -> str: - """Styles a text with ANSI styles and returns the new string. By - default the styling is self contained which means that at the end - of the string a reset code is issued. This can be prevented by - passing ``reset=False``. - - Examples:: - - click.echo(click.style('Hello World!', fg='green')) - click.echo(click.style('ATTENTION!', blink=True)) - click.echo(click.style('Some things', reverse=True, fg='cyan')) - click.echo(click.style('More colors', fg=(255, 12, 128), bg=117)) - - Supported color names: - - * ``black`` (might be a gray) - * ``red`` - * ``green`` - * ``yellow`` (might be an orange) - * ``blue`` - * ``magenta`` - * ``cyan`` - * ``white`` (might be light gray) - * ``bright_black`` - * ``bright_red`` - * ``bright_green`` - * ``bright_yellow`` - * ``bright_blue`` - * ``bright_magenta`` - * ``bright_cyan`` - * ``bright_white`` - * ``reset`` (reset the color code only) - - If the terminal supports it, color may also be specified as: - - - An integer in the interval [0, 255]. The terminal must support - 8-bit/256-color mode. - - An RGB tuple of three integers in [0, 255]. The terminal must - support 24-bit/true-color mode. - - See https://en.wikipedia.org/wiki/ANSI_color and - https://gist.github.com/XVilka/8346728 for more information. - - :param text: the string to style with ansi codes. - :param fg: if provided this will become the foreground color. - :param bg: if provided this will become the background color. - :param bold: if provided this will enable or disable bold mode. - :param dim: if provided this will enable or disable dim mode. This is - badly supported. - :param underline: if provided this will enable or disable underline. - :param overline: if provided this will enable or disable overline. - :param italic: if provided this will enable or disable italic. - :param blink: if provided this will enable or disable blinking. - :param reverse: if provided this will enable or disable inverse - rendering (foreground becomes background and the - other way round). - :param strikethrough: if provided this will enable or disable - striking through text. - :param reset: by default a reset-all code is added at the end of the - string which means that styles do not carry over. This - can be disabled to compose styles. - - .. versionchanged:: 8.0 - A non-string ``message`` is converted to a string. - - .. versionchanged:: 8.0 - Added support for 256 and RGB color codes. - - .. versionchanged:: 8.0 - Added the ``strikethrough``, ``italic``, and ``overline`` - parameters. - - .. versionchanged:: 7.0 - Added support for bright colors. - - .. versionadded:: 2.0 - """ - if not isinstance(text, str): - text = str(text) - - bits = [] - - if fg: - try: - bits.append(f"\033[{_interpret_color(fg)}m") - except KeyError: - raise TypeError(_("Unknown color {colour!r}").format(colour=fg)) from None - - if bg: - try: - bits.append(f"\033[{_interpret_color(bg, 10)}m") - except KeyError: - raise TypeError(_("Unknown color {colour!r}").format(colour=bg)) from None - - if bold is not None: - bits.append(f"\033[{1 if bold else 22}m") - if dim is not None: - bits.append(f"\033[{2 if dim else 22}m") - if underline is not None: - bits.append(f"\033[{4 if underline else 24}m") - if overline is not None: - bits.append(f"\033[{53 if overline else 55}m") - if italic is not None: - bits.append(f"\033[{3 if italic else 23}m") - if blink is not None: - bits.append(f"\033[{5 if blink else 25}m") - if reverse is not None: - bits.append(f"\033[{7 if reverse else 27}m") - if strikethrough is not None: - bits.append(f"\033[{9 if strikethrough else 29}m") - bits.append(text) - if reset: - bits.append(_ansi_reset_all) - return "".join(bits) - - -def unstyle(text: str) -> str: - """Removes ANSI styling information from a string. Usually it's not - necessary to use this function as Click's echo function will - automatically remove styling if necessary. - - .. versionadded:: 2.0 - - :param text: the text to remove style information from. - """ - return strip_ansi(text) - - -def secho( - message: t.Any | None = None, - file: t.IO[t.AnyStr] | None = None, - nl: bool = True, - err: bool = False, - color: bool | None = None, - **styles: t.Any, -) -> None: - """This function combines :func:`echo` and :func:`style` into one - call. As such the following two calls are the same:: - - click.secho('Hello World!', fg='green') - click.echo(click.style('Hello World!', fg='green')) - - All keyword arguments are forwarded to the underlying functions - depending on which one they go with. - - Non-string types will be converted to :class:`str`. However, - :class:`bytes` are passed directly to :meth:`echo` without applying - style. If you want to style bytes that represent text, call - :meth:`bytes.decode` first. - - .. versionchanged:: 8.0 - A non-string ``message`` is converted to a string. Bytes are - passed through without style applied. - - .. versionadded:: 2.0 - """ - if message is not None and not isinstance(message, (bytes, bytearray)): - message = style(message, **styles) - - return echo(message, file=file, nl=nl, err=err, color=color) - - -@t.overload -def edit( - text: bytes | bytearray, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = False, - extension: str = ".txt", -) -> bytes | None: ... - - -@t.overload -def edit( - text: str, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", -) -> str | None: ... - - -@t.overload -def edit( - text: None = None, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", - filename: str | cabc.Iterable[str] | None = None, -) -> None: ... - - -def edit( - text: str | bytes | bytearray | None = None, - editor: str | None = None, - env: cabc.Mapping[str, str] | None = None, - require_save: bool = True, - extension: str = ".txt", - filename: str | cabc.Iterable[str] | None = None, -) -> str | bytes | bytearray | None: - r"""Edits the given text in the defined editor. If an editor is given - (should be the full path to the executable but the regular operating - system search path is used for finding the executable) it overrides - the detected editor. Optionally, some environment variables can be - used. If the editor is closed without changes, `None` is returned. In - case a file is edited directly the return value is always `None` and - `require_save` and `extension` are ignored. - - If the editor cannot be opened a :exc:`UsageError` is raised. - - Note for Windows: to simplify cross-platform usage, the newlines are - automatically converted from POSIX to Windows and vice versa. As such, - the message here will have ``\n`` as newline markers. - - :param text: the text to edit. - :param editor: optionally the editor to use. Defaults to automatic - detection. - :param env: environment variables to forward to the editor. - :param require_save: if this is true, then not saving in the editor - will make the return value become `None`. - :param extension: the extension to tell the editor about. This defaults - to `.txt` but changing this might change syntax - highlighting. - :param filename: if provided it will edit this file instead of the - provided text contents. It will not use a temporary - file as an indirection in that case. If the editor supports - editing multiple files at once, a sequence of files may be - passed as well. Invoke `click.file` once per file instead - if multiple files cannot be managed at once or editing the - files serially is desired. - - .. versionchanged:: 8.2.0 - ``filename`` now accepts any ``Iterable[str]`` in addition to a ``str`` - if the ``editor`` supports editing multiple files at once. - - """ - from ._termui_impl import Editor - - ed = Editor(editor=editor, env=env, require_save=require_save, extension=extension) - - if filename is None: - return ed.edit(text) - - if isinstance(filename, str): - filename = (filename,) - - ed.edit_files(filenames=filename) - return None - - -def launch(url: str, wait: bool = False, locate: bool = False) -> int: - """This function launches the given URL (or filename) in the default - viewer application for this file type. If this is an executable, it - might launch the executable in a new session. The return value is - the exit code of the launched application. Usually, ``0`` indicates - success. - - Examples:: - - click.launch('https://click.palletsprojects.com/') - click.launch('/my/downloaded/file', locate=True) - - .. versionadded:: 2.0 - - :param url: URL or filename of the thing to launch. - :param wait: Wait for the program to exit before returning. This - only works if the launched program blocks. In particular, - ``xdg-open`` on Linux does not block. - :param locate: if this is set to `True` then instead of launching the - application associated with the URL it will attempt to - launch a file manager with the file located. This - might have weird effects if the URL does not point to - the filesystem. - """ - from ._termui_impl import open_url - - return open_url(url, wait=wait, locate=locate) - - -# If this is provided, getchar() calls into this instead. This is used -# for unittesting purposes. -_getchar: t.Callable[[bool], str] | None = None - - -def getchar(echo: bool = False) -> str: - """Fetches a single character from the terminal and returns it. This - will always return a unicode character and under certain rare - circumstances this might return more than one character. The - situations which more than one character is returned is when for - whatever reason multiple characters end up in the terminal buffer or - standard input was not actually a terminal. - - Note that this will always read from the terminal, even if something - is piped into the standard input. - - Note for Windows: in rare cases when typing non-ASCII characters, this - function might wait for a second character and then return both at once. - This is because certain Unicode characters look like special-key markers. - - .. versionadded:: 2.0 - - :param echo: if set to `True`, the character read will also show up on - the terminal. The default is to not show it. - """ - global _getchar - - if _getchar is None: - from ._termui_impl import getchar as f - - _getchar = f - - return _getchar(echo) - - -def raw_terminal() -> AbstractContextManager[int]: - from ._termui_impl import raw_terminal as f - - return f() - - -def pause(info: str | None = None, err: bool = False) -> None: - """This command stops execution and waits for the user to press any - key to continue. This is similar to the Windows batch "pause" - command. If the program is not run through a terminal, this command - will instead do nothing. - - .. versionadded:: 2.0 - - .. versionadded:: 4.0 - Added the `err` parameter. - - :param info: The message to print before pausing. Defaults to - ``"Press any key to continue..."``. - :param err: if set to message goes to ``stderr`` instead of - ``stdout``, the same as with echo. - """ - if not isatty(sys.stdin) or not isatty(sys.stdout): - return - - if info is None: - info = _("Press any key to continue...") - - try: - if info: - echo(info, nl=False, err=err) - try: - getchar() - except (KeyboardInterrupt, EOFError): - pass - finally: - if info: - echo(err=err) diff --git a/functions/python/data-processor/click/testing.py b/functions/python/data-processor/click/testing.py deleted file mode 100644 index 1eb81d29e..000000000 --- a/functions/python/data-processor/click/testing.py +++ /dev/null @@ -1,736 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import contextlib -import io -import os -import pdb -import shlex -import sys -import tempfile -import typing as t -from types import TracebackType - -from . import _compat -from . import formatting -from . import termui -from . import utils -from ._compat import _find_binary_reader - -if t.TYPE_CHECKING: - from _typeshed import ReadableBuffer - - from .core import Command - -CaptureMode = t.Literal["sys", "fd"] - - -class EchoingStdin: - def __init__(self, input: t.BinaryIO, output: t.BinaryIO) -> None: - self._input = input - self._output = output - self._paused = False - - def __getattr__(self, x: str) -> t.Any: - return getattr(self._input, x) - - def _echo(self, rv: bytes) -> bytes: - if not self._paused: - self._output.write(rv) - - return rv - - def read(self, n: int = -1) -> bytes: - return self._echo(self._input.read(n)) - - def read1(self, n: int = -1) -> bytes: - return self._echo(self._input.read1(n)) # type: ignore - - def readline(self, n: int = -1) -> bytes: - return self._echo(self._input.readline(n)) - - def readlines(self) -> list[bytes]: - return [self._echo(x) for x in self._input.readlines()] - - def __iter__(self) -> cabc.Iterator[bytes]: - return iter(self._echo(x) for x in self._input) - - def __repr__(self) -> str: - return repr(self._input) - - -@contextlib.contextmanager -def _pause_echo(stream: EchoingStdin | None) -> cabc.Iterator[None]: - if stream is None: - yield - else: - stream._paused = True - yield - stream._paused = False - - -class _FDCapture: - """Redirect a file descriptor to a temporary file for capture. - - Saves the current target of *targetfd* via :func:`os.dup`, then - redirects it to a temporary file via :func:`os.dup2`. On - :meth:`stop`, restores the original ``fd`` and returns the captured - bytes. Inspired by Pytest's ``FDCapture``. - - .. versionadded:: 8.4.0 - """ - - def __init__(self, targetfd: int) -> None: - self._targetfd = targetfd - self.saved_fd: int = -1 - self._tmpfile: t.BinaryIO | None = None - - def start(self) -> None: - self.saved_fd = os.dup(self._targetfd) - self._tmpfile = tempfile.TemporaryFile(buffering=0) - os.dup2(self._tmpfile.fileno(), self._targetfd) - - def stop(self) -> bytes: - assert self._tmpfile is not None, "_FDCapture.start() was not called" - os.dup2(self.saved_fd, self._targetfd) - os.close(self.saved_fd) - self.saved_fd = -1 - self._tmpfile.seek(0) - data = self._tmpfile.read() - self._tmpfile.close() - self._tmpfile = None - return data - - -class BytesIOCopy(io.BytesIO): - """Patch ``io.BytesIO`` to let the written stream be copied to another. - - .. versionadded:: 8.2 - """ - - def __init__(self, copy_to: io.BytesIO) -> None: - super().__init__() - self.copy_to = copy_to - - def flush(self) -> None: - super().flush() - self.copy_to.flush() - - def write(self, b: ReadableBuffer) -> int: - self.copy_to.write(b) - return super().write(b) - - -class StreamMixer: - """Mixes `` and `` streams. - - The result is available in the ``output`` attribute. - - .. versionadded:: 8.2 - """ - - def __init__(self) -> None: - self.output: io.BytesIO = io.BytesIO() - self.stdout: io.BytesIO = BytesIOCopy(copy_to=self.output) - self.stderr: io.BytesIO = BytesIOCopy(copy_to=self.output) - - -class _NamedTextIOWrapper(io.TextIOWrapper): - """A :class:`~io.TextIOWrapper` with custom ``name`` and ``mode`` - that does not close its underlying buffer. - - When ``CliRunner`` runs in ``fd`` mode, ``_original_fd`` is patched to - point at the saved (pre-redirection) ``fd``, so C-level consumers that call - :meth:`fileno` (like ``faulthandler`` or ``subprocess``) keep working. In - the default ``sys`` mode ``_original_fd`` stays at ``-1`` and - :meth:`fileno` raises :exc:`io.UnsupportedOperation`, matching the - pre-``8.3.3`` behavior. - """ - - def __init__( - self, - buffer: t.BinaryIO, - name: str, - mode: str, - **kwargs: t.Any, - ) -> None: - super().__init__(buffer, **kwargs) - self._name = name - self._mode = mode - self._original_fd: int = -1 - - def close(self) -> None: - """The buffer this object contains belongs to some other object, - so prevent the default ``__del__`` implementation from closing - that buffer. - - .. versionadded:: 8.3.2 - """ - - def fileno(self) -> int: - """Return the file descriptor of the saved original stream when - ``CliRunner`` runs in ``fd`` mode. Otherwise delegate to - :class:`~io.TextIOWrapper`, which raises - :exc:`io.UnsupportedOperation` for a ``BytesIO``-backed buffer. - """ - if self._original_fd >= 0: - return self._original_fd - return super().fileno() - - @property - def name(self) -> str: - return self._name - - @property - def mode(self) -> str: - return self._mode - - -def make_input_stream( - input: str | bytes | t.IO[t.Any] | None, charset: str -) -> t.BinaryIO: - # Is already an input stream. - if hasattr(input, "read"): - rv = _find_binary_reader(t.cast("t.IO[t.Any]", input)) - - if rv is not None: - return rv - - raise TypeError("Could not find binary reader for input stream.") - - if input is None: - input = b"" - elif isinstance(input, str): - input = input.encode(charset) - - return io.BytesIO(input) - - -class Result: - """Holds the captured result of an invoked CLI script. - - :param runner: The runner that created the result - :param stdout_bytes: The standard output as bytes. - :param stderr_bytes: The standard error as bytes. - :param output_bytes: A mix of ``stdout_bytes`` and ``stderr_bytes``, as the - user would see it in its terminal. - :param return_value: The value returned from the invoked command. - :param exit_code: The exit code as integer. - :param exception: The exception that happened if one did. - :param exc_info: Exception information (exception type, exception instance, - traceback type). - - .. versionchanged:: 8.2 - ``stderr_bytes`` no longer optional, ``output_bytes`` introduced and - ``mix_stderr`` has been removed. - - .. versionadded:: 8.0 - Added ``return_value``. - """ - - def __init__( - self, - runner: CliRunner, - stdout_bytes: bytes, - stderr_bytes: bytes, - output_bytes: bytes, - return_value: t.Any, - exit_code: int, - exception: BaseException | None, - exc_info: tuple[type[BaseException], BaseException, TracebackType] - | None = None, - ): - self.runner = runner - self.stdout_bytes = stdout_bytes - self.stderr_bytes = stderr_bytes - self.output_bytes = output_bytes - self.return_value = return_value - self.exit_code = exit_code - self.exception = exception - self.exc_info = exc_info - - @property - def output(self) -> str: - """The terminal output as unicode string, as the user would see it. - - .. versionchanged:: 8.2 - No longer a proxy for ``self.stdout``. Now has its own independent stream - that is mixing `` and ``, in the order they were written. - """ - return self.output_bytes.decode(self.runner.charset, "replace").replace( - "\r\n", "\n" - ) - - @property - def stdout(self) -> str: - """The standard output as unicode string.""" - return self.stdout_bytes.decode(self.runner.charset, "replace").replace( - "\r\n", "\n" - ) - - @property - def stderr(self) -> str: - """The standard error as unicode string. - - .. versionchanged:: 8.2 - No longer raise an exception, always returns the `` string. - """ - return self.stderr_bytes.decode(self.runner.charset, "replace").replace( - "\r\n", "\n" - ) - - def __repr__(self) -> str: - exc_str = repr(self.exception) if self.exception else "okay" - return f"<{type(self).__name__} {exc_str}>" - - -class CliRunner: - """The CLI runner provides functionality to invoke a Click command line - script for unittesting purposes in a isolated environment. This only - works in single-threaded systems without any concurrency as it changes the - global interpreter state. - - :param charset: the character set for the input and output data. - :param env: a dictionary with environment variables for overriding. - :param echo_stdin: if this is set to `True`, then reading from `` writes - to ``. This is useful for showing examples in - some circumstances. Note that regular prompts - will automatically echo the input. - :param catch_exceptions: Whether to catch any exceptions other than - ``SystemExit`` when running :meth:`~CliRunner.invoke`. - :param capture: Selects the output capture strategy. ``sys`` (default) - captures Python-level writes only and leaves - :meth:`sys.stdout.fileno` raising :exc:`io.UnsupportedOperation`, so - user code that calls :func:`os.dup2` on ``sys.stdout.fileno()`` cannot - clobber the host runner's stdout. ``fd`` redirects file descriptors - ``1`` and ``2`` via :func:`os.dup2` to a temporary file, also catching - output from stale stream references, C extensions, and subprocesses. - ``fd`` is not supported on Windows. - - .. versionchanged:: 8.4.0 - Added the ``capture`` parameter. The default ``sys`` mode no longer - exposes the original fd through :meth:`fileno`, reverting the change - introduced in ``8.3.3`` that broke Pytest's ``fd``-level capture - teardown. Use ``capture="fd"`` to restore that behavior with proper - isolation. :issue:`3384` - - .. versionchanged:: 8.2 - Added the ``catch_exceptions`` parameter. - - .. versionchanged:: 8.2 - ``mix_stderr`` parameter has been removed. - """ - - def __init__( - self, - charset: str = "utf-8", - env: cabc.Mapping[str, str | None] | None = None, - echo_stdin: bool = False, - catch_exceptions: bool = True, - capture: CaptureMode = "sys", - ) -> None: - if capture not in {"sys", "fd"}: - raise ValueError( - f"capture={capture!r} is not valid. Choose from 'sys' or 'fd'." - ) - if capture == "fd" and sys.platform == "win32": - raise ValueError( - f"capture={capture!r} is not supported on Windows. Use 'sys'." - ) - self.charset = charset - self.env: cabc.Mapping[str, str | None] = env or {} - self.echo_stdin = echo_stdin - self.catch_exceptions = catch_exceptions - self.capture: CaptureMode = capture - - def get_default_prog_name(self, cli: Command) -> str: - """Given a command object it will return the default program name - for it. The default is the `name` attribute or ``"root"`` if not - set. - """ - return cli.name or "root" - - def make_env( - self, overrides: cabc.Mapping[str, str | None] | None = None - ) -> cabc.Mapping[str, str | None]: - """Returns the environment overrides for invoking a script.""" - rv = dict(self.env) - if overrides: - rv.update(overrides) - return rv - - @contextlib.contextmanager - def isolation( - self, - input: str | bytes | t.IO[t.Any] | None = None, - env: cabc.Mapping[str, str | None] | None = None, - color: bool = False, - ) -> cabc.Iterator[tuple[io.BytesIO, io.BytesIO, io.BytesIO]]: - """A context manager that sets up the isolation for invoking of a - command line tool. This sets up `` with the given input data - and `os.environ` with the overrides from the given dictionary. - This also rebinds some internals in Click to be mocked (like the - prompt functionality). - - This is automatically done in the :meth:`invoke` method. - - :param input: the input stream to put into `sys.stdin`. - :param env: the environment overrides as dictionary. - :param color: whether the output should contain color codes. The - application can still override this explicitly. - - .. versionadded:: 8.2 - An additional output stream is returned, which is a mix of - `` and `` streams. - - .. versionchanged:: 8.2 - Always returns the `` stream. - - .. versionchanged:: 8.0 - `` is opened with ``errors="backslashreplace"`` - instead of the default ``"strict"``. - - .. versionchanged:: 4.0 - Added the ``color`` parameter. - """ - bytes_input = make_input_stream(input, self.charset) - echo_input = None - - old_stdin = sys.stdin - old_stdout = sys.stdout - old_stderr = sys.stderr - old_forced_width = formatting.FORCED_WIDTH - formatting.FORCED_WIDTH = 80 - - env = self.make_env(env) - - stream_mixer = StreamMixer() - - if self.echo_stdin: - bytes_input = echo_input = t.cast( - t.BinaryIO, EchoingStdin(bytes_input, stream_mixer.stdout) - ) - - sys.stdin = text_input = _NamedTextIOWrapper( - bytes_input, encoding=self.charset, name="", mode="r" - ) - - if self.echo_stdin: - # Force unbuffered reads, otherwise TextIOWrapper reads a - # large chunk which is echoed early. - text_input._CHUNK_SIZE = 1 # type: ignore - - sys.stdout = _NamedTextIOWrapper( - stream_mixer.stdout, - encoding=self.charset, - name="", - mode="w", - ) - - sys.stderr = _NamedTextIOWrapper( - stream_mixer.stderr, - encoding=self.charset, - name="", - mode="w", - errors="backslashreplace", - ) - - @_pause_echo(echo_input) # type: ignore - def visible_input(prompt: str | None = None) -> str: - sys.stdout.write(prompt or "") - try: - val = next(text_input).rstrip("\r\n") - except StopIteration as e: - raise EOFError() from e - sys.stdout.write(f"{val}\n") - sys.stdout.flush() - return val - - @_pause_echo(echo_input) # type: ignore - def hidden_input(prompt: str | None = None) -> str: - sys.stdout.write(f"{prompt or ''}\n") - sys.stdout.flush() - try: - return next(text_input).rstrip("\r\n") - except StopIteration as e: - raise EOFError() from e - - @_pause_echo(echo_input) # type: ignore - def _getchar(echo: bool) -> str: - char = sys.stdin.read(1) - - if echo: - sys.stdout.write(char) - - sys.stdout.flush() - return char - - default_color = color - - def should_strip_ansi( - stream: t.IO[t.Any] | None = None, color: bool | None = None - ) -> bool: - if color is None: - return not default_color - return not color - - old_visible_prompt_func = termui.visible_prompt_func - old_hidden_prompt_func = termui.hidden_prompt_func - old__getchar_func = termui._getchar - old_should_strip_ansi = utils.should_strip_ansi # type: ignore - old__compat_should_strip_ansi = _compat.should_strip_ansi - old_pdb_init = pdb.Pdb.__init__ - termui.visible_prompt_func = visible_input - termui.hidden_prompt_func = hidden_input - termui._getchar = _getchar - utils.should_strip_ansi = should_strip_ansi # type: ignore - _compat.should_strip_ansi = should_strip_ansi - - def _patched_pdb_init( - self: pdb.Pdb, - completekey: str = "tab", - stdin: t.IO[str] | None = None, - stdout: t.IO[str] | None = None, - **kwargs: t.Any, - ) -> None: - """Default ``pdb.Pdb`` to real terminal streams during - ``CliRunner`` isolation. - - Without this patch, ``pdb.Pdb.__init__`` inherits from - ``cmd.Cmd`` which falls back to ``sys.stdin``/``sys.stdout`` - when no explicit streams are provided. During isolation - those are ``BytesIO``-backed wrappers, so the debugger - reads from an empty buffer and writes to captured output, - making interactive debugging impossible. - - By defaulting to ``sys.__stdin__``/``sys.__stdout__`` (the - original terminal streams Python preserves regardless of - redirection), debuggers can interact with the user while - ``click.echo`` output is still captured normally. - - This covers ``pdb.set_trace()``, ``breakpoint()``, - ``pdb.post_mortem()``, and debuggers that subclass - ``pdb.Pdb`` (ipdb, pdbpp). Explicit ``stdin``/``stdout`` - arguments are honored and not overridden. Debuggers that - do not subclass ``pdb.Pdb`` (pudb, debugpy) are not - covered. - """ - if stdin is None: - stdin = sys.__stdin__ - if stdout is None: - stdout = sys.__stdout__ - old_pdb_init( - self, completekey=completekey, stdin=stdin, stdout=stdout, **kwargs - ) - - pdb.Pdb.__init__ = _patched_pdb_init # type: ignore[assignment] - - old_env = {} - try: - for key, value in env.items(): - old_env[key] = os.environ.get(key) - if value is None: - try: - del os.environ[key] - except Exception: - pass - else: - os.environ[key] = value - yield (stream_mixer.stdout, stream_mixer.stderr, stream_mixer.output) - finally: - for key, value in old_env.items(): - if value is None: - try: - del os.environ[key] - except Exception: - pass - else: - os.environ[key] = value - sys.stdout = old_stdout - sys.stderr = old_stderr - sys.stdin = old_stdin - termui.visible_prompt_func = old_visible_prompt_func - termui.hidden_prompt_func = old_hidden_prompt_func - termui._getchar = old__getchar_func - utils.should_strip_ansi = old_should_strip_ansi # type: ignore - _compat.should_strip_ansi = old__compat_should_strip_ansi - formatting.FORCED_WIDTH = old_forced_width - pdb.Pdb.__init__ = old_pdb_init # type: ignore[method-assign] - - def invoke( - self, - cli: Command, - args: str | cabc.Sequence[str] | None = None, - input: str | bytes | t.IO[t.Any] | None = None, - env: cabc.Mapping[str, str | None] | None = None, - catch_exceptions: bool | None = None, - color: bool = False, - **extra: t.Any, - ) -> Result: - """Invokes a command in an isolated environment. The arguments are - forwarded directly to the command line script, the `extra` keyword - arguments are passed to the :meth:`~clickpkg.Command.main` function of - the command. - - This returns a :class:`Result` object. - - :param cli: the command to invoke - :param args: the arguments to invoke. It may be given as an iterable - or a string. When given as string it will be interpreted - as a Unix shell command. More details at - :func:`shlex.split`. - :param input: the input data for `sys.stdin`. - :param env: the environment overrides. - :param catch_exceptions: Whether to catch any other exceptions than - ``SystemExit``. If :data:`None`, the value - from :class:`CliRunner` is used. - :param extra: the keyword arguments to pass to :meth:`main`. - :param color: whether the output should contain color codes. The - application can still override this explicitly. - - .. versionadded:: 8.2 - The result object has the ``output_bytes`` attribute with - the mix of ``stdout_bytes`` and ``stderr_bytes``, as the user would - see it in its terminal. - - .. versionchanged:: 8.2 - The result object always returns the ``stderr_bytes`` stream. - - .. versionchanged:: 8.0 - The result object has the ``return_value`` attribute with - the value returned from the invoked command. - - .. versionchanged:: 4.0 - Added the ``color`` parameter. - - .. versionchanged:: 3.0 - Added the ``catch_exceptions`` parameter. - - .. versionchanged:: 3.0 - The result object has the ``exc_info`` attribute with the - traceback if available. - """ - exc_info = None - if catch_exceptions is None: - catch_exceptions = self.catch_exceptions - - # Set up fd capture before isolation replaces sys.stdout and sys.stderr. - cap_out: _FDCapture | None = None - cap_err: _FDCapture | None = None - - if self.capture == "fd": - cap_out = _FDCapture(1) - cap_err = _FDCapture(2) - try: - cap_out.start() - cap_err.start() - except OSError: - cap_out = cap_err = None - - with self.isolation(input=input, env=env, color=color) as outstreams: - # Point the captured streams' fileno() at the saved (original) - # fd so that C-level consumers like faulthandler keep working - # while fd 1/2 are redirected to the capture tmpfile. - if cap_out is not None and cap_err is not None: - sys.stdout._original_fd = cap_out.saved_fd # type: ignore[union-attr] - sys.stderr._original_fd = cap_err.saved_fd # type: ignore[union-attr] - - return_value = None - exception: BaseException | None = None - exit_code = 0 - - if isinstance(args, str): - args = shlex.split(args) - - try: - prog_name = extra.pop("prog_name") - except KeyError: - prog_name = self.get_default_prog_name(cli) - - try: - return_value = cli.main(args=args or (), prog_name=prog_name, **extra) - except SystemExit as e: - exc_info = sys.exc_info() - e_code = t.cast("int | t.Any | None", e.code) - - if e_code is None: - e_code = 0 - - if e_code != 0: - exception = e - - if not isinstance(e_code, int): - sys.stdout.write(str(e_code)) - sys.stdout.write("\n") - e_code = 1 - - exit_code = e_code - - except Exception as e: - if not catch_exceptions: - raise - exception = e - exit_code = 1 - exc_info = sys.exc_info() - finally: - sys.stdout.flush() - sys.stderr.flush() - - # Stop fd capture and merge the captured bytes into - # the stdout/stderr BytesIO streams. BytesIOCopy mirrors - # those writes into outstreams[2] automatically. - if cap_out is not None and cap_err is not None: - fd_out = cap_out.stop() - fd_err = cap_err.stop() - if fd_out: - outstreams[0].write(fd_out) - if fd_err: - outstreams[1].write(fd_err) - - stdout = outstreams[0].getvalue() - stderr = outstreams[1].getvalue() - output = outstreams[2].getvalue() - - return Result( - runner=self, - stdout_bytes=stdout, - stderr_bytes=stderr, - output_bytes=output, - return_value=return_value, - exit_code=exit_code, - exception=exception, - exc_info=exc_info, # type: ignore - ) - - @contextlib.contextmanager - def isolated_filesystem( - self, temp_dir: str | os.PathLike[str] | None = None - ) -> cabc.Iterator[str]: - """A context manager that creates a temporary directory and - changes the current working directory to it. This isolates tests - that affect the contents of the CWD to prevent them from - interfering with each other. - - :param temp_dir: Create the temporary directory under this - directory. If given, the created directory is not removed - when exiting. - - .. versionchanged:: 8.0 - Added the ``temp_dir`` parameter. - """ - cwd = os.getcwd() - dt = tempfile.mkdtemp(dir=temp_dir) - os.chdir(dt) - - try: - yield dt - finally: - os.chdir(cwd) - - if temp_dir is None: - import shutil - - try: - shutil.rmtree(dt) - except OSError: - pass diff --git a/functions/python/data-processor/click/types.py b/functions/python/data-processor/click/types.py deleted file mode 100644 index 98c2d616b..000000000 --- a/functions/python/data-processor/click/types.py +++ /dev/null @@ -1,1309 +0,0 @@ -from __future__ import annotations - -import abc -import collections.abc as cabc -import enum -import os -import stat -import sys -import typing as t -import uuid -from datetime import datetime -from gettext import gettext as _ -from gettext import ngettext - -from ._compat import _get_argv_encoding -from ._compat import open_stream -from .exceptions import BadParameter -from .utils import format_filename -from .utils import LazyFile -from .utils import safecall - -if t.TYPE_CHECKING: - import typing_extensions as te - - from .core import Context - from .core import Parameter - from .shell_completion import CompletionItem - -ParamTypeValue = t.TypeVar("ParamTypeValue") - - -class ParamTypeInfoDict(t.TypedDict): - param_type: str - name: str - - -class ParamType(t.Generic[ParamTypeValue], abc.ABC): - """Represents the type of a parameter. Validates and converts values - from the command line or Python into the correct type. - - To implement a custom type, subclass and implement at least the - following: - - - The :attr:`name` class attribute must be set. - - Calling an instance of the type with ``None`` must return - ``None``. This is already implemented by default. - - :meth:`convert` must convert string values to the correct type. - - :meth:`convert` must accept values that are already the correct - type. - - It must be able to convert a value if the ``ctx`` and ``param`` - arguments are ``None``. This can occur when converting prompt - input. - - .. versionchanged:: 8.4.0 - Now a generic abstract base class. Parameterize with the - converted value type (``ParamType[int]`` for an integer-returning - type) so that :meth:`convert` and downstream consumers carry the - narrowed return type. - """ - - is_composite: t.ClassVar[bool] = False - arity: t.ClassVar[int] = 1 - - #: the descriptive name of this type - name: str - - #: if a list of this type is expected and the value is pulled from a - #: string environment variable, this is what splits it up. `None` - #: means any whitespace. For all parameters the general rule is that - #: whitespace splits them up. The exception are paths and files which - #: are split by ``os.path.pathsep`` by default (":" on Unix and ";" on - #: Windows). - envvar_list_splitter: t.ClassVar[str | None] = None - - def to_info_dict(self) -> ParamTypeInfoDict: - """Gather information that could be useful for a tool generating - user-facing documentation. - - Use :meth:`click.Context.to_info_dict` to traverse the entire - CLI structure. - - .. versionadded:: 8.0 - """ - # The class name without the "ParamType" suffix. - param_type = type(self).__name__.partition("ParamType")[0] - param_type = param_type.partition("ParameterType")[0] - - # Custom subclasses might not remember to set a name. - if hasattr(self, "name"): - name = self.name - else: - name = param_type - - return {"param_type": param_type, "name": name} - - def __call__( - self, - value: t.Any, - param: Parameter | None = None, - ctx: Context | None = None, - ) -> ParamTypeValue | None: - if value is not None: - return self.convert(value, param, ctx) - return None - - def get_metavar(self, param: Parameter, ctx: Context) -> str | None: - """Returns the metavar default for this param if it provides one.""" - - def get_missing_message(self, param: Parameter, ctx: Context | None) -> str | None: - """Optionally might return extra information about a missing - parameter. - - .. versionadded:: 2.0 - """ - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> ParamTypeValue: - """Convert the value to the correct type. This is not called if - the value is ``None`` (the missing value). - - This must accept string values from the command line, as well as - values that are already the correct type. It may also convert - other compatible types. - - The ``param`` and ``ctx`` arguments may be ``None`` in certain - situations, such as when converting prompt input. - - If the value cannot be converted, call :meth:`fail` with a - descriptive message. - - :param value: The value to convert. - :param param: The parameter that is using this type to convert - its value. May be ``None``. - :param ctx: The current context that arrived at this value. May - be ``None``. - """ - # The default returns the value as-is so subclasses that only customize - # metadata are not forced to redeclare ``convert``. - return t.cast("ParamTypeValue", value) - - def split_envvar_value(self, rv: str) -> cabc.Sequence[str]: - """Given a value from an environment variable this splits it up - into small chunks depending on the defined envvar list splitter. - - If the splitter is set to `None`, which means that whitespace splits, - then leading and trailing whitespace is ignored. Otherwise, leading - and trailing splitters usually lead to empty items being included. - """ - return (rv or "").split(self.envvar_list_splitter) - - def fail( - self, - message: str, - param: Parameter | None = None, - ctx: Context | None = None, - ) -> t.NoReturn: - """Helper method to fail with an invalid value message.""" - raise BadParameter(message, ctx=ctx, param=param) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Return a list of - :class:`~click.shell_completion.CompletionItem` objects for the - incomplete value. Most types do not provide completions, but - some do, and this allows custom types to provide custom - completions as well. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - return [] - - -class CompositeParamType(ParamType[ParamTypeValue]): - is_composite = True - - @property - @abc.abstractmethod - def arity(self) -> int: ... # type: ignore[override] - - -class FuncParamTypeInfoDict(ParamTypeInfoDict): - func: t.Callable[[t.Any], t.Any] - - -class FuncParamType(ParamType[ParamTypeValue]): - def __init__(self, func: t.Callable[[t.Any], ParamTypeValue]) -> None: - self.name: str = func.__name__ - self.func = func - - def to_info_dict(self) -> FuncParamTypeInfoDict: - return {"func": self.func, **super().to_info_dict()} - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> ParamTypeValue: - try: - return self.func(value) - except ValueError as exc: - message = str(exc) - - if not message: - try: - message = str(value) - except UnicodeError: - message = value.decode("utf-8", "replace") - - self.fail(message, param, ctx) - - -class UnprocessedParamType(ParamType[t.Any]): - name = "text" - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> t.Any: - return value - - def __repr__(self) -> str: - return "UNPROCESSED" - - -class StringParamType(ParamType[str]): - name = "text" - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> str: - if isinstance(value, bytes): - enc = _get_argv_encoding() - try: - value = value.decode(enc) - except UnicodeError: - fs_enc = sys.getfilesystemencoding() - if fs_enc != enc: - try: - value = value.decode(fs_enc) - except UnicodeError: - value = value.decode("utf-8", "replace") - else: - value = value.decode("utf-8", "replace") - return value # type: ignore[no-any-return] - return str(value) - - def __repr__(self) -> str: - return "STRING" - - -class ChoiceInfoDict(ParamTypeInfoDict): - choices: cabc.Sequence[t.Any] - case_sensitive: bool - - -class Choice(ParamType[ParamTypeValue], t.Generic[ParamTypeValue]): - """The choice type allows a value to be checked against a fixed set - of supported values. - - You may pass any iterable value which will be converted to a tuple - and thus will only be iterated once. - - The resulting value will always be one of the originally passed choices. - See :meth:`normalize_choice` for more info on the mapping of strings - to choices. See :ref:`choice-opts` for an example. - - :param case_sensitive: Set to false to make choices case - insensitive. Defaults to true. - - .. versionchanged:: 8.4.0 - Now generic in the choice value type. Parameterize with the type of - the choice values (``Choice[HashType]`` for an enum, ``Choice[str]`` - for plain strings) to enable type-checked consumers. - - .. versionchanged:: 8.2.0 - Non-``str`` ``choices`` are now supported. It can additionally be any - iterable. Before you were not recommended to pass anything but a list or - tuple. - - .. versionadded:: 8.2.0 - Choice normalization can be overridden via :meth:`normalize_choice`. - """ - - name = "choice" - - def __init__( - self, choices: cabc.Iterable[ParamTypeValue], case_sensitive: bool = True - ) -> None: - self.choices: cabc.Sequence[ParamTypeValue] = tuple(choices) - self.case_sensitive = case_sensitive - - def to_info_dict(self) -> ChoiceInfoDict: - return { - "choices": self.choices, - "case_sensitive": self.case_sensitive, - **super().to_info_dict(), - } - - def _normalized_mapping( - self, ctx: Context | None = None - ) -> cabc.Mapping[ParamTypeValue, str]: - """ - Returns mapping where keys are the original choices and the values are - the normalized values that are accepted via the command line. - - This is a simple wrapper around :meth:`normalize_choice`, use that - instead which is supported. - """ - return { - choice: self.normalize_choice( - choice=choice, - ctx=ctx, - ) - for choice in self.choices - } - - def normalize_choice(self, choice: ParamTypeValue, ctx: Context | None) -> str: - """ - Normalize a choice value, used to map a passed string to a choice. - Each choice must have a unique normalized value. - - By default uses :meth:`Context.token_normalize_func` and if not case - sensitive, convert it to a casefolded value. - - .. versionadded:: 8.2.0 - """ - normed_value = choice.name if isinstance(choice, enum.Enum) else str(choice) - - if ctx is not None and ctx.token_normalize_func is not None: - normed_value = ctx.token_normalize_func(normed_value) - - if not self.case_sensitive: - normed_value = normed_value.casefold() - - return normed_value - - def get_metavar(self, param: Parameter, ctx: Context) -> str | None: - if param.param_type_name == "option" and not param.show_choices: # type: ignore - choice_metavars = [ - convert_type(type(choice)).name.upper() for choice in self.choices - ] - choices_str = "|".join([*dict.fromkeys(choice_metavars)]) - else: - choices_str = "|".join( - [str(i) for i in self._normalized_mapping(ctx=ctx).values()] - ) - - # Use curly braces to indicate a required argument. - if param.required and param.param_type_name == "argument": - return f"{{{choices_str}}}" - - # Use square braces to indicate an option or optional argument. - return f"[{choices_str}]" - - def get_missing_message(self, param: Parameter, ctx: Context | None) -> str: - """ - Message shown when no choice is passed. - - .. versionchanged:: 8.2.0 Added ``ctx`` argument. - """ - return _("Choose from:\n\t{choices}").format( - choices=",\n\t".join(self._normalized_mapping(ctx=ctx).values()) - ) - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> ParamTypeValue: - """ - For a given value from the parser, normalize it and find its - matching normalized value in the list of choices. Then return the - matched "original" choice. - """ - normed_value = self.normalize_choice(choice=value, ctx=ctx) - normalized_mapping = self._normalized_mapping(ctx=ctx) - - try: - return next( - original - for original, normalized in normalized_mapping.items() - if normalized == normed_value - ) - except StopIteration: - self.fail( - self.get_invalid_choice_message(value=value, ctx=ctx), - param=param, - ctx=ctx, - ) - - def get_invalid_choice_message(self, value: t.Any, ctx: Context | None) -> str: - """Get the error message when the given choice is invalid. - - :param value: The invalid value. - - .. versionadded:: 8.2 - """ - choices_str = ", ".join(map(repr, self._normalized_mapping(ctx=ctx).values())) - return ngettext( - "{value!r} is not {choice}.", - "{value!r} is not one of {choices}.", - len(self.choices), - ).format(value=value, choice=choices_str, choices=choices_str) - - def __repr__(self) -> str: - return _("Choice({choices})").format(choices=list(self.choices)) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Complete choices that start with the incomplete value. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - str_choices = [self.normalize_choice(choice, ctx) for choice in self.choices] - if self.case_sensitive: - matched = (c for c in str_choices if c.startswith(incomplete)) - else: - incomplete = incomplete.lower() - matched = (c for c in str_choices if c.lower().startswith(incomplete)) - - return [CompletionItem(c) for c in matched] - - -class DateTimeInfoDict(ParamTypeInfoDict): - formats: cabc.Sequence[str] - - -class DateTime(ParamType[datetime]): - """The DateTime type converts date strings into `datetime` objects. - - The format strings which are checked are configurable, but default to some - common (non-timezone aware) ISO 8601 formats. - - When specifying *DateTime* formats, you should only pass a list or a tuple. - Other iterables, like generators, may lead to surprising results. - - The format strings are processed using ``datetime.strptime``, and this - consequently defines the format strings which are allowed. - - Parsing is tried using each format, in order, and the first format which - parses successfully is used. - - :param formats: A list or tuple of date format strings, in the order in - which they should be tried. Defaults to - ``'%Y-%m-%d'``, ``'%Y-%m-%dT%H:%M:%S'``, - ``'%Y-%m-%d %H:%M:%S'``. - """ - - name = "datetime" - - def __init__(self, formats: cabc.Sequence[str] | None = None): - self.formats: cabc.Sequence[str] = formats or [ - "%Y-%m-%d", - "%Y-%m-%dT%H:%M:%S", - "%Y-%m-%d %H:%M:%S", - ] - - def to_info_dict(self) -> DateTimeInfoDict: - return {"formats": self.formats, **super().to_info_dict()} - - def get_metavar(self, param: Parameter, ctx: Context) -> str | None: - return f"[{'|'.join(self.formats)}]" - - def _try_to_convert_date(self, value: t.Any, format: str) -> datetime | None: - try: - return datetime.strptime(value, format) - except ValueError: - return None - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> datetime: - if isinstance(value, datetime): - return value - - for format in self.formats: - converted = self._try_to_convert_date(value, format) - - if converted is not None: - return converted - - formats_str = ", ".join(map(repr, self.formats)) - self.fail( - ngettext( - "{value!r} does not match the format {format}.", - "{value!r} does not match the formats {formats}.", - len(self.formats), - ).format(value=value, format=formats_str, formats=formats_str), - param, - ctx, - ) - - def __repr__(self) -> str: - return "DateTime" - - -class _NumberParamTypeBase(ParamType[ParamTypeValue]): - _number_class: t.Callable[[t.Any], ParamTypeValue] - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> ParamTypeValue: - try: - return self._number_class(value) - except ValueError: - self.fail( - _("{value!r} is not a valid {number_type}.").format( - value=value, number_type=self.name - ), - param, - ctx, - ) - - -class NumberRangeInfoDict(ParamTypeInfoDict): - min: float | None - max: float | None - min_open: bool - max_open: bool - clamp: bool - - -class _NumberRangeBase(_NumberParamTypeBase[ParamTypeValue]): - def __init__( - self, - min: float | None = None, - max: float | None = None, - min_open: bool = False, - max_open: bool = False, - clamp: bool = False, - ) -> None: - self.min = min - self.max = max - self.min_open = min_open - self.max_open = max_open - self.clamp = clamp - - def to_info_dict(self) -> NumberRangeInfoDict: - return { - "min": self.min, - "max": self.max, - "min_open": self.min_open, - "max_open": self.max_open, - "clamp": self.clamp, - **super().to_info_dict(), - } - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> ParamTypeValue: - import operator - - rv = super().convert(value, param, ctx) - min = self.min - max = self.max - lt_min: bool = min is not None and ( - operator.le if self.min_open else operator.lt - )(rv, min) # type: ignore[arg-type] - gt_max: bool = max is not None and ( - operator.ge if self.max_open else operator.gt - )(rv, max) # type: ignore[arg-type] - - if self.clamp: - if min is not None and lt_min: - return self._clamp(min, 1, self.min_open) # type: ignore[arg-type] - - if max is not None and gt_max: - return self._clamp(max, -1, self.max_open) # type: ignore[arg-type] - - if lt_min or gt_max: - self.fail( - _("{value} is not in the range {range}.").format( - value=rv, range=self._describe_range() - ), - param, - ctx, - ) - - return rv - - @abc.abstractmethod - def _clamp( - self, bound: ParamTypeValue, dir: t.Literal[1, -1], open: bool - ) -> ParamTypeValue: - """Find the valid value to clamp to bound in the given - direction. - - :param bound: The boundary value. - :param dir: 1 or -1 indicating the direction to move. - :param open: If true, the range does not include the bound. - """ - ... - - def _describe_range(self) -> str: - """Describe the range for use in help text.""" - if self.min is None: - op = "<" if self.max_open else "<=" - return f"x{op}{self.max}" - - if self.max is None: - op = ">" if self.min_open else ">=" - return f"x{op}{self.min}" - - lop = "<" if self.min_open else "<=" - rop = "<" if self.max_open else "<=" - return f"{self.min}{lop}x{rop}{self.max}" - - def __repr__(self) -> str: - clamp = " clamped" if self.clamp else "" - return f"<{type(self).__name__} {self._describe_range()}{clamp}>" - - -class IntParamType(_NumberParamTypeBase[int]): - name = "integer" - _number_class = int - - def __repr__(self) -> str: - return "INT" - - -class IntRange(_NumberRangeBase[int], IntParamType): - """Restrict an :data:`click.INT` value to a range of accepted - values. See :ref:`ranges`. - - If ``min`` or ``max`` are not passed, any value is accepted in that - direction. If ``min_open`` or ``max_open`` are enabled, the - corresponding boundary is not included in the range. - - If ``clamp`` is enabled, a value outside the range is clamped to the - boundary instead of failing. - - .. versionchanged:: 8.0 - Added the ``min_open`` and ``max_open`` parameters. - """ - - name = "integer range" - - def _clamp(self, bound: int, dir: t.Literal[1, -1], open: bool) -> int: - if not open: - return bound - - return bound + dir - - -class FloatParamType(_NumberParamTypeBase[float]): - name = "float" - _number_class = float - - def __repr__(self) -> str: - return "FLOAT" - - -class FloatRange(_NumberRangeBase[float], FloatParamType): - """Restrict a :data:`click.FLOAT` value to a range of accepted - values. See :ref:`ranges`. - - If ``min`` or ``max`` are not passed, any value is accepted in that - direction. If ``min_open`` or ``max_open`` are enabled, the - corresponding boundary is not included in the range. - - If ``clamp`` is enabled, a value outside the range is clamped to the - boundary instead of failing. This is not supported if either - boundary is marked ``open``. - - .. versionchanged:: 8.0 - Added the ``min_open`` and ``max_open`` parameters. - """ - - name = "float range" - - def __init__( - self, - min: float | None = None, - max: float | None = None, - min_open: bool = False, - max_open: bool = False, - clamp: bool = False, - ) -> None: - super().__init__( - min=min, max=max, min_open=min_open, max_open=max_open, clamp=clamp - ) - - if (min_open or max_open) and clamp: - raise TypeError("Clamping is not supported for open bounds.") - - def _clamp(self, bound: float, dir: t.Literal[1, -1], open: bool) -> float: - if not open: - return bound - - # Could use math.nextafter here, but clamping an - # open float range doesn't seem to be particularly useful. It's - # left up to the user to write a callback to do it if needed. - raise RuntimeError("Clamping is not supported for open bounds.") - - -class BoolParamType(ParamType[bool]): - name = "boolean" - - bool_states: dict[str, bool] = { - "1": True, - "0": False, - "yes": True, - "no": False, - "true": True, - "false": False, - "on": True, - "off": False, - "t": True, - "f": False, - "y": True, - "n": False, - # Absence of value is considered False. - "": False, - } - """A mapping of string values to boolean states. - - Mapping is inspired by :py:attr:`configparser.ConfigParser.BOOLEAN_STATES` - and extends it. - - .. caution:: - String values are lower-cased, as the ``str_to_bool`` comparison function - below is case-insensitive. - - .. warning:: - The mapping is not exhaustive, and does not cover all possible boolean strings - representations. It will remains as it is to avoid endless bikeshedding. - - Future work my be considered to make this mapping user-configurable from public - API. - """ - - @staticmethod - def str_to_bool(value: str | bool) -> bool | None: - """Convert a string to a boolean value. - - If the value is already a boolean, it is returned as-is. If the value is a - string, it is stripped of whitespaces and lower-cased, then checked against - the known boolean states pre-defined in the `BoolParamType.bool_states` mapping - above. - - Returns `None` if the value does not match any known boolean state. - """ - if isinstance(value, bool): - return value - return BoolParamType.bool_states.get(value.strip().lower()) - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> bool: - normalized = self.str_to_bool(value) - if normalized is None: - self.fail( - _( - "{value!r} is not a valid boolean. Recognized values: {states}" - ).format(value=value, states=", ".join(sorted(self.bool_states))), - param, - ctx, - ) - return normalized - - def __repr__(self) -> str: - return "BOOL" - - -class UUIDParameterType(ParamType[uuid.UUID]): - name = "uuid" - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> uuid.UUID: - if isinstance(value, uuid.UUID): - return value - - value = value.strip() - - try: - return uuid.UUID(value) - except ValueError: - self.fail( - _("{value!r} is not a valid UUID.").format(value=value), param, ctx - ) - - def __repr__(self) -> str: - return "UUID" - - -class FileInfoDict(ParamTypeInfoDict): - mode: str - encoding: str | None - - -class File(ParamType[t.IO[t.Any]]): - """Declares a parameter to be a file for reading or writing. The file - is automatically closed once the context tears down (after the command - finished working). - - Files can be opened for reading or writing. The special value ``-`` - indicates stdin or stdout depending on the mode. - - By default, the file is opened for reading text data, but it can also be - opened in binary mode or for writing. The encoding parameter can be used - to force a specific encoding. - - The `lazy` flag controls if the file should be opened immediately or upon - first IO. The default is to be non-lazy for standard input and output - streams as well as files opened for reading, `lazy` otherwise. When opening a - file lazily for reading, it is still opened temporarily for validation, but - will not be held open until first IO. lazy is mainly useful when opening - for writing to avoid creating the file until it is needed. - - Files can also be opened atomically in which case all writes go into a - separate file in the same folder and upon completion the file will - be moved over to the original location. This is useful if a file - regularly read by other users is modified. - - See :ref:`file-args` for more information. - - .. versionchanged:: 2.0 - Added the ``atomic`` parameter. - """ - - name = "filename" - envvar_list_splitter: t.ClassVar[str] = os.path.pathsep - - def __init__( - self, - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - lazy: bool | None = None, - atomic: bool = False, - ) -> None: - self.mode = mode - self.encoding = encoding - self.errors = errors - self.lazy = lazy - self.atomic = atomic - - def to_info_dict(self) -> FileInfoDict: - return { - "mode": self.mode, - "encoding": self.encoding, - **super().to_info_dict(), - } - - def resolve_lazy_flag(self, value: str | os.PathLike[str]) -> bool: - if self.lazy is not None: - return self.lazy - if os.fspath(value) == "-": - return False - elif "w" in self.mode: - return True - return False - - def convert( - self, - value: str | os.PathLike[str] | t.IO[t.Any], - param: Parameter | None, - ctx: Context | None, - ) -> t.IO[t.Any]: - if _is_file_like(value): - return value - - value = t.cast("str | os.PathLike[str]", value) - - try: - lazy = self.resolve_lazy_flag(value) - - if lazy: - lf = LazyFile( - value, self.mode, self.encoding, self.errors, atomic=self.atomic - ) - - if ctx is not None: - ctx.call_on_close(lf.close_intelligently) - - return t.cast("t.IO[t.Any]", lf) - - f, should_close = open_stream( - value, self.mode, self.encoding, self.errors, atomic=self.atomic - ) - - # If a context is provided, we automatically close the file - # at the end of the context execution (or flush out). If a - # context does not exist, it's the caller's responsibility to - # properly close the file. This for instance happens when the - # type is used with prompts. - if ctx is not None: - if should_close: - ctx.call_on_close(safecall(f.close)) - else: - ctx.call_on_close(safecall(f.flush)) - - return f - except OSError as e: - self.fail( - f"'{format_filename(value)}': {e.strerror}", - param, - ctx, - ) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Return a special completion marker that tells the completion - system to use the shell to provide file path completions. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - return [CompletionItem(incomplete, type="file")] - - -def _is_file_like(value: t.Any) -> te.TypeGuard[t.IO[t.Any]]: - return hasattr(value, "read") or hasattr(value, "write") - - -class PathInfoDict(ParamTypeInfoDict): - exists: bool - file_okay: bool - dir_okay: bool - writable: bool - readable: bool - allow_dash: bool - - -class Path(ParamType[str | bytes | os.PathLike[str]]): - """The ``Path`` type is similar to the :class:`File` type, but - returns the filename instead of an open file. Various checks can be - enabled to validate the type of file and permissions. - - :param exists: The file or directory needs to exist for the value to - be valid. If this is not set to ``True``, and the file does not - exist, then all further checks are silently skipped. - :param file_okay: Allow a file as a value. - :param dir_okay: Allow a directory as a value. - :param readable: if true, a readable check is performed. - :param writable: if true, a writable check is performed. - :param executable: if true, an executable check is performed. - :param resolve_path: Make the value absolute and resolve any - symlinks. A ``~`` is not expanded, as this is supposed to be - done by the shell only. - :param allow_dash: Allow a single dash as a value, which indicates - a standard stream (but does not open it). Use - :func:`~click.open_file` to handle opening this value. - :param path_type: Convert the incoming path value to this type. If - ``None``, keep Python's default, which is ``str``. Useful to - convert to :class:`pathlib.Path`. - - .. versionchanged:: 8.1 - Added the ``executable`` parameter. - - .. versionchanged:: 8.0 - Allow passing ``path_type=pathlib.Path``. - - .. versionchanged:: 6.0 - Added the ``allow_dash`` parameter. - """ - - envvar_list_splitter: t.ClassVar[str] = os.path.pathsep - - def __init__( - self, - exists: bool = False, - file_okay: bool = True, - dir_okay: bool = True, - writable: bool = False, - readable: bool = True, - resolve_path: bool = False, - allow_dash: bool = False, - path_type: type[t.Any] | None = None, - executable: bool = False, - ): - self.exists = exists - self.file_okay = file_okay - self.dir_okay = dir_okay - self.readable = readable - self.writable = writable - self.executable = executable - self.resolve_path = resolve_path - self.allow_dash = allow_dash - self.type = path_type - - if self.file_okay and not self.dir_okay: - self.name: str = _("file") - elif self.dir_okay and not self.file_okay: - self.name = _("directory") - else: - self.name = _("path") - - def to_info_dict(self) -> PathInfoDict: - return { - "exists": self.exists, - "file_okay": self.file_okay, - "dir_okay": self.dir_okay, - "writable": self.writable, - "readable": self.readable, - "allow_dash": self.allow_dash, - **super().to_info_dict(), - } - - def coerce_path_result( - self, value: str | os.PathLike[str] - ) -> str | bytes | os.PathLike[str]: - if self.type is not None and not isinstance(value, self.type): - if self.type is str: - return os.fsdecode(value) - elif self.type is bytes: - return os.fsencode(value) - else: - return t.cast("os.PathLike[str]", self.type(value)) - - return value - - def convert( - self, - value: str | os.PathLike[str], - param: Parameter | None, - ctx: Context | None, - ) -> str | bytes | os.PathLike[str]: - rv = value - - is_dash = self.file_okay and self.allow_dash and rv in (b"-", "-") - - if not is_dash: - if self.resolve_path: - rv = os.path.realpath(rv) - - try: - st = os.stat(rv) - except OSError: - if not self.exists: - return self.coerce_path_result(rv) - self.fail( - _("{name} {filename!r} does not exist.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if not self.file_okay and stat.S_ISREG(st.st_mode): - self.fail( - _("{name} {filename!r} is a file.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - if not self.dir_okay and stat.S_ISDIR(st.st_mode): - self.fail( - _("{name} {filename!r} is a directory.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if self.readable and not os.access(rv, os.R_OK): - self.fail( - _("{name} {filename!r} is not readable.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if self.writable and not os.access(rv, os.W_OK): - self.fail( - _("{name} {filename!r} is not writable.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - if self.executable and not os.access(value, os.X_OK): - self.fail( - _("{name} {filename!r} is not executable.").format( - name=self.name.title(), filename=format_filename(value) - ), - param, - ctx, - ) - - return self.coerce_path_result(rv) - - def shell_complete( - self, ctx: Context, param: Parameter, incomplete: str - ) -> list[CompletionItem]: - """Return a special completion marker that tells the completion - system to use the shell to provide path completions for only - directories or any paths. - - :param ctx: Invocation context for this command. - :param param: The parameter that is requesting completion. - :param incomplete: Value being completed. May be empty. - - .. versionadded:: 8.0 - """ - from click.shell_completion import CompletionItem - - type = "dir" if self.dir_okay and not self.file_okay else "file" - return [CompletionItem(incomplete, type=type)] - - -class TupleInfoDict(ParamTypeInfoDict): - types: cabc.Sequence[ParamTypeInfoDict] - - -class Tuple(CompositeParamType[tuple[t.Any, ...]]): - """The default behavior of Click is to apply a type on a value directly. - This works well in most cases, except for when `nargs` is set to a fixed - count and different types should be used for different items. In this - case the :class:`Tuple` type can be used. This type can only be used - if `nargs` is set to a fixed number. - - For more information see :ref:`tuple-type`. - - This can be selected by using a Python tuple literal as a type. - - :param types: a list of types that should be used for the tuple items. - """ - - def __init__(self, types: cabc.Sequence[type[t.Any] | ParamType[t.Any]]) -> None: - self.types: cabc.Sequence[ParamType[t.Any]] = [convert_type(ty) for ty in types] - - def to_info_dict(self) -> TupleInfoDict: - return { - "types": [ty.to_info_dict() for ty in self.types], - **super().to_info_dict(), - } - - @property - def name(self) -> str: # type: ignore[override] - return f"<{' '.join(ty.name for ty in self.types)}>" - - @property - def arity(self) -> int: # type: ignore[override] - return len(self.types) - - def convert( - self, value: t.Any, param: Parameter | None, ctx: Context | None - ) -> tuple[t.Any, ...]: - len_type = len(self.types) - len_value = len(value) - - if len_value != len_type: - self.fail( - ngettext( - "{len_type} values are required, but {len_value} was given.", - "{len_type} values are required, but {len_value} were given.", - len_value, - ).format(len_type=len_type, len_value=len_value), - param=param, - ctx=ctx, - ) - - return tuple( - ty(x, param, ctx) for ty, x in zip(self.types, value, strict=False) - ) - - -def _guess_type( - ty: type[t.Any] | ParamType[t.Any] | None, - default: t.Any | None, -) -> type[t.Any] | tuple[type[t.Any], ...] | ParamType[t.Any] | None: - """Infer a type from *ty* or *default*. - - Returns *ty* unchanged when it is not ``None``. Otherwise inspects - *default* to produce a ``type``, a ``tuple`` of types (for tuple - defaults), or ``None``. - """ - if ty is not None: - return ty - - if default is None: - return None - - if not isinstance(default, (tuple, list)): - return type(default) - - # If the default is empty, return None so convert_type falls - # through to STRING. - if not default: - return None - - item = default[0] - - # A sequence of iterables needs to detect the inner types. - # Can't call convert_type recursively because that would - # incorrectly unwind the tuple to a single type. - if isinstance(item, (tuple, list)): - return tuple(map(type, item)) - - return type(item) - - -@t.overload -def convert_type(ty: None, default: None = None) -> StringParamType: ... - - -@t.overload -def convert_type( - ty: type[t.Any] | ParamType[t.Any], default: t.Any | None = None -) -> ParamType[t.Any]: ... - - -@t.overload -def convert_type( - ty: t.Any | None, default: t.Any | None = None -) -> ParamType[t.Any]: ... - - -def convert_type( - ty: t.Any | None = None, default: t.Any | None = None -) -> ParamType[t.Any]: - """Find the most appropriate :class:`ParamType` for the given Python - type. If the type isn't provided, it can be inferred from a default - value. - """ - guessed = _guess_type(ty, default) - is_guessed = guessed is not ty - - if isinstance(guessed, tuple): - return Tuple(guessed) - - if isinstance(guessed, ParamType): - return guessed - - if guessed is str or guessed is None: - return STRING - - if guessed is int: - return INT - - if guessed is float: - return FLOAT - - if guessed is bool: - return BOOL - - if is_guessed: - return STRING - - if __debug__: - try: - if issubclass(guessed, ParamType): - raise AssertionError( - f"Attempted to use an uninstantiated parameter type ({guessed})." - ) - except TypeError: - # guessed is an instance (correct), so issubclass fails. - pass - - return FuncParamType(guessed) - - -#: A dummy parameter type that just does nothing. From a user's -#: perspective this appears to just be the same as `STRING` but -#: internally no string conversion takes place if the input was bytes. -#: This is usually useful when working with file paths as they can -#: appear in bytes and unicode. -#: -#: For path related uses the :class:`Path` type is a better choice but -#: there are situations where an unprocessed type is useful which is why -#: it is provided. -#: -#: .. versionadded:: 4.0 -UNPROCESSED = UnprocessedParamType() - -#: A unicode string parameter type which is the implicit default. This -#: can also be selected by using ``str`` as type. -STRING = StringParamType() - -#: An integer parameter. This can also be selected by using ``int`` as -#: type. -INT = IntParamType() - -#: A floating point value parameter. This can also be selected by using -#: ``float`` as type. -FLOAT = FloatParamType() - -#: A boolean parameter. This is the default for boolean flags. This can -#: also be selected by using ``bool`` as a type. -BOOL = BoolParamType() - -#: A UUID parameter. -UUID = UUIDParameterType() - - -class OptionHelpExtra(t.TypedDict, total=False): - envvars: tuple[str, ...] - default: str - range: str - required: str diff --git a/functions/python/data-processor/click/utils.py b/functions/python/data-processor/click/utils.py deleted file mode 100644 index 67922b894..000000000 --- a/functions/python/data-processor/click/utils.py +++ /dev/null @@ -1,641 +0,0 @@ -from __future__ import annotations - -import collections.abc as cabc -import os -import re -import sys -import typing as t -from functools import update_wrapper -from gettext import gettext as _ -from types import ModuleType -from types import TracebackType - -from ._compat import _default_text_stderr -from ._compat import _default_text_stdout -from ._compat import _find_binary_writer -from ._compat import auto_wrap_for_ansi -from ._compat import binary_streams -from ._compat import open_stream -from ._compat import should_strip_ansi -from ._compat import strip_ansi -from ._compat import text_streams -from ._compat import WIN -from .globals import resolve_color_default - -if t.TYPE_CHECKING: - import typing_extensions as te - - P = te.ParamSpec("P") - -R = t.TypeVar("R") - - -def _posixify(name: str) -> str: - return "-".join(name.split()).lower() - - -def safecall(func: t.Callable[P, R]) -> t.Callable[P, R | None]: - """Wraps a function so that it swallows exceptions.""" - - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R | None: - try: - return func(*args, **kwargs) - except Exception: - pass - return None - - return update_wrapper(wrapper, func) - - -def make_str(value: t.Any) -> str: - """Converts a value into a valid string.""" - if isinstance(value, bytes): - try: - return value.decode(sys.getfilesystemencoding()) - except UnicodeError: - return value.decode("utf-8", "replace") - return str(value) - - -def make_default_short_help(help: str, max_length: int = 45) -> str: - """Returns a condensed version of help string. - - :meta private: - """ - # Consider only the first paragraph. - paragraph_end = help.find("\n\n") - - if paragraph_end != -1: - help = help[:paragraph_end] - - # Collapse newlines, tabs, and spaces. - words = help.split() - - if not words: - return "" - - # The first paragraph started with a "no rewrap" marker, ignore it. - if words[0] == "\b": - words = words[1:] - - total_length = 0 - last_index = len(words) - 1 - - for i, word in enumerate(words): - total_length += len(word) + (i > 0) - - if total_length > max_length: # too long, truncate - break - - if word[-1] == ".": # sentence end, truncate without "..." - return " ".join(words[: i + 1]) - - if total_length == max_length and i != last_index: - break # not at sentence end, truncate with "..." - else: - return " ".join(words) # no truncation needed - - # Account for the length of the suffix. - total_length += len("...") - - # remove words until the length is short enough - while i > 0: - total_length -= len(words[i]) + (i > 0) - - if total_length <= max_length: - break - - i -= 1 - - return " ".join(words[:i]) + "..." - - -class LazyFile: - """A lazy file works like a regular file but it does not fully open - the file but it does perform some basic checks early to see if the - filename parameter does make sense. This is useful for safely opening - files for writing. - """ - - name: str - mode: str - encoding: str | None - errors: str | None - atomic: bool - _f: t.IO[t.Any] | None - should_close: bool - - def __init__( - self, - filename: str | os.PathLike[str], - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - atomic: bool = False, - ) -> None: - self.name = os.fspath(filename) - self.mode = mode - self.encoding = encoding - self.errors = errors - self.atomic = atomic - - if self.name == "-": - self._f, self.should_close = open_stream(filename, mode, encoding, errors) - else: - if "r" in mode: - # Open and close the file in case we're opening it for - # reading so that we can catch at least some errors in - # some cases early. - open(filename, mode).close() - self._f = None - self.should_close = True - - def __getattr__(self, name: str) -> t.Any: - return getattr(self.open(), name) - - def __repr__(self) -> str: - if self._f is not None: - return repr(self._f) - return f"" - - def open(self) -> t.IO[t.Any]: - """Opens the file if it's not yet open. This call might fail with - a :exc:`FileError`. Not handling this error will produce an error - that Click shows. - """ - if self._f is not None: - return self._f - try: - rv, self.should_close = open_stream( - self.name, self.mode, self.encoding, self.errors, atomic=self.atomic - ) - except OSError as e: - from .exceptions import FileError - - raise FileError(self.name, hint=e.strerror) from e - self._f = rv - return rv - - def close(self) -> None: - """Closes the underlying file, no matter what.""" - if self._f is not None: - self._f.close() - - def close_intelligently(self) -> None: - """This function only closes the file if it was opened by the lazy - file wrapper. For instance this will never close stdin. - """ - if self.should_close: - self.close() - - def __enter__(self) -> LazyFile: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - self.close_intelligently() - - def __iter__(self) -> cabc.Iterator[t.AnyStr]: - self.open() - return iter(self._f) # type: ignore - - -class KeepOpenFile: - _file: t.IO[t.Any] - - def __init__(self, file: t.IO[t.Any]) -> None: - self._file = file - - def __getattr__(self, name: str) -> t.Any: - return getattr(self._file, name) - - def __enter__(self) -> KeepOpenFile: - return self - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - tb: TracebackType | None, - ) -> None: - pass - - def __repr__(self) -> str: - return repr(self._file) - - def __iter__(self) -> cabc.Iterator[t.AnyStr]: - return iter(self._file) - - -def echo( - message: object = None, - file: t.IO[t.Any] | None = None, - nl: bool = True, - err: bool = False, - color: bool | None = None, -) -> None: - """Print a message and newline to stdout or a file. This should be - used instead of :func:`print` because it provides better support - for different data, files, and environments. - - Compared to :func:`print`, this does the following: - - - Ensures that the output encoding is not misconfigured on Linux. - - Supports Unicode in the Windows console. - - Supports writing to binary outputs, and supports writing bytes - to text outputs. - - Supports colors and styles on Windows. - - Removes ANSI color and style codes if the output does not look - like an interactive terminal. - - Always flushes the output. - - :param message: The string or bytes to output. Other objects are - converted to strings. - :param file: The file to write to. Defaults to ``stdout``. - :param err: Write to ``stderr`` instead of ``stdout``. - :param nl: Print a newline after the message. Enabled by default. - :param color: Force showing or hiding colors and other styles. By - default Click will remove color if the output does not look like - an interactive terminal. - - .. versionchanged:: 6.0 - Support Unicode output on the Windows console. Click does not - modify ``sys.stdout``, so ``sys.stdout.write()`` and ``print()`` - will still not support Unicode. - - .. versionchanged:: 4.0 - Added the ``color`` parameter. - - .. versionadded:: 3.0 - Added the ``err`` parameter. - - .. versionchanged:: 2.0 - Support colors on Windows if colorama is installed. - """ - if file is None: - if err: - file = _default_text_stderr() - else: - file = _default_text_stdout() - - # There are no standard streams attached to write to. For example, - # pythonw on Windows. - if file is None: - return - - match message: - case str() | bytes() | bytearray(): - out = message - case None: - out = "" - case _: - out = str(message) - - if nl: - if isinstance(out, str): - out += "\n" - else: - out += b"\n" - - if not out: - file.flush() - return - - # If there is a message and the value looks like bytes, we manually - # need to find the binary stream and write the message in there. - # This is done separately so that most stream types will work as you - # would expect. Eg: you can write to StringIO for other cases. - if isinstance(out, (bytes, bytearray)): - binary_file = _find_binary_writer(file) - if binary_file is not None: - file.flush() - binary_file.write(out) - binary_file.flush() - return - - # ANSI style code support. For no message or bytes, nothing happens. - # When outputting to a file instead of a terminal, strip codes. - else: - color = resolve_color_default(color) - - if should_strip_ansi(file, color): - out = strip_ansi(out) - elif WIN: - if auto_wrap_for_ansi is not None: - file = auto_wrap_for_ansi(file, color) # type: ignore - elif not color: - out = strip_ansi(out) - - file.write(out) # type: ignore - file.flush() - - -def get_binary_stream(name: t.Literal["stdin", "stdout", "stderr"]) -> t.BinaryIO: - """Returns a system stream for byte processing. - - :param name: the name of the stream to open. Valid names are ``'stdin'``, - ``'stdout'`` and ``'stderr'`` - """ - opener = binary_streams.get(name) - if opener is None: - raise TypeError(_("Unknown standard stream '{name}'").format(name=name)) - return opener() - - -def get_text_stream( - name: t.Literal["stdin", "stdout", "stderr"], - encoding: str | None = None, - errors: str | None = "strict", -) -> t.TextIO: - """Returns a system stream for text processing. This usually returns - a wrapped stream around a binary stream returned from - :func:`get_binary_stream` but it also can take shortcuts for already - correctly configured streams. - - :param name: the name of the stream to open. Valid names are ``'stdin'``, - ``'stdout'`` and ``'stderr'`` - :param encoding: overrides the detected default encoding. - :param errors: overrides the default error mode. - """ - opener = text_streams.get(name) - if opener is None: - raise TypeError(_("Unknown standard stream '{name}'").format(name=name)) - return opener(encoding, errors) - - -def open_file( - filename: str | os.PathLike[str], - mode: str = "r", - encoding: str | None = None, - errors: str | None = "strict", - lazy: bool = False, - atomic: bool = False, -) -> t.IO[t.Any]: - """Open a file, with extra behavior to handle ``'-'`` to indicate - a standard stream, lazy open on write, and atomic write. Similar to - the behavior of the :class:`~click.File` param type. - - If ``'-'`` is given to open ``stdout`` or ``stdin``, the stream is - wrapped so that using it in a context manager will not close it. - This makes it possible to use the function without accidentally - closing a standard stream: - - .. code-block:: python - - with open_file(filename) as f: - ... - - :param filename: The name or Path of the file to open, or ``'-'`` for - ``stdin``/``stdout``. - :param mode: The mode in which to open the file. - :param encoding: The encoding to decode or encode a file opened in - text mode. - :param errors: The error handling mode. - :param lazy: Wait to open the file until it is accessed. For read - mode, the file is temporarily opened to raise access errors - early, then closed until it is read again. - :param atomic: Write to a temporary file and replace the given file - on close. - - .. versionadded:: 3.0 - """ - if lazy: - return t.cast( - "t.IO[t.Any]", LazyFile(filename, mode, encoding, errors, atomic=atomic) - ) - - f, should_close = open_stream(filename, mode, encoding, errors, atomic=atomic) - - if not should_close: - f = t.cast("t.IO[t.Any]", KeepOpenFile(f)) - - return f - - -def format_filename( - filename: str | bytes | os.PathLike[str] | os.PathLike[bytes], - shorten: bool = False, -) -> str: - """Format a filename as a string for display. Ensures the filename can be - displayed by replacing any invalid bytes or surrogate escapes in the name - with the replacement character ``�``. - - Invalid bytes or surrogate escapes will raise an error when written to a - stream with ``errors="strict"``. This will typically happen with ``stdout`` - when the locale is something like ``en_GB.UTF-8``. - - Many scenarios *are* safe to write surrogates though, due to PEP 538 and - PEP 540, including: - - - Writing to ``stderr``, which uses ``errors="backslashreplace"``. - - The system has ``LANG=C.UTF-8``, ``C``, or ``POSIX``. Python opens - stdout and stderr with ``errors="surrogateescape"``. - - None of ``LANG/LC_*`` are set. Python assumes ``LANG=C.UTF-8``. - - Python is started in UTF-8 mode with ``PYTHONUTF8=1`` or ``-X utf8``. - Python opens stdout and stderr with ``errors="surrogateescape"``. - - :param filename: formats a filename for UI display. This will also convert - the filename into unicode without failing. - :param shorten: this optionally shortens the filename to strip of the - path that leads up to it. - """ - if shorten: - filename = os.path.basename(filename) - else: - filename = os.fspath(filename) - - if isinstance(filename, bytes): - filename = filename.decode(sys.getfilesystemencoding(), "replace") - else: - filename = filename.encode("utf-8", "surrogateescape").decode( - "utf-8", "replace" - ) - - return filename - - -def get_app_dir(app_name: str, roaming: bool = True, force_posix: bool = False) -> str: - r"""Returns the config folder for the application. The default behavior - is to return whatever is most appropriate for the operating system. - - To give you an idea, for an app called ``"Foo Bar"``, something like - the following folders could be returned: - - Mac OS X: - ``~/Library/Application Support/Foo Bar`` - Mac OS X (POSIX): - ``~/.foo-bar`` - Unix: - ``~/.config/foo-bar`` - Unix (POSIX): - ``~/.foo-bar`` - Windows (roaming): - ``C:\Users\\AppData\Roaming\Foo Bar`` - Windows (not roaming): - ``C:\Users\\AppData\Local\Foo Bar`` - - .. versionadded:: 2.0 - - :param app_name: the application name. This should be properly capitalized - and can contain whitespace. - :param roaming: controls if the folder should be roaming or not on Windows. - Has no effect otherwise. - :param force_posix: if this is set to `True` then on any POSIX system the - folder will be stored in the home folder with a leading - dot instead of the XDG config home or darwin's - application support folder. - """ - if WIN: - key = "APPDATA" if roaming else "LOCALAPPDATA" - folder = os.environ.get(key) - if folder is None: - folder = os.path.expanduser("~") - return os.path.join(folder, app_name) - if force_posix: - return os.path.join(os.path.expanduser(f"~/.{_posixify(app_name)}")) - if sys.platform == "darwin": - return os.path.join( - os.path.expanduser("~/Library/Application Support"), app_name - ) - return os.path.join( - os.environ.get("XDG_CONFIG_HOME", os.path.expanduser("~/.config")), - _posixify(app_name), - ) - - -class PacifyFlushWrapper: - """This wrapper is used to catch and suppress BrokenPipeErrors resulting - from ``.flush()`` being called on broken pipe during the shutdown/final-GC - of the Python interpreter. Notably ``.flush()`` is always called on - ``sys.stdout`` and ``sys.stderr``. So as to have minimal impact on any - other cleanup code, and the case where the underlying file is not a broken - pipe, all calls and attributes are proxied. - """ - - wrapped: t.IO[t.Any] - - def __init__(self, wrapped: t.IO[t.Any]) -> None: - self.wrapped = wrapped - - def flush(self) -> None: - try: - self.wrapped.flush() - except OSError as e: - import errno - - if e.errno != errno.EPIPE: - raise - - def __getattr__(self, attr: str) -> t.Any: - return getattr(self.wrapped, attr) - - -def _detect_program_name( - path: str | None = None, _main: ModuleType | None = None -) -> str: - """Determine the command used to run the program, for use in help - text. If a file or entry point was executed, the file name is - returned. If ``python -m`` was used to execute a module or package, - ``python -m name`` is returned. - - This doesn't try to be too precise, the goal is to give a concise - name for help text. Files are only shown as their name without the - path. ``python`` is only shown for modules, and the full path to - ``sys.executable`` is not shown. - - :param path: The Python file being executed. Python puts this in - ``sys.argv[0]``, which is used by default. - :param _main: The ``__main__`` module. This should only be passed - during internal testing. - - .. versionadded:: 8.0 - Based on command args detection in the Werkzeug reloader. - - :meta private: - """ - if _main is None: - _main = sys.modules["__main__"] - - if not path: - path = sys.argv[0] - - # The value of __package__ indicates how Python was called. It may - # not exist if a setuptools script is installed as an egg. It may be - # set incorrectly for entry points created with pip on Windows. - # It is set to "" inside a Shiv or PEX zipapp. - if getattr(_main, "__package__", None) in {None, ""} or ( - os.name == "nt" - and _main.__package__ == "" - and not os.path.exists(path) - and os.path.exists(f"{path}.exe") - ): - # Executed a file, like "python app.py". - return os.path.basename(path) - - # Executed a module, like "python -m example". - # Rewritten by Python from "-m script" to "/path/to/script.py". - # Need to look at main module to determine how it was executed. - py_module = t.cast(str, _main.__package__) - name = os.path.splitext(os.path.basename(path))[0] - - # A submodule like "example.cli". - if name != "__main__": - py_module = f"{py_module}.{name}" - - return f"python -m {py_module.lstrip('.')}" - - -def _expand_args( - args: cabc.Iterable[str], - *, - user: bool = True, - env: bool = True, - glob_recursive: bool = True, -) -> list[str]: - """Simulate Unix shell expansion with Python functions. - - See :func:`glob.glob`, :func:`os.path.expanduser`, and - :func:`os.path.expandvars`. - - This is intended for use on Windows, where the shell does not do any - expansion. It may not exactly match what a Unix shell would do. - - :param args: List of command line arguments to expand. - :param user: Expand user home directory. - :param env: Expand environment variables. - :param glob_recursive: ``**`` matches directories recursively. - - .. versionchanged:: 8.1 - Invalid glob patterns are treated as empty expansions rather - than raising an error. - - .. versionadded:: 8.0 - - :meta private: - """ - from glob import glob - - out = [] - - for arg in args: - if user: - arg = os.path.expanduser(arg) - - if env: - arg = os.path.expandvars(arg) - - try: - matches = glob(arg, recursive=glob_recursive) - except re.error: - matches = [] - - if not matches: - out.append(arg) - else: - out.extend(matches) - - return out diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/INSTALLER b/functions/python/data-processor/fastapi-0.136.3.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/fastapi-0.136.3.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/METADATA b/functions/python/data-processor/fastapi-0.136.3.dist-info/METADATA deleted file mode 100644 index d044e6b2a..000000000 --- a/functions/python/data-processor/fastapi-0.136.3.dist-info/METADATA +++ /dev/null @@ -1,618 +0,0 @@ -Metadata-Version: 2.4 -Name: fastapi -Version: 0.136.3 -Summary: FastAPI framework, high performance, easy to learn, fast to code, ready for production -Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= -License-Expression: MIT -License-File: LICENSE -Classifier: Intended Audience :: Information Technology -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Application Frameworks -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: Software Development -Classifier: Typing :: Typed -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Web Environment -Classifier: Framework :: AsyncIO -Classifier: Framework :: FastAPI -Classifier: Framework :: Pydantic -Classifier: Framework :: Pydantic :: 2 -Classifier: Intended Audience :: Developers -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers -Classifier: Topic :: Internet :: WWW/HTTP -Project-URL: Homepage, https://github.com/fastapi/fastapi -Project-URL: Documentation, https://fastapi.tiangolo.com/ -Project-URL: Repository, https://github.com/fastapi/fastapi -Project-URL: Issues, https://github.com/fastapi/fastapi/issues -Project-URL: Changelog, https://fastapi.tiangolo.com/release-notes/ -Requires-Python: >=3.10 -Requires-Dist: starlette>=0.46.0 -Requires-Dist: pydantic>=2.9.0 -Requires-Dist: typing-extensions>=4.8.0 -Requires-Dist: typing-inspection>=0.4.2 -Requires-Dist: annotated-doc>=0.0.2 -Provides-Extra: standard -Requires-Dist: fastapi-cli[standard]>=0.0.8; extra == "standard" -Requires-Dist: fastar>=0.9.0; extra == "standard" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "standard" -Requires-Dist: jinja2>=3.1.5; extra == "standard" -Requires-Dist: python-multipart>=0.0.18; extra == "standard" -Requires-Dist: email-validator>=2.0.0; extra == "standard" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard" -Requires-Dist: pydantic-settings>=2.0.0; extra == "standard" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "standard" -Provides-Extra: standard-no-fastapi-cloud-cli -Requires-Dist: fastapi-cli[standard-no-fastapi-cloud-cli]>=0.0.8; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: jinja2>=3.1.5; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: python-multipart>=0.0.18; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: email-validator>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: pydantic-settings>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Provides-Extra: all -Requires-Dist: fastapi-cli[standard]>=0.0.8; extra == "all" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "all" -Requires-Dist: jinja2>=3.1.5; extra == "all" -Requires-Dist: python-multipart>=0.0.18; extra == "all" -Requires-Dist: itsdangerous>=1.1.0; extra == "all" -Requires-Dist: pyyaml>=5.3.1; extra == "all" -Requires-Dist: email-validator>=2.0.0; extra == "all" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "all" -Requires-Dist: pydantic-settings>=2.0.0; extra == "all" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "all" -Description-Content-Type: text/markdown - -

- FastAPI -

-

- FastAPI framework, high performance, easy to learn, fast to code, ready for production -

-

- - Test - - - Coverage - - - Package version - - - Supported Python versions - -

- ---- - -**Documentation**: [https://fastapi.tiangolo.com](https://fastapi.tiangolo.com) - -**Source Code**: [https://github.com/fastapi/fastapi](https://github.com/fastapi/fastapi) - ---- - -FastAPI is a modern, fast (high-performance), web framework for building APIs with Python based on standard Python type hints. - -The key features are: - -* **Fast**: Very high performance, on par with **NodeJS** and **Go** (thanks to Starlette and Pydantic). [One of the fastest Python frameworks available](#performance). -* **Fast to code**: Increase the speed to develop features by about 200% to 300%. * -* **Fewer bugs**: Reduce about 40% of human (developer) induced errors. * -* **Intuitive**: Great editor support. Completion everywhere. Less time debugging. -* **Easy**: Designed to be easy to use and learn. Less time reading docs. -* **Short**: Minimize code duplication. Multiple features from each parameter declaration. Fewer bugs. -* **Robust**: Get production-ready code. With automatic interactive documentation. -* **Standards-based**: Based on (and fully compatible with) the open standards for APIs: [OpenAPI](https://github.com/OAI/OpenAPI-Specification) (previously known as Swagger) and [JSON Schema](https://json-schema.org/). - -* estimation based on tests conducted by an internal development team, building production applications. - -## Sponsors - - -### Keystone Sponsor - - - -### Gold Sponsors - - - - - - - - - - - - -### Silver Sponsors - - - - - - - - - - - -[Other sponsors](https://fastapi.tiangolo.com/fastapi-people/#sponsors) - -## Opinions - - - -
- -"_[...] I'm using **FastAPI** a ton these days. [...] I'm actually planning to use it for all of my team's **ML services at Microsoft**. Some of them are getting integrated into the core **Windows** product and some **Office** products._" - -
Kabir Khan - Microsoft (ref)
- ---- - -"_We adopted the **FastAPI** library to spawn a **REST** server that can be queried to obtain **predictions**. [for Ludwig]_" - -
Piero Molino, Yaroslav Dudin, and Sai Sumanth Miryala - Uber (ref)
- ---- - -"_**Netflix** is pleased to announce the open-source release of our **crisis management** orchestration framework: **Dispatch**! [built with **FastAPI**]_" - -
Kevin Glisson, Marc Vilanova, Forest Monsen - Netflix (ref)
- ---- - -"_If anyone is looking to build a production Python API, I would highly recommend **FastAPI**. It is **beautifully designed**, **simple to use** and **highly scalable**, it has become a **key component** in our API first development strategy and is driving many automations and services such as our Virtual TAC Engineer._" - -
Deon Pillsbury - Cisco (ref)
- ---- - -
- -## FastAPI Conf - -[**FastAPI Conf '26**](https://fastapiconf.com) is happening on **October 28, 2026** in **Amsterdam, NL**. All about FastAPI, right from the source. 🎤 - -FastAPI Conf '26 - October 28, 2026 - Amsterdam, NL - -## FastAPI mini documentary - -There's a [FastAPI mini documentary](https://www.youtube.com/watch?v=mpR8ngthqiE) released at the end of 2025, you can watch it online: - -FastAPI Mini Documentary - -## **Typer**, the FastAPI of CLIs - - - -If you are building a CLI app to be used in the terminal instead of a web API, check out [**Typer**](https://typer.tiangolo.com/). - -**Typer** is FastAPI's little sibling. And it's intended to be the **FastAPI of CLIs**. ⌨️ 🚀 - -## Requirements - -FastAPI stands on the shoulders of giants: - -* [Starlette](https://www.starlette.dev/) for the web parts. -* [Pydantic](https://docs.pydantic.dev/) for the data parts. - -## Installation - -Create and activate a [virtual environment](https://fastapi.tiangolo.com/virtual-environments/) and then install FastAPI: - -
- -```console -$ pip install "fastapi[standard]" - ----> 100% -``` - -
- -**Note**: Make sure you put `"fastapi[standard]"` in quotes to ensure it works in all terminals. - -## Example - -### Create it - -Create a file `main.py` with: - -```Python -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} -``` - -
-Or use async def... - -If your code uses `async` / `await`, use `async def`: - -```Python hl_lines="7 12" -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -async def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -async def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} -``` - -**Note**: - -If you don't know, check the _"In a hurry?"_ section about [`async` and `await` in the docs](https://fastapi.tiangolo.com/async/#in-a-hurry). - -
- -### Run it - -Run the server with: - -
- -```console -$ fastapi dev - - ╭────────── FastAPI CLI - Development mode ───────────╮ - │ │ - │ Serving at: http://127.0.0.1:8000 │ - │ │ - │ API docs: http://127.0.0.1:8000/docs │ - │ │ - │ Running in development mode, for production use: │ - │ │ - │ fastapi run │ - │ │ - ╰─────────────────────────────────────────────────────╯ - -INFO: Will watch for changes in these directories: ['/home/user/code/awesomeapp'] -INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) -INFO: Started reloader process [2248755] using WatchFiles -INFO: Started server process [2248757] -INFO: Waiting for application startup. -INFO: Application startup complete. -``` - -
- -
-About the command fastapi dev... - -The command `fastapi dev` reads your `main.py` file automatically, detects the **FastAPI** app in it, and starts a server using [Uvicorn](https://www.uvicorn.dev). - -By default, `fastapi dev` will start with auto-reload enabled for local development. - -You can read more about it in the [FastAPI CLI docs](https://fastapi.tiangolo.com/fastapi-cli/). - -
- -### Check it - -Open your browser at [http://127.0.0.1:8000/items/5?q=somequery](http://127.0.0.1:8000/items/5?q=somequery). - -You will see the JSON response as: - -```JSON -{"item_id": 5, "q": "somequery"} -``` - -You already created an API that: - -* Receives HTTP requests in the _paths_ `/` and `/items/{item_id}`. -* Both _paths_ take `GET` operations (also known as HTTP _methods_). -* The _path_ `/items/{item_id}` has a _path parameter_ `item_id` that should be an `int`. -* The _path_ `/items/{item_id}` has an optional `str` _query parameter_ `q`. - -### Interactive API docs - -Now go to [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs). - -You will see the automatic interactive API documentation (provided by [Swagger UI](https://github.com/swagger-api/swagger-ui)): - -![Swagger UI](https://fastapi.tiangolo.com/img/index/index-01-swagger-ui-simple.png) - -### Alternative API docs - -And now, go to [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc). - -You will see the alternative automatic documentation (provided by [ReDoc](https://github.com/Rebilly/ReDoc)): - -![ReDoc](https://fastapi.tiangolo.com/img/index/index-02-redoc-simple.png) - -## Example upgrade - -Now modify the file `main.py` to receive a body from a `PUT` request. - -Declare the body using standard Python types, thanks to Pydantic. - -```Python hl_lines="2 7-10 23-25" -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - price: float - is_offer: bool | None = None - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} - - -@app.put("/items/{item_id}") -def update_item(item_id: int, item: Item): - return {"item_name": item.name, "item_id": item_id} -``` - -The `fastapi dev` server should reload automatically. - -### Interactive API docs upgrade - -Now go to [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs). - -* The interactive API documentation will be automatically updated, including the new body: - -![Swagger UI](https://fastapi.tiangolo.com/img/index/index-03-swagger-02.png) - -* Click on the button "Try it out", it allows you to fill the parameters and directly interact with the API: - -![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-04-swagger-03.png) - -* Then click on the "Execute" button, the user interface will communicate with your API, send the parameters, get the results and show them on the screen: - -![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-05-swagger-04.png) - -### Alternative API docs upgrade - -And now, go to [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc). - -* The alternative documentation will also reflect the new query parameter and body: - -![ReDoc](https://fastapi.tiangolo.com/img/index/index-06-redoc-02.png) - -### Recap - -In summary, you declare **once** the types of parameters, body, etc. as function parameters. - -You do that with standard modern Python types. - -You don't have to learn a new syntax, the methods or classes of a specific library, etc. - -Just standard **Python**. - -For example, for an `int`: - -```Python -item_id: int -``` - -or for a more complex `Item` model: - -```Python -item: Item -``` - -...and with that single declaration you get: - -* Editor support, including: - * Completion. - * Type checks. -* Validation of data: - * Automatic and clear errors when the data is invalid. - * Validation even for deeply nested JSON objects. -* Conversion of input data: coming from the network to Python data and types. Reading from: - * JSON. - * Path parameters. - * Query parameters. - * Cookies. - * Headers. - * Forms. - * Files. -* Conversion of output data: converting from Python data and types to network data (as JSON): - * Convert Python types (`str`, `int`, `float`, `bool`, `list`, etc). - * `datetime` objects. - * `UUID` objects. - * Database models. - * ...and many more. -* Automatic interactive API documentation, including 2 alternative user interfaces: - * Swagger UI. - * ReDoc. - ---- - -Coming back to the previous code example, **FastAPI** will: - -* Validate that there is an `item_id` in the path for `GET` and `PUT` requests. -* Validate that the `item_id` is of type `int` for `GET` and `PUT` requests. - * If it is not, the client will see a useful, clear error. -* Check if there is an optional query parameter named `q` (as in `http://127.0.0.1:8000/items/foo?q=somequery`) for `GET` requests. - * As the `q` parameter is declared with `= None`, it is optional. - * Without the `None` it would be required (as is the body in the case with `PUT`). -* For `PUT` requests to `/items/{item_id}`, read the body as JSON: - * Check that it has a required attribute `name` that should be a `str`. - * Check that it has a required attribute `price` that has to be a `float`. - * Check that it has an optional attribute `is_offer`, that should be a `bool`, if present. - * All this would also work for deeply nested JSON objects. -* Convert from and to JSON automatically. -* Document everything with OpenAPI, that can be used by: - * Interactive documentation systems. - * Automatic client code generation systems, for many languages. -* Provide 2 interactive documentation web interfaces directly. - ---- - -We just scratched the surface, but you already get the idea of how it all works. - -Try changing the line with: - -```Python - return {"item_name": item.name, "item_id": item_id} -``` - -...from: - -```Python - ... "item_name": item.name ... -``` - -...to: - -```Python - ... "item_price": item.price ... -``` - -...and see how your editor will auto-complete the attributes and know their types: - -![editor support](https://fastapi.tiangolo.com/img/vscode-completion.png) - -For a more complete example including more features, see the Tutorial - User Guide. - -**Spoiler alert**: the tutorial - user guide includes: - -* Declaration of **parameters** from other different places as: **headers**, **cookies**, **form fields** and **files**. -* How to set **validation constraints** as `maximum_length` or `regex`. -* A very powerful and easy to use **Dependency Injection** system. -* Security and authentication, including support for **OAuth2** with **JWT tokens** and **HTTP Basic** auth. -* More advanced (but equally easy) techniques for declaring **deeply nested JSON models** (thanks to Pydantic). -* **GraphQL** integration with [Strawberry](https://strawberry.rocks) and other libraries. -* Many extra features (thanks to Starlette) as: - * **WebSockets** - * extremely easy tests based on HTTPX and `pytest` - * **CORS** - * **Cookie Sessions** - * ...and more. - -### Deploy your app (optional) - -You can optionally deploy your FastAPI app to [FastAPI Cloud](https://fastapicloud.com), go and join the waiting list if you haven't. 🚀 - -If you already have a **FastAPI Cloud** account (we invited you from the waiting list 😉), you can deploy your application with one command. - -
- -```console -$ fastapi deploy - -Deploying to FastAPI Cloud... - -✅ Deployment successful! - -🐔 Ready the chicken! Your app is ready at https://myapp.fastapicloud.dev -``` - -
- -That's it! Now you can access your app at that URL. ✨ - -#### About FastAPI Cloud - -**[FastAPI Cloud](https://fastapicloud.com)** is built by the same author and team behind **FastAPI**. - -It streamlines the process of **building**, **deploying**, and **accessing** an API with minimal effort. - -It brings the same **developer experience** of building apps with FastAPI to **deploying** them to the cloud. 🎉 - -FastAPI Cloud is the primary sponsor and funding provider for the *FastAPI and friends* open source projects. ✨ - -#### Deploy to other cloud providers - -FastAPI is open source and based on standards. You can deploy FastAPI apps to any cloud provider you choose. - -Follow your cloud provider's guides to deploy FastAPI apps with them. 🤓 - -## Performance - -Independent TechEmpower benchmarks show **FastAPI** applications running under Uvicorn as [one of the fastest Python frameworks available](https://www.techempower.com/benchmarks/#section=test&runid=7464e520-0dc2-473d-bd34-dbdfd7e85911&hw=ph&test=query&l=zijzen-7), only below Starlette and Uvicorn themselves (used internally by FastAPI). (*) - -To understand more about it, see the section [Benchmarks](https://fastapi.tiangolo.com/benchmarks/). - -## Dependencies - -FastAPI depends on Pydantic and Starlette. - -### `standard` Dependencies - -When you install FastAPI with `pip install "fastapi[standard]"` it comes with the `standard` group of optional dependencies: - -Used by Pydantic: - -* [`email-validator`](https://github.com/JoshData/python-email-validator) - for email validation. - -Used by Starlette: - -* [`httpx`](https://www.python-httpx.org) - Required if you want to use the `TestClient`. -* [`jinja2`](https://jinja.palletsprojects.com) - Required if you want to use the default template configuration. -* [`python-multipart`](https://github.com/Kludex/python-multipart) - Required if you want to support form "parsing", with `request.form()`. - -Used by FastAPI: - -* [`uvicorn`](https://www.uvicorn.dev) - for the server that loads and serves your application. This includes `uvicorn[standard]`, which includes some dependencies (e.g. `uvloop`) needed for high performance serving. -* `fastapi-cli[standard]` - to provide the `fastapi` command. - * This includes `fastapi-cloud-cli`, which allows you to deploy your FastAPI application to [FastAPI Cloud](https://fastapicloud.com). - -### Without `standard` Dependencies - -If you don't want to include the `standard` optional dependencies, you can install with `pip install fastapi` instead of `pip install "fastapi[standard]"`. - -### Without `fastapi-cloud-cli` - -If you want to install FastAPI with the standard dependencies but without the `fastapi-cloud-cli`, you can install with `pip install "fastapi[standard-no-fastapi-cloud-cli]"`. - -### Additional Optional Dependencies - -There are some additional dependencies you might want to install. - -Additional optional Pydantic dependencies: - -* [`pydantic-settings`](https://docs.pydantic.dev/latest/usage/pydantic_settings/) - for settings management. -* [`pydantic-extra-types`](https://docs.pydantic.dev/latest/usage/types/extra_types/extra_types/) - for extra types to be used with Pydantic. - -Additional optional FastAPI dependencies: - -* [`orjson`](https://github.com/ijl/orjson) - Required if you want to use `ORJSONResponse`. -* [`ujson`](https://github.com/esnme/ultrajson) - Required if you want to use `UJSONResponse`. - -## License - -This project is licensed under the terms of the MIT license. diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/RECORD b/functions/python/data-processor/fastapi-0.136.3.dist-info/RECORD deleted file mode 100644 index 2c8d6baea..000000000 --- a/functions/python/data-processor/fastapi-0.136.3.dist-info/RECORD +++ /dev/null @@ -1,109 +0,0 @@ -../../bin/fastapi,sha256=CrDjk8rnrVWs3h6pixOsNJ3Jn_6tnao51ZaI-jmAi7s,186 -fastapi-0.136.3.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -fastapi-0.136.3.dist-info/METADATA,sha256=A3C-9XpE7xiBQxP6yf-nt8f50hEvpNbkkTTML4WfabQ,27647 -fastapi-0.136.3.dist-info/RECORD,, -fastapi-0.136.3.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi-0.136.3.dist-info/WHEEL,sha256=Z36eTX6lG3PITRleSd5hAZHCcz52yg3c0JQVxKBbLW0,90 -fastapi-0.136.3.dist-info/entry_points.txt,sha256=GCf-WbIZxyGT4MUmrPGj1cOHYZoGsNPHAvNkT6hnGeA,61 -fastapi-0.136.3.dist-info/licenses/LICENSE,sha256=Tsif_IFIW5f-xYSy1KlhAy7v_oNEU4lP2cEnSQbMdE4,1086 -fastapi/.agents/skills/fastapi/SKILL.md,sha256=PIsrztMiIFHK1gMtBsrkrEDooTdiVeMI0sb0Bc8uo1U,10388 -fastapi/.agents/skills/fastapi/references/dependencies.md,sha256=KiNTXAVU6AAirx6kI0iF6Wf265hjUvDGMiIlRPvjcVc,3263 -fastapi/.agents/skills/fastapi/references/other-tools.md,sha256=EbpvhDr9TUuRmQHqaD7vflRIkmfVhHbygTpwC2mREj4,1526 -fastapi/.agents/skills/fastapi/references/streaming.md,sha256=bHaIKnwbTkd7TUVQm_uxapUnAlEG4rdvALV9koD5ypI,2581 -fastapi/__init__.py,sha256=epmuTVX6u4SymDjYeDwmzlCBRB8GKjXOjkI0dLVr3iw,1081 -fastapi/__main__.py,sha256=bKePXLdO4SsVSM6r9SVoLickJDcR2c0cTOxZRKq26YQ,37 -fastapi/__pycache__/__init__.cpython-314.pyc,, -fastapi/__pycache__/__main__.cpython-314.pyc,, -fastapi/__pycache__/applications.cpython-314.pyc,, -fastapi/__pycache__/background.cpython-314.pyc,, -fastapi/__pycache__/cli.cpython-314.pyc,, -fastapi/__pycache__/concurrency.cpython-314.pyc,, -fastapi/__pycache__/datastructures.cpython-314.pyc,, -fastapi/__pycache__/encoders.cpython-314.pyc,, -fastapi/__pycache__/exception_handlers.cpython-314.pyc,, -fastapi/__pycache__/exceptions.cpython-314.pyc,, -fastapi/__pycache__/logger.cpython-314.pyc,, -fastapi/__pycache__/param_functions.cpython-314.pyc,, -fastapi/__pycache__/params.cpython-314.pyc,, -fastapi/__pycache__/requests.cpython-314.pyc,, -fastapi/__pycache__/responses.cpython-314.pyc,, -fastapi/__pycache__/routing.cpython-314.pyc,, -fastapi/__pycache__/sse.cpython-314.pyc,, -fastapi/__pycache__/staticfiles.cpython-314.pyc,, -fastapi/__pycache__/templating.cpython-314.pyc,, -fastapi/__pycache__/testclient.cpython-314.pyc,, -fastapi/__pycache__/types.cpython-314.pyc,, -fastapi/__pycache__/utils.cpython-314.pyc,, -fastapi/__pycache__/websockets.cpython-314.pyc,, -fastapi/_compat/__init__.py,sha256=PYOR-8vJ5va4Qjl810FcQmJbqmpyyeQMoZ9R86CeE2U,2095 -fastapi/_compat/__pycache__/__init__.cpython-314.pyc,, -fastapi/_compat/__pycache__/shared.cpython-314.pyc,, -fastapi/_compat/__pycache__/v2.cpython-314.pyc,, -fastapi/_compat/shared.py,sha256=XJl8cVYOTSgfK-1fiJHLeHHeSGLfC00H7MrtpyYpQgU,7118 -fastapi/_compat/v2.py,sha256=sDGyi1iKSFW9LuJ7n4B61-1yrQI1NHRSwsFRRLuFJ8k,17601 -fastapi/applications.py,sha256=1R8FZK22pzBCsCyLx4PzQTdRzHKgPoM3odR4Rsv_AzY,181285 -fastapi/background.py,sha256=TADzAethOAaqpVvckYuTT3c4O9N1HaFQysFCPt0MsgU,1820 -fastapi/cli.py,sha256=OYhZb0NR_deuT5ofyPF2NoNBzZDNOP8Salef2nk-HqA,418 -fastapi/concurrency.py,sha256=xHGDEOQAA6cvFEDX46oq3r2t1Zd4sVvreaRgdIE4juM,1489 -fastapi/datastructures.py,sha256=XPugnojHc4N07eF5Xp1TTndP2gMv1fG4k-0L4Vdr7Kc,5321 -fastapi/dependencies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/dependencies/__pycache__/__init__.cpython-314.pyc,, -fastapi/dependencies/__pycache__/models.cpython-314.pyc,, -fastapi/dependencies/__pycache__/utils.cpython-314.pyc,, -fastapi/dependencies/models.py,sha256=XPIkEaukHaTaFz9LoeI9f_T-3otOAVkoOC113TpFbWI,7250 -fastapi/dependencies/utils.py,sha256=XIpRML7ttx5EhmchgWz6R5CMsBuepx7KHdmi38F_e_s,39728 -fastapi/encoders.py,sha256=m2OGnq9k94Alcd6UQM8r7iwn-cjvJdpOKRVJuQR68T8,11603 -fastapi/exception_handlers.py,sha256=YVcT8Zy021VYYeecgdyh5YEUjEIHKcLspbkSf4OfbJI,1275 -fastapi/exceptions.py,sha256=fUNOBRdIsULU0TnO8aNBg3HoJeyJxXp9UV6D8wYu9lI,7453 -fastapi/logger.py,sha256=I9NNi3ov8AcqbsbC9wl1X-hdItKgYt2XTrx1f99Zpl4,54 -fastapi/middleware/__init__.py,sha256=oQDxiFVcc1fYJUOIFvphnK7pTT5kktmfL32QXpBFvvo,58 -fastapi/middleware/__pycache__/__init__.cpython-314.pyc,, -fastapi/middleware/__pycache__/asyncexitstack.cpython-314.pyc,, -fastapi/middleware/__pycache__/cors.cpython-314.pyc,, -fastapi/middleware/__pycache__/gzip.cpython-314.pyc,, -fastapi/middleware/__pycache__/httpsredirect.cpython-314.pyc,, -fastapi/middleware/__pycache__/trustedhost.cpython-314.pyc,, -fastapi/middleware/__pycache__/wsgi.cpython-314.pyc,, -fastapi/middleware/asyncexitstack.py,sha256=RKGlQpGzg3GLosqVhrxBy_NCZ9qJS7zQeNHt5Y3x-00,637 -fastapi/middleware/cors.py,sha256=ynwjWQZoc_vbhzZ3_ZXceoaSrslHFHPdoM52rXr0WUU,79 -fastapi/middleware/gzip.py,sha256=xM5PcsH8QlAimZw4VDvcmTnqQamslThsfe3CVN2voa0,79 -fastapi/middleware/httpsredirect.py,sha256=rL8eXMnmLijwVkH7_400zHri1AekfeBd6D6qs8ix950,115 -fastapi/middleware/trustedhost.py,sha256=eE5XGRxGa7c5zPnMJDGp3BxaL25k5iVQlhnv-Pk0Pss,109 -fastapi/middleware/wsgi.py,sha256=a_FMDoeTwcdig9wdAGumIH82oDFfuj4pxtQxLbAw2Ns,107 -fastapi/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/openapi/__pycache__/__init__.cpython-314.pyc,, -fastapi/openapi/__pycache__/constants.cpython-314.pyc,, -fastapi/openapi/__pycache__/docs.cpython-314.pyc,, -fastapi/openapi/__pycache__/models.cpython-314.pyc,, -fastapi/openapi/__pycache__/utils.cpython-314.pyc,, -fastapi/openapi/constants.py,sha256=adGzmis1L1HJRTE3kJ5fmHS_Noq6tIY6pWv_SFzoFDU,153 -fastapi/openapi/docs.py,sha256=PcjH0Sn-yp97O0exXincG8epsY7ATyJC5662bK1DRao,12425 -fastapi/openapi/models.py,sha256=twjJWGf6lcKRJZgaZCcnQ0Ten1hDhcyCLgbr7tXIP30,14608 -fastapi/openapi/utils.py,sha256=kgF0fQ5Sd_NlxIE3QzEzKOkHjmhDSJTSYkdUOtkb6oA,26240 -fastapi/param_functions.py,sha256=4fTCVlvEDbAQr7gvWYkOX8hktdl2Iwk-kv4FaTz7P8w,69596 -fastapi/params.py,sha256=1fNNSK5J7PM5Fw-F5_9uOysA2NgC-61mb3LlrBWpqJM,26205 -fastapi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/requests.py,sha256=zayepKFcienBllv3snmWI20Gk0oHNVLU4DDhqXBb4LU,142 -fastapi/responses.py,sha256=BM7JtiZ_G4j2Z7QvHUzV1KtZPnjOoCfX35RqYXlZDh0,4144 -fastapi/routing.py,sha256=dphfjEJRFgaOsdJV74vPq7bP-CaM02T6LHUZbPv61qE,197568 -fastapi/security/__init__.py,sha256=bO8pNmxqVRXUjfl2mOKiVZLn0FpBQ61VUYVjmppnbJw,881 -fastapi/security/__pycache__/__init__.cpython-314.pyc,, -fastapi/security/__pycache__/api_key.cpython-314.pyc,, -fastapi/security/__pycache__/base.cpython-314.pyc,, -fastapi/security/__pycache__/http.cpython-314.pyc,, -fastapi/security/__pycache__/oauth2.cpython-314.pyc,, -fastapi/security/__pycache__/open_id_connect_url.cpython-314.pyc,, -fastapi/security/__pycache__/utils.cpython-314.pyc,, -fastapi/security/api_key.py,sha256=4CNLNVAStOsMhytH9C5EOUEOZrtLg_IpMQS_HcRDP4M,9793 -fastapi/security/base.py,sha256=dl4pvbC-RxjfbWgPtCWd8MVU-7CB2SZ22rJDXVCXO6c,141 -fastapi/security/http.py,sha256=Z0xALDqwgJZRAaDs40Sa68rAnjFzEL99UEmO5PJzTKA,13410 -fastapi/security/oauth2.py,sha256=sSqW4tbvoHaWNld46TYatta5rgFfjod4LhxmedXzkI8,24178 -fastapi/security/open_id_connect_url.py,sha256=V8WLPEsEq_WJlIjPwJO2vCoJWGLu-VTt1-N2H7aV-D4,3136 -fastapi/security/utils.py,sha256=E9YIoez-H2k1oBLEdxqJEi8sV1umunJYKjQHvZG3FRY,261 -fastapi/sse.py,sha256=adtwP03ePndMQOuRCNRHrTfFQxMpRwrSUoRMHyB8OvA,6848 -fastapi/staticfiles.py,sha256=iirGIt3sdY2QZXd36ijs3Cj-T0FuGFda3cd90kM9Ikw,69 -fastapi/templating.py,sha256=4zsuTWgcjcEainMJFAlW6-gnslm6AgOS1SiiDWfmQxk,76 -fastapi/testclient.py,sha256=nBvaAmX66YldReJNZXPOk1sfuo2Q6hs8bOvIaCep6LQ,66 -fastapi/types.py,sha256=g2tD842BUHC2C3_P8P06albQ4MhCb9RybrSmp5rODgU,438 -fastapi/utils.py,sha256=DX0VrnMwfVsZxRz8IitQ42c2---fDzmFZkeRMZR2UMo,4341 -fastapi/websockets.py,sha256=419uncYObEKZG0YcrXscfQQYLSWoE10jqxVMetGdR98,222 diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/REQUESTED b/functions/python/data-processor/fastapi-0.136.3.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/WHEEL b/functions/python/data-processor/fastapi-0.136.3.dist-info/WHEEL deleted file mode 100644 index 82d3df6a7..000000000 --- a/functions/python/data-processor/fastapi-0.136.3.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: pdm-backend (2.4.8) -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/entry_points.txt b/functions/python/data-processor/fastapi-0.136.3.dist-info/entry_points.txt deleted file mode 100644 index b81849e1e..000000000 --- a/functions/python/data-processor/fastapi-0.136.3.dist-info/entry_points.txt +++ /dev/null @@ -1,5 +0,0 @@ -[console_scripts] -fastapi = fastapi.cli:main - -[gui_scripts] - diff --git a/functions/python/data-processor/fastapi-0.136.3.dist-info/licenses/LICENSE b/functions/python/data-processor/fastapi-0.136.3.dist-info/licenses/LICENSE deleted file mode 100644 index 3e92463e6..000000000 --- a/functions/python/data-processor/fastapi-0.136.3.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Sebastián Ramírez - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/INSTALLER b/functions/python/data-processor/fastapi-0.137.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/fastapi-0.137.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/METADATA b/functions/python/data-processor/fastapi-0.137.0.dist-info/METADATA deleted file mode 100644 index 67793b60f..000000000 --- a/functions/python/data-processor/fastapi-0.137.0.dist-info/METADATA +++ /dev/null @@ -1,615 +0,0 @@ -Metadata-Version: 2.4 -Name: fastapi -Version: 0.137.0 -Summary: FastAPI framework, high performance, easy to learn, fast to code, ready for production -Author-Email: =?utf-8?q?Sebasti=C3=A1n_Ram=C3=ADrez?= -License-Expression: MIT -License-File: LICENSE -Classifier: Intended Audience :: Information Technology -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Application Frameworks -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Software Development :: Libraries -Classifier: Topic :: Software Development -Classifier: Typing :: Typed -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Web Environment -Classifier: Framework :: AsyncIO -Classifier: Framework :: FastAPI -Classifier: Framework :: Pydantic -Classifier: Framework :: Pydantic :: 2 -Classifier: Intended Audience :: Developers -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Internet :: WWW/HTTP :: HTTP Servers -Classifier: Topic :: Internet :: WWW/HTTP -Project-URL: Homepage, https://github.com/fastapi/fastapi -Project-URL: Documentation, https://fastapi.tiangolo.com/ -Project-URL: Repository, https://github.com/fastapi/fastapi -Project-URL: Issues, https://github.com/fastapi/fastapi/issues -Project-URL: Changelog, https://fastapi.tiangolo.com/release-notes/ -Requires-Python: >=3.10 -Requires-Dist: starlette>=0.46.0 -Requires-Dist: pydantic>=2.9.0 -Requires-Dist: typing-extensions>=4.8.0 -Requires-Dist: typing-inspection>=0.4.2 -Requires-Dist: annotated-doc>=0.0.2 -Provides-Extra: standard -Requires-Dist: fastapi-cli[standard]>=0.0.8; extra == "standard" -Requires-Dist: fastar>=0.9.0; extra == "standard" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "standard" -Requires-Dist: jinja2>=3.1.5; extra == "standard" -Requires-Dist: python-multipart>=0.0.18; extra == "standard" -Requires-Dist: email-validator>=2.0.0; extra == "standard" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard" -Requires-Dist: pydantic-settings>=2.0.0; extra == "standard" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "standard" -Provides-Extra: standard-no-fastapi-cloud-cli -Requires-Dist: fastapi-cli[standard-no-fastapi-cloud-cli]>=0.0.8; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: jinja2>=3.1.5; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: python-multipart>=0.0.18; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: email-validator>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: pydantic-settings>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "standard-no-fastapi-cloud-cli" -Provides-Extra: all -Requires-Dist: fastapi-cli[standard]>=0.0.8; extra == "all" -Requires-Dist: httpx<1.0.0,>=0.23.0; extra == "all" -Requires-Dist: jinja2>=3.1.5; extra == "all" -Requires-Dist: python-multipart>=0.0.18; extra == "all" -Requires-Dist: itsdangerous>=1.1.0; extra == "all" -Requires-Dist: pyyaml>=5.3.1; extra == "all" -Requires-Dist: email-validator>=2.0.0; extra == "all" -Requires-Dist: uvicorn[standard]>=0.12.0; extra == "all" -Requires-Dist: pydantic-settings>=2.0.0; extra == "all" -Requires-Dist: pydantic-extra-types>=2.0.0; extra == "all" -Description-Content-Type: text/markdown - -

- FastAPI -

-

- FastAPI framework, high performance, easy to learn, fast to code, ready for production -

-

- - Test - - - Coverage - - - Package version - - - Supported Python versions - -

- ---- - -**Documentation**: [https://fastapi.tiangolo.com](https://fastapi.tiangolo.com) - -**Source Code**: [https://github.com/fastapi/fastapi](https://github.com/fastapi/fastapi) - ---- - -FastAPI is a modern, fast (high-performance), web framework for building APIs with Python based on standard Python type hints. - -The key features are: - -* **Fast**: Very high performance, on par with **NodeJS** and **Go** (thanks to Starlette and Pydantic). [One of the fastest Python frameworks available](#performance). -* **Fast to code**: Increase the speed to develop features by about 200% to 300%. * -* **Fewer bugs**: Reduce about 40% of human (developer) induced errors. * -* **Intuitive**: Great editor support. Completion everywhere. Less time debugging. -* **Easy**: Designed to be easy to use and learn. Less time reading docs. -* **Short**: Minimize code duplication. Multiple features from each parameter declaration. Fewer bugs. -* **Robust**: Get production-ready code. With automatic interactive documentation. -* **Standards-based**: Based on (and fully compatible with) the open standards for APIs: [OpenAPI](https://github.com/OAI/OpenAPI-Specification) (previously known as Swagger) and [JSON Schema](https://json-schema.org/). - -* estimation based on tests conducted by an internal development team, building production applications. - -## Sponsors - - -### Keystone Sponsor - - - -### Gold Sponsors - - - - - - - - - - -### Silver Sponsors - - - - - - - - - - -[Other sponsors](https://fastapi.tiangolo.com/fastapi-people/#sponsors) - -## Opinions - - - -
- -"_[...] I'm using **FastAPI** a ton these days. [...] I'm actually planning to use it for all of my team's **ML services at Microsoft**. Some of them are getting integrated into the core **Windows** product and some **Office** products._" - -
Kabir Khan - Microsoft (ref)
- ---- - -"_We adopted the **FastAPI** library to spawn a **REST** server that can be queried to obtain **predictions**. [for Ludwig]_" - -
Piero Molino, Yaroslav Dudin, and Sai Sumanth Miryala - Uber (ref)
- ---- - -"_**Netflix** is pleased to announce the open-source release of our **crisis management** orchestration framework: **Dispatch**! [built with **FastAPI**]_" - -
Kevin Glisson, Marc Vilanova, Forest Monsen - Netflix (ref)
- ---- - -"_If anyone is looking to build a production Python API, I would highly recommend **FastAPI**. It is **beautifully designed**, **simple to use** and **highly scalable**, it has become a **key component** in our API first development strategy and is driving many automations and services such as our Virtual TAC Engineer._" - -
Deon Pillsbury - Cisco (ref)
- ---- - -
- -## FastAPI Conf - -[**FastAPI Conf '26**](https://fastapiconf.com) is happening on **October 28, 2026** in **Amsterdam, NL**. All about FastAPI, right from the source. 🎤 - -FastAPI Conf '26 - October 28, 2026 - Amsterdam, NL - -## FastAPI mini documentary - -There's a [FastAPI mini documentary](https://www.youtube.com/watch?v=mpR8ngthqiE) released at the end of 2025, you can watch it online: - -FastAPI Mini Documentary - -## **Typer**, the FastAPI of CLIs - - - -If you are building a CLI app to be used in the terminal instead of a web API, check out [**Typer**](https://typer.tiangolo.com/). - -**Typer** is FastAPI's little sibling. And it's intended to be the **FastAPI of CLIs**. ⌨️ 🚀 - -## Requirements - -FastAPI stands on the shoulders of giants: - -* [Starlette](https://www.starlette.dev/) for the web parts. -* [Pydantic](https://docs.pydantic.dev/) for the data parts. - -## Installation - -Create and activate a [virtual environment](https://fastapi.tiangolo.com/virtual-environments/) and then install FastAPI: - -
- -```console -$ pip install "fastapi[standard]" - ----> 100% -``` - -
- -**Note**: Make sure you put `"fastapi[standard]"` in quotes to ensure it works in all terminals. - -## Example - -### Create it - -Create a file `main.py` with: - -```Python -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} -``` - -
-Or use async def... - -If your code uses `async` / `await`, use `async def`: - -```Python hl_lines="7 12" -from fastapi import FastAPI - -app = FastAPI() - - -@app.get("/") -async def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -async def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} -``` - -**Note**: - -If you don't know, check the _"In a hurry?"_ section about [`async` and `await` in the docs](https://fastapi.tiangolo.com/async/#in-a-hurry). - -
- -### Run it - -Run the server with: - -
- -```console -$ fastapi dev - - ╭────────── FastAPI CLI - Development mode ───────────╮ - │ │ - │ Serving at: http://127.0.0.1:8000 │ - │ │ - │ API docs: http://127.0.0.1:8000/docs │ - │ │ - │ Running in development mode, for production use: │ - │ │ - │ fastapi run │ - │ │ - ╰─────────────────────────────────────────────────────╯ - -INFO: Will watch for changes in these directories: ['/home/user/code/awesomeapp'] -INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) -INFO: Started reloader process [2248755] using WatchFiles -INFO: Started server process [2248757] -INFO: Waiting for application startup. -INFO: Application startup complete. -``` - -
- -
-About the command fastapi dev... - -The command `fastapi dev` reads your `main.py` file automatically, detects the **FastAPI** app in it, and starts a server using [Uvicorn](https://www.uvicorn.dev). - -By default, `fastapi dev` will start with auto-reload enabled for local development. - -You can read more about it in the [FastAPI CLI docs](https://fastapi.tiangolo.com/fastapi-cli/). - -
- -### Check it - -Open your browser at [http://127.0.0.1:8000/items/5?q=somequery](http://127.0.0.1:8000/items/5?q=somequery). - -You will see the JSON response as: - -```JSON -{"item_id": 5, "q": "somequery"} -``` - -You already created an API that: - -* Receives HTTP requests in the _paths_ `/` and `/items/{item_id}`. -* Both _paths_ take `GET` operations (also known as HTTP _methods_). -* The _path_ `/items/{item_id}` has a _path parameter_ `item_id` that should be an `int`. -* The _path_ `/items/{item_id}` has an optional `str` _query parameter_ `q`. - -### Interactive API docs - -Now go to [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs). - -You will see the automatic interactive API documentation (provided by [Swagger UI](https://github.com/swagger-api/swagger-ui)): - -![Swagger UI](https://fastapi.tiangolo.com/img/index/index-01-swagger-ui-simple.png) - -### Alternative API docs - -And now, go to [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc). - -You will see the alternative automatic documentation (provided by [ReDoc](https://github.com/Rebilly/ReDoc)): - -![ReDoc](https://fastapi.tiangolo.com/img/index/index-02-redoc-simple.png) - -## Example upgrade - -Now modify the file `main.py` to receive a body from a `PUT` request. - -Declare the body using standard Python types, thanks to Pydantic. - -```Python hl_lines="2 7-10 23-25" -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - price: float - is_offer: bool | None = None - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str | None = None): - return {"item_id": item_id, "q": q} - - -@app.put("/items/{item_id}") -def update_item(item_id: int, item: Item): - return {"item_name": item.name, "item_id": item_id} -``` - -The `fastapi dev` server should reload automatically. - -### Interactive API docs upgrade - -Now go to [http://127.0.0.1:8000/docs](http://127.0.0.1:8000/docs). - -* The interactive API documentation will be automatically updated, including the new body: - -![Swagger UI](https://fastapi.tiangolo.com/img/index/index-03-swagger-02.png) - -* Click on the button "Try it out", it allows you to fill the parameters and directly interact with the API: - -![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-04-swagger-03.png) - -* Then click on the "Execute" button, the user interface will communicate with your API, send the parameters, get the results and show them on the screen: - -![Swagger UI interaction](https://fastapi.tiangolo.com/img/index/index-05-swagger-04.png) - -### Alternative API docs upgrade - -And now, go to [http://127.0.0.1:8000/redoc](http://127.0.0.1:8000/redoc). - -* The alternative documentation will also reflect the new query parameter and body: - -![ReDoc](https://fastapi.tiangolo.com/img/index/index-06-redoc-02.png) - -### Recap - -In summary, you declare **once** the types of parameters, body, etc. as function parameters. - -You do that with standard modern Python types. - -You don't have to learn a new syntax, the methods or classes of a specific library, etc. - -Just standard **Python**. - -For example, for an `int`: - -```Python -item_id: int -``` - -or for a more complex `Item` model: - -```Python -item: Item -``` - -...and with that single declaration you get: - -* Editor support, including: - * Completion. - * Type checks. -* Validation of data: - * Automatic and clear errors when the data is invalid. - * Validation even for deeply nested JSON objects. -* Conversion of input data: coming from the network to Python data and types. Reading from: - * JSON. - * Path parameters. - * Query parameters. - * Cookies. - * Headers. - * Forms. - * Files. -* Conversion of output data: converting from Python data and types to network data (as JSON): - * Convert Python types (`str`, `int`, `float`, `bool`, `list`, etc). - * `datetime` objects. - * `UUID` objects. - * Database models. - * ...and many more. -* Automatic interactive API documentation, including 2 alternative user interfaces: - * Swagger UI. - * ReDoc. - ---- - -Coming back to the previous code example, **FastAPI** will: - -* Validate that there is an `item_id` in the path for `GET` and `PUT` requests. -* Validate that the `item_id` is of type `int` for `GET` and `PUT` requests. - * If it is not, the client will see a useful, clear error. -* Check if there is an optional query parameter named `q` (as in `http://127.0.0.1:8000/items/foo?q=somequery`) for `GET` requests. - * As the `q` parameter is declared with `= None`, it is optional. - * Without the `None` it would be required (as is the body in the case with `PUT`). -* For `PUT` requests to `/items/{item_id}`, read the body as JSON: - * Check that it has a required attribute `name` that should be a `str`. - * Check that it has a required attribute `price` that has to be a `float`. - * Check that it has an optional attribute `is_offer`, that should be a `bool`, if present. - * All this would also work for deeply nested JSON objects. -* Convert from and to JSON automatically. -* Document everything with OpenAPI, that can be used by: - * Interactive documentation systems. - * Automatic client code generation systems, for many languages. -* Provide 2 interactive documentation web interfaces directly. - ---- - -We just scratched the surface, but you already get the idea of how it all works. - -Try changing the line with: - -```Python - return {"item_name": item.name, "item_id": item_id} -``` - -...from: - -```Python - ... "item_name": item.name ... -``` - -...to: - -```Python - ... "item_price": item.price ... -``` - -...and see how your editor will auto-complete the attributes and know their types: - -![editor support](https://fastapi.tiangolo.com/img/vscode-completion.png) - -For a more complete example including more features, see the Tutorial - User Guide. - -**Spoiler alert**: the tutorial - user guide includes: - -* Declaration of **parameters** from other different places as: **headers**, **cookies**, **form fields** and **files**. -* How to set **validation constraints** as `maximum_length` or `regex`. -* A very powerful and easy to use **Dependency Injection** system. -* Security and authentication, including support for **OAuth2** with **JWT tokens** and **HTTP Basic** auth. -* More advanced (but equally easy) techniques for declaring **deeply nested JSON models** (thanks to Pydantic). -* **GraphQL** integration with [Strawberry](https://strawberry.rocks) and other libraries. -* Many extra features (thanks to Starlette) as: - * **WebSockets** - * extremely easy tests based on HTTPX and `pytest` - * **CORS** - * **Cookie Sessions** - * ...and more. - -### Deploy your app (optional) - -You can optionally deploy your FastAPI app to [FastAPI Cloud](https://fastapicloud.com) with a single command. 🚀 - -
- -```console -$ fastapi deploy - -Deploying to FastAPI Cloud... - -✅ Deployment successful! - -🐔 Ready the chicken! Your app is ready at https://myapp.fastapicloud.dev -``` - -
- -The CLI will automatically detect your FastAPI application and deploy it to the cloud. If you are not logged in, your browser will open to complete the authentication process. - -That's it! Now you can access your app at that URL. ✨ - -#### About FastAPI Cloud - -**[FastAPI Cloud](https://fastapicloud.com)** is built by the same author and team behind **FastAPI**. - -It streamlines the process of **building**, **deploying**, and **accessing** an API with minimal effort. - -It brings the same **developer experience** of building apps with FastAPI to **deploying** them to the cloud. 🎉 - -FastAPI Cloud is the primary sponsor and funding provider for the *FastAPI and friends* open source projects. ✨ - -#### Deploy to other cloud providers - -FastAPI is open source and based on standards. You can deploy FastAPI apps to any cloud provider you choose. - -Follow your cloud provider's guides to deploy FastAPI apps with them. 🤓 - -## Performance - -Independent TechEmpower benchmarks show **FastAPI** applications running under Uvicorn as [one of the fastest Python frameworks available](https://www.techempower.com/benchmarks/#section=test&runid=7464e520-0dc2-473d-bd34-dbdfd7e85911&hw=ph&test=query&l=zijzen-7), only below Starlette and Uvicorn themselves (used internally by FastAPI). (*) - -To understand more about it, see the section [Benchmarks](https://fastapi.tiangolo.com/benchmarks/). - -## Dependencies - -FastAPI depends on Pydantic and Starlette. - -### `standard` Dependencies - -When you install FastAPI with `pip install "fastapi[standard]"` it comes with the `standard` group of optional dependencies: - -Used by Pydantic: - -* [`email-validator`](https://github.com/JoshData/python-email-validator) - for email validation. - -Used by Starlette: - -* [`httpx`](https://www.python-httpx.org) - Required if you want to use the `TestClient`. -* [`jinja2`](https://jinja.palletsprojects.com) - Required if you want to use the default template configuration. -* [`python-multipart`](https://github.com/Kludex/python-multipart) - Required if you want to support form "parsing", with `request.form()`. - -Used by FastAPI: - -* [`uvicorn`](https://www.uvicorn.dev) - for the server that loads and serves your application. This includes `uvicorn[standard]`, which includes some dependencies (e.g. `uvloop`) needed for high performance serving. -* `fastapi-cli[standard]` - to provide the `fastapi` command. - * This includes `fastapi-cloud-cli`, which allows you to deploy your FastAPI application to [FastAPI Cloud](https://fastapicloud.com). - -### Without `standard` Dependencies - -If you don't want to include the `standard` optional dependencies, you can install with `pip install fastapi` instead of `pip install "fastapi[standard]"`. - -### Without `fastapi-cloud-cli` - -If you want to install FastAPI with the standard dependencies but without the `fastapi-cloud-cli`, you can install with `pip install "fastapi[standard-no-fastapi-cloud-cli]"`. - -### Additional Optional Dependencies - -There are some additional dependencies you might want to install. - -Additional optional Pydantic dependencies: - -* [`pydantic-settings`](https://docs.pydantic.dev/latest/usage/pydantic_settings/) - for settings management. -* [`pydantic-extra-types`](https://docs.pydantic.dev/latest/usage/types/extra_types/extra_types/) - for extra types to be used with Pydantic. - -Additional optional FastAPI dependencies: - -* [`orjson`](https://github.com/ijl/orjson) - Required if you want to use `ORJSONResponse`. -* [`ujson`](https://github.com/esnme/ultrajson) - Required if you want to use `UJSONResponse`. - -## License - -This project is licensed under the terms of the MIT license. diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/RECORD b/functions/python/data-processor/fastapi-0.137.0.dist-info/RECORD deleted file mode 100644 index 419a73f1c..000000000 --- a/functions/python/data-processor/fastapi-0.137.0.dist-info/RECORD +++ /dev/null @@ -1,109 +0,0 @@ -../../bin/fastapi,sha256=CrDjk8rnrVWs3h6pixOsNJ3Jn_6tnao51ZaI-jmAi7s,186 -fastapi-0.137.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -fastapi-0.137.0.dist-info/METADATA,sha256=QWqHxumFIOtslHRpIysB0-RiPnnvLd4ECFGGdz54jxA,26988 -fastapi-0.137.0.dist-info/RECORD,, -fastapi-0.137.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi-0.137.0.dist-info/WHEEL,sha256=VP-D4TPS230sME9Z3vb3INXvo1yt0924YRm5AOsk_dE,90 -fastapi-0.137.0.dist-info/entry_points.txt,sha256=GCf-WbIZxyGT4MUmrPGj1cOHYZoGsNPHAvNkT6hnGeA,61 -fastapi-0.137.0.dist-info/licenses/LICENSE,sha256=Tsif_IFIW5f-xYSy1KlhAy7v_oNEU4lP2cEnSQbMdE4,1086 -fastapi/.agents/skills/fastapi/SKILL.md,sha256=PIsrztMiIFHK1gMtBsrkrEDooTdiVeMI0sb0Bc8uo1U,10388 -fastapi/.agents/skills/fastapi/references/dependencies.md,sha256=KiNTXAVU6AAirx6kI0iF6Wf265hjUvDGMiIlRPvjcVc,3263 -fastapi/.agents/skills/fastapi/references/other-tools.md,sha256=EbpvhDr9TUuRmQHqaD7vflRIkmfVhHbygTpwC2mREj4,1526 -fastapi/.agents/skills/fastapi/references/streaming.md,sha256=bHaIKnwbTkd7TUVQm_uxapUnAlEG4rdvALV9koD5ypI,2581 -fastapi/__init__.py,sha256=lh3X0Lwc_6AnmBPRX8I7R4gCtZAsvgz77DLee0dbL0E,1081 -fastapi/__main__.py,sha256=bKePXLdO4SsVSM6r9SVoLickJDcR2c0cTOxZRKq26YQ,37 -fastapi/__pycache__/__init__.cpython-314.pyc,, -fastapi/__pycache__/__main__.cpython-314.pyc,, -fastapi/__pycache__/applications.cpython-314.pyc,, -fastapi/__pycache__/background.cpython-314.pyc,, -fastapi/__pycache__/cli.cpython-314.pyc,, -fastapi/__pycache__/concurrency.cpython-314.pyc,, -fastapi/__pycache__/datastructures.cpython-314.pyc,, -fastapi/__pycache__/encoders.cpython-314.pyc,, -fastapi/__pycache__/exception_handlers.cpython-314.pyc,, -fastapi/__pycache__/exceptions.cpython-314.pyc,, -fastapi/__pycache__/logger.cpython-314.pyc,, -fastapi/__pycache__/param_functions.cpython-314.pyc,, -fastapi/__pycache__/params.cpython-314.pyc,, -fastapi/__pycache__/requests.cpython-314.pyc,, -fastapi/__pycache__/responses.cpython-314.pyc,, -fastapi/__pycache__/routing.cpython-314.pyc,, -fastapi/__pycache__/sse.cpython-314.pyc,, -fastapi/__pycache__/staticfiles.cpython-314.pyc,, -fastapi/__pycache__/templating.cpython-314.pyc,, -fastapi/__pycache__/testclient.cpython-314.pyc,, -fastapi/__pycache__/types.cpython-314.pyc,, -fastapi/__pycache__/utils.cpython-314.pyc,, -fastapi/__pycache__/websockets.cpython-314.pyc,, -fastapi/_compat/__init__.py,sha256=PYOR-8vJ5va4Qjl810FcQmJbqmpyyeQMoZ9R86CeE2U,2095 -fastapi/_compat/__pycache__/__init__.cpython-314.pyc,, -fastapi/_compat/__pycache__/shared.cpython-314.pyc,, -fastapi/_compat/__pycache__/v2.cpython-314.pyc,, -fastapi/_compat/shared.py,sha256=XJl8cVYOTSgfK-1fiJHLeHHeSGLfC00H7MrtpyYpQgU,7118 -fastapi/_compat/v2.py,sha256=sDGyi1iKSFW9LuJ7n4B61-1yrQI1NHRSwsFRRLuFJ8k,17601 -fastapi/applications.py,sha256=X2VVrhf3ufokw1sk2_ZycuzO1mn3GxF5qeIiY9E-kzg,181508 -fastapi/background.py,sha256=TADzAethOAaqpVvckYuTT3c4O9N1HaFQysFCPt0MsgU,1820 -fastapi/cli.py,sha256=OYhZb0NR_deuT5ofyPF2NoNBzZDNOP8Salef2nk-HqA,418 -fastapi/concurrency.py,sha256=xHGDEOQAA6cvFEDX46oq3r2t1Zd4sVvreaRgdIE4juM,1489 -fastapi/datastructures.py,sha256=XPugnojHc4N07eF5Xp1TTndP2gMv1fG4k-0L4Vdr7Kc,5321 -fastapi/dependencies/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/dependencies/__pycache__/__init__.cpython-314.pyc,, -fastapi/dependencies/__pycache__/models.cpython-314.pyc,, -fastapi/dependencies/__pycache__/utils.cpython-314.pyc,, -fastapi/dependencies/models.py,sha256=XPIkEaukHaTaFz9LoeI9f_T-3otOAVkoOC113TpFbWI,7250 -fastapi/dependencies/utils.py,sha256=XIpRML7ttx5EhmchgWz6R5CMsBuepx7KHdmi38F_e_s,39728 -fastapi/encoders.py,sha256=m2OGnq9k94Alcd6UQM8r7iwn-cjvJdpOKRVJuQR68T8,11603 -fastapi/exception_handlers.py,sha256=YVcT8Zy021VYYeecgdyh5YEUjEIHKcLspbkSf4OfbJI,1275 -fastapi/exceptions.py,sha256=fUNOBRdIsULU0TnO8aNBg3HoJeyJxXp9UV6D8wYu9lI,7453 -fastapi/logger.py,sha256=I9NNi3ov8AcqbsbC9wl1X-hdItKgYt2XTrx1f99Zpl4,54 -fastapi/middleware/__init__.py,sha256=oQDxiFVcc1fYJUOIFvphnK7pTT5kktmfL32QXpBFvvo,58 -fastapi/middleware/__pycache__/__init__.cpython-314.pyc,, -fastapi/middleware/__pycache__/asyncexitstack.cpython-314.pyc,, -fastapi/middleware/__pycache__/cors.cpython-314.pyc,, -fastapi/middleware/__pycache__/gzip.cpython-314.pyc,, -fastapi/middleware/__pycache__/httpsredirect.cpython-314.pyc,, -fastapi/middleware/__pycache__/trustedhost.cpython-314.pyc,, -fastapi/middleware/__pycache__/wsgi.cpython-314.pyc,, -fastapi/middleware/asyncexitstack.py,sha256=RKGlQpGzg3GLosqVhrxBy_NCZ9qJS7zQeNHt5Y3x-00,637 -fastapi/middleware/cors.py,sha256=ynwjWQZoc_vbhzZ3_ZXceoaSrslHFHPdoM52rXr0WUU,79 -fastapi/middleware/gzip.py,sha256=xM5PcsH8QlAimZw4VDvcmTnqQamslThsfe3CVN2voa0,79 -fastapi/middleware/httpsredirect.py,sha256=rL8eXMnmLijwVkH7_400zHri1AekfeBd6D6qs8ix950,115 -fastapi/middleware/trustedhost.py,sha256=eE5XGRxGa7c5zPnMJDGp3BxaL25k5iVQlhnv-Pk0Pss,109 -fastapi/middleware/wsgi.py,sha256=a_FMDoeTwcdig9wdAGumIH82oDFfuj4pxtQxLbAw2Ns,107 -fastapi/openapi/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/openapi/__pycache__/__init__.cpython-314.pyc,, -fastapi/openapi/__pycache__/constants.cpython-314.pyc,, -fastapi/openapi/__pycache__/docs.cpython-314.pyc,, -fastapi/openapi/__pycache__/models.cpython-314.pyc,, -fastapi/openapi/__pycache__/utils.cpython-314.pyc,, -fastapi/openapi/constants.py,sha256=adGzmis1L1HJRTE3kJ5fmHS_Noq6tIY6pWv_SFzoFDU,153 -fastapi/openapi/docs.py,sha256=PcjH0Sn-yp97O0exXincG8epsY7ATyJC5662bK1DRao,12425 -fastapi/openapi/models.py,sha256=twjJWGf6lcKRJZgaZCcnQ0Ten1hDhcyCLgbr7tXIP30,14608 -fastapi/openapi/utils.py,sha256=bIuaVelow4JA3oRWJqQGxUjpx-LMpJXGnpoVcc7TV20,27083 -fastapi/param_functions.py,sha256=4fTCVlvEDbAQr7gvWYkOX8hktdl2Iwk-kv4FaTz7P8w,69596 -fastapi/params.py,sha256=1fNNSK5J7PM5Fw-F5_9uOysA2NgC-61mb3LlrBWpqJM,26205 -fastapi/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -fastapi/requests.py,sha256=zayepKFcienBllv3snmWI20Gk0oHNVLU4DDhqXBb4LU,142 -fastapi/responses.py,sha256=BM7JtiZ_G4j2Z7QvHUzV1KtZPnjOoCfX35RqYXlZDh0,4144 -fastapi/routing.py,sha256=_jJGQSrUq2iOo7iU4cCvxMJADCCgP5GFOiLiGQ592-M,222885 -fastapi/security/__init__.py,sha256=bO8pNmxqVRXUjfl2mOKiVZLn0FpBQ61VUYVjmppnbJw,881 -fastapi/security/__pycache__/__init__.cpython-314.pyc,, -fastapi/security/__pycache__/api_key.cpython-314.pyc,, -fastapi/security/__pycache__/base.cpython-314.pyc,, -fastapi/security/__pycache__/http.cpython-314.pyc,, -fastapi/security/__pycache__/oauth2.cpython-314.pyc,, -fastapi/security/__pycache__/open_id_connect_url.cpython-314.pyc,, -fastapi/security/__pycache__/utils.cpython-314.pyc,, -fastapi/security/api_key.py,sha256=4CNLNVAStOsMhytH9C5EOUEOZrtLg_IpMQS_HcRDP4M,9793 -fastapi/security/base.py,sha256=dl4pvbC-RxjfbWgPtCWd8MVU-7CB2SZ22rJDXVCXO6c,141 -fastapi/security/http.py,sha256=Z0xALDqwgJZRAaDs40Sa68rAnjFzEL99UEmO5PJzTKA,13410 -fastapi/security/oauth2.py,sha256=sSqW4tbvoHaWNld46TYatta5rgFfjod4LhxmedXzkI8,24178 -fastapi/security/open_id_connect_url.py,sha256=V8WLPEsEq_WJlIjPwJO2vCoJWGLu-VTt1-N2H7aV-D4,3136 -fastapi/security/utils.py,sha256=E9YIoez-H2k1oBLEdxqJEi8sV1umunJYKjQHvZG3FRY,261 -fastapi/sse.py,sha256=adtwP03ePndMQOuRCNRHrTfFQxMpRwrSUoRMHyB8OvA,6848 -fastapi/staticfiles.py,sha256=iirGIt3sdY2QZXd36ijs3Cj-T0FuGFda3cd90kM9Ikw,69 -fastapi/templating.py,sha256=4zsuTWgcjcEainMJFAlW6-gnslm6AgOS1SiiDWfmQxk,76 -fastapi/testclient.py,sha256=nBvaAmX66YldReJNZXPOk1sfuo2Q6hs8bOvIaCep6LQ,66 -fastapi/types.py,sha256=g2tD842BUHC2C3_P8P06albQ4MhCb9RybrSmp5rODgU,438 -fastapi/utils.py,sha256=DX0VrnMwfVsZxRz8IitQ42c2---fDzmFZkeRMZR2UMo,4341 -fastapi/websockets.py,sha256=419uncYObEKZG0YcrXscfQQYLSWoE10jqxVMetGdR98,222 diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/REQUESTED b/functions/python/data-processor/fastapi-0.137.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/WHEEL b/functions/python/data-processor/fastapi-0.137.0.dist-info/WHEEL deleted file mode 100644 index e651d8efb..000000000 --- a/functions/python/data-processor/fastapi-0.137.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: pdm-backend (2.4.9) -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/entry_points.txt b/functions/python/data-processor/fastapi-0.137.0.dist-info/entry_points.txt deleted file mode 100644 index b81849e1e..000000000 --- a/functions/python/data-processor/fastapi-0.137.0.dist-info/entry_points.txt +++ /dev/null @@ -1,5 +0,0 @@ -[console_scripts] -fastapi = fastapi.cli:main - -[gui_scripts] - diff --git a/functions/python/data-processor/fastapi-0.137.0.dist-info/licenses/LICENSE b/functions/python/data-processor/fastapi-0.137.0.dist-info/licenses/LICENSE deleted file mode 100644 index 3e92463e6..000000000 --- a/functions/python/data-processor/fastapi-0.137.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2018 Sebastián Ramírez - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/functions/python/data-processor/fastapi/.agents/skills/fastapi/SKILL.md b/functions/python/data-processor/fastapi/.agents/skills/fastapi/SKILL.md deleted file mode 100644 index 48cfdabb8..000000000 --- a/functions/python/data-processor/fastapi/.agents/skills/fastapi/SKILL.md +++ /dev/null @@ -1,436 +0,0 @@ ---- -name: fastapi -description: FastAPI best practices and conventions. Use when working with FastAPI APIs and Pydantic models for them. Keeps FastAPI code clean and up to date with the latest features and patterns, updated with new versions. Write new code or refactor and update old code. ---- - -# FastAPI - -Official FastAPI skill to write code with best practices, keeping up to date with new versions and features. - -## Use the `fastapi` CLI - -Run the development server on localhost with reload: - -```bash -fastapi dev -``` - - -Run the production server: - -```bash -fastapi run -``` - -### Add an entrypoint in `pyproject.toml` - -FastAPI CLI will read the entrypoint in `pyproject.toml` to know where the FastAPI app is declared. - -```toml -[tool.fastapi] -entrypoint = "my_app.main:app" -``` - -### Use `fastapi` with a path - -When adding the entrypoint to `pyproject.toml` is not possible, or the user explicitly asks not to, or it's running an independent small app, you can pass the app file path to the `fastapi` command: - -```bash -fastapi dev my_app/main.py -``` - -Prefer to set the entrypoint in `pyproject.toml` when possible. - -## Use `Annotated` - -Always prefer the `Annotated` style for parameter and dependency declarations. - -It keeps the function signatures working in other contexts, respects the types, allows reusability. - -### In Parameter Declarations - -Use `Annotated` for parameter declarations, including `Path`, `Query`, `Header`, etc.: - -```python -from typing import Annotated - -from fastapi import FastAPI, Path, Query - -app = FastAPI() - - -@app.get("/items/{item_id}") -async def read_item( - item_id: Annotated[int, Path(ge=1, description="The item ID")], - q: Annotated[str | None, Query(max_length=50)] = None, -): - return {"message": "Hello World"} -``` - -instead of: - -```python -# DO NOT DO THIS -@app.get("/items/{item_id}") -async def read_item( - item_id: int = Path(ge=1, description="The item ID"), - q: str | None = Query(default=None, max_length=50), -): - return {"message": "Hello World"} -``` - -### For Dependencies - -Use `Annotated` for dependencies with `Depends()`. - -Unless asked not to, create a new type alias for the dependency to allow re-using it. - -```python -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -def get_current_user(): - return {"username": "johndoe"} - - -CurrentUserDep = Annotated[dict, Depends(get_current_user)] - - -@app.get("/items/") -async def read_item(current_user: CurrentUserDep): - return {"message": "Hello World"} -``` - -instead of: - -```python -# DO NOT DO THIS -@app.get("/items/") -async def read_item(current_user: dict = Depends(get_current_user)): - return {"message": "Hello World"} -``` - -## Do not use Ellipsis for *path operations* or Pydantic models - -Do not use `...` as a default value for required parameters, it's not needed and not recommended. - -Do this, without Ellipsis (`...`): - -```python -from typing import Annotated - -from fastapi import FastAPI, Query -from pydantic import BaseModel, Field - - -class Item(BaseModel): - name: str - description: str | None = None - price: float = Field(gt=0) - - -app = FastAPI() - - -@app.post("/items/") -async def create_item(item: Item, project_id: Annotated[int, Query()]): ... -``` - -instead of this: - -```python -# DO NOT DO THIS -class Item(BaseModel): - name: str = ... - description: str | None = None - price: float = Field(..., gt=0) - - -app = FastAPI() - - -@app.post("/items/") -async def create_item(item: Item, project_id: Annotated[int, Query(...)]): ... -``` - -## Return Type or Response Model - -When possible, include a return type. It will be used to validate, filter, document, and serialize the response. - -```python -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me") -async def get_item() -> Item: - return Item(name="Plumbus", description="All-purpose home device") -``` - -**Important**: Return types or response models are what filter data ensuring no sensitive information is exposed. And they are used to serialize data with Pydantic (in Rust), this is the main idea that can increase response performance. - -The return type doesn't have to be a Pydantic model, it could be a different type, like a list of integers, or a dict, etc. - -### When to use `response_model` instead - -If the return type is not the same as the type that you want to use to validate, filter, or serialize, use the `response_model` parameter on the decorator instead. - -```python -from typing import Any - -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me", response_model=Item) -async def get_item() -> Any: - return {"name": "Foo", "description": "A very nice Item"} -``` - -This can be particularly useful when filtering data to expose only the public fields and avoid exposing sensitive information. - -```python -from typing import Any - -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class InternalItem(BaseModel): - name: str - description: str | None = None - secret_key: str - - -class Item(BaseModel): - name: str - description: str | None = None - - -@app.get("/items/me", response_model=Item) -async def get_item() -> Any: - item = InternalItem( - name="Foo", description="A very nice Item", secret_key="supersecret" - ) - return item -``` - -## Performance - -Do not use `ORJSONResponse` or `UJSONResponse`, they are deprecated. - -Instead, declare a return type or response model. Pydantic will handle the data serialization on the Rust side. - -## Including Routers - -When declaring routers, prefer to add router level parameters like prefix, tags, etc. to the router itself, instead of in `include_router()`. - -Do this: - -```python -from fastapi import APIRouter, FastAPI - -app = FastAPI() - -router = APIRouter(prefix="/items", tags=["items"]) - - -@router.get("/") -async def list_items(): - return [] - - -# In main.py -app.include_router(router) -``` - -instead of this: - -```python -# DO NOT DO THIS -from fastapi import APIRouter, FastAPI - -app = FastAPI() - -router = APIRouter() - - -@router.get("/") -async def list_items(): - return [] - - -# In main.py -app.include_router(router, prefix="/items", tags=["items"]) -``` - -There could be exceptions, but try to follow this convention. - -Apply shared dependencies at the router level via `dependencies=[Depends(...)]`. - -## Dependency Injection - -See [the dependency injection reference](references/dependencies.md) for detailed patterns including `yield` with `scope`, and class dependencies. - -Use dependencies when the logic can't be declared in Pydantic validation, depends on external resources, needs cleanup (with `yield`), or is shared across endpoints. - -Apply shared dependencies at the router level via `dependencies=[Depends(...)]`. - -## Async vs Sync *path operations* - -Use `async` *path operations* only when fully certain that the logic called inside is compatible with async and await (it's called with `await`) or that doesn't block. - -```python -from fastapi import FastAPI - -app = FastAPI() - - -# Use async def when calling async code -@app.get("/async-items/") -async def read_async_items(): - data = await some_async_library.fetch_items() - return data - - -# Use plain def when calling blocking/sync code or when in doubt -@app.get("/items/") -def read_items(): - data = some_blocking_library.fetch_items() - return data -``` - -In case of doubt, or by default, use regular `def` functions, those will be run in a threadpool so they don't block the event loop. - -The same rules apply to dependencies. - -Make sure blocking code is not run inside of `async` functions. The logic will work, but will damage the performance heavily. - -When needing to mix blocking and async code, see Asyncer in [the other tools reference](references/other-tools.md). - -## Streaming (JSON Lines, SSE, bytes) - -See [the streaming reference](references/streaming.md) for JSON Lines, Server-Sent Events (`EventSourceResponse`, `ServerSentEvent`), and byte streaming (`StreamingResponse`) patterns. - -## Tooling - -See [the other tools reference](references/other-tools.md) for details on uv, Ruff, ty for package management, linting, type checking, formatting, etc. - -## Other Libraries - -See [the other tools reference](references/other-tools.md) for details on other libraries: - -* Asyncer for handling async and await, concurrency, mixing async and blocking code, prefer it over AnyIO or asyncio. -* SQLModel for working with SQL databases, prefer it over SQLAlchemy. -* HTTPX for interacting with HTTP (other APIs), prefer it over Requests. - -## Do not use Pydantic RootModels - -Do not use Pydantic `RootModel`, instead use regular type annotations with `Annotated` and Pydantic validation utilities. - -For example, for a list with validations you could do: - -```python -from typing import Annotated - -from fastapi import Body, FastAPI -from pydantic import Field - -app = FastAPI() - - -@app.post("/items/") -async def create_items(items: Annotated[list[int], Field(min_length=1), Body()]): - return items -``` - -instead of: - -```python -# DO NOT DO THIS -from typing import Annotated - -from fastapi import FastAPI -from pydantic import Field, RootModel - -app = FastAPI() - - -class ItemList(RootModel[Annotated[list[int], Field(min_length=1)]]): - pass - - -@app.post("/items/") -async def create_items(items: ItemList): - return items - -``` - -FastAPI supports these type annotations and will create a Pydantic `TypeAdapter` for them, so that types can work as normally and there's no need for the custom logic and types in RootModels. - -## Use one HTTP operation per function - -Don't mix HTTP operations in a single function, having one function per HTTP operation helps separate concerns and organize the code. - -Do this: - -```python -from fastapi import FastAPI -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - - -@app.get("/items/") -async def list_items(): - return [] - - -@app.post("/items/") -async def create_item(item: Item): - return item -``` - -instead of this: - -```python -# DO NOT DO THIS -from fastapi import FastAPI, Request -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - - -@app.api_route("/items/", methods=["GET", "POST"]) -async def handle_items(request: Request): - if request.method == "GET": - return [] -``` diff --git a/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/dependencies.md b/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/dependencies.md deleted file mode 100644 index ca709096e..000000000 --- a/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/dependencies.md +++ /dev/null @@ -1,142 +0,0 @@ -# Dependency Injection - -Use dependencies when: - -* They can't be declared in Pydantic validation and require additional logic -* The logic depends on external resources or could block in any other way -* Other dependencies need their results (it's a sub-dependency) -* The logic can be shared by multiple endpoints to do things like error early, authentication, etc. -* They need to handle cleanup (e.g., DB sessions, file handles), using dependencies with `yield` -* Their logic needs input data from the request, like headers, query parameters, etc. - -## Dependencies with `yield` and `scope` - -When using dependencies with `yield`, they can have a `scope` that defines when the exit code is run. - -Use the default scope `"request"` to run the exit code after the response is sent back. - -```python -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -def get_db(): - db = DBSession() - try: - yield db - finally: - db.close() - - -DBDep = Annotated[DBSession, Depends(get_db)] - - -@app.get("/items/") -async def read_items(db: DBDep): - return db.query(Item).all() -``` - -Use the scope `"function"` when they should run the exit code after the response data is generated but before the response is sent back to the client. - -```python -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -def get_username(): - try: - yield "Rick" - finally: - print("Cleanup up before response is sent") - -UserNameDep = Annotated[str, Depends(get_username, scope="function")] - -@app.get("/users/me") -def get_user_me(username: UserNameDep): - return username -``` - -## Class Dependencies - -Avoid creating class dependencies when possible. - -If a class is needed, instead create a regular function dependency that returns a class instance. - -Do this: - -```python -from dataclasses import dataclass -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -@dataclass -class DatabasePaginator: - offset: int = 0 - limit: int = 100 - q: str | None = None - - def get_page(self) -> dict: - # Simulate a page of data - return { - "offset": self.offset, - "limit": self.limit, - "q": self.q, - "items": [], - } - - -def get_db_paginator( - offset: int = 0, limit: int = 100, q: str | None = None -) -> DatabasePaginator: - return DatabasePaginator(offset=offset, limit=limit, q=q) - - -PaginatorDep = Annotated[DatabasePaginator, Depends(get_db_paginator)] - - -@app.get("/items/") -async def read_items(paginator: PaginatorDep): - return paginator.get_page() -``` - -instead of this: - -```python -# DO NOT DO THIS -from typing import Annotated - -from fastapi import Depends, FastAPI - -app = FastAPI() - - -class DatabasePaginator: - def __init__(self, offset: int = 0, limit: int = 100, q: str | None = None): - self.offset = offset - self.limit = limit - self.q = q - - def get_page(self) -> dict: - # Simulate a page of data - return { - "offset": self.offset, - "limit": self.limit, - "q": self.q, - "items": [], - } - - -@app.get("/items/") -async def read_items(paginator: Annotated[DatabasePaginator, Depends()]): - return paginator.get_page() -``` diff --git a/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/other-tools.md b/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/other-tools.md deleted file mode 100644 index 58b19d096..000000000 --- a/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/other-tools.md +++ /dev/null @@ -1,76 +0,0 @@ -# Other Tools - -## uv - -If uv is available, use it to manage dependencies. - -## Ruff - -If Ruff is available, use it to lint and format the code. Consider enabling the FastAPI rules. - -## ty - -If ty is available, use it to check types. - -## Asyncer - -When needing to run blocking code inside of async functions, or async code inside of blocking functions, suggest using Asyncer. - -Prefer it over AnyIO or asyncio. - -Install: - -```bash -uv add asyncer -``` - -Run blocking sync code inside of async with `asyncify()`: - -```python -from asyncer import asyncify -from fastapi import FastAPI - -app = FastAPI() - - -def do_blocking_work(name: str) -> str: - # Some blocking I/O operation - return f"Hello {name}" - - -@app.get("/items/") -async def read_items(): - result = await asyncify(do_blocking_work)(name="World") - return {"message": result} -``` - -And run async code inside of blocking sync code with `syncify()`: - -```python -from asyncer import syncify -from fastapi import FastAPI - -app = FastAPI() - - -async def do_async_work(name: str) -> str: - return f"Hello {name}" - - -@app.get("/items/") -def read_items(): - result = syncify(do_async_work)(name="World") - return {"message": result} -``` - -## SQLModel for SQL databases - -When working with SQL databases, prefer using SQLModel as it is integrated with Pydantic and will allow declaring data validation with the same models. - -Prefer it over SQLAlchemy. - -## HTTPX - -Use HTTPX for handling HTTP communication (e.g. with other APIs). It support sync and async usage. - -Prefer it over Requests. diff --git a/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/streaming.md b/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/streaming.md deleted file mode 100644 index 0832eedcb..000000000 --- a/functions/python/data-processor/fastapi/.agents/skills/fastapi/references/streaming.md +++ /dev/null @@ -1,105 +0,0 @@ -# Streaming - -## Stream JSON Lines - -To stream JSON Lines, declare the return type and use `yield` to return the data. - -```python -@app.get("/items/stream") -async def stream_items() -> AsyncIterable[Item]: - for item in items: - yield item -``` - -## Server-Sent Events (SSE) - -To stream Server-Sent Events, use `response_class=EventSourceResponse` and `yield` items from the endpoint. - -Plain objects are automatically JSON-serialized as `data:` fields, declare the return type so the serialization is done by Pydantic: - -```python -from collections.abc import AsyncIterable - -from fastapi import FastAPI -from fastapi.sse import EventSourceResponse -from pydantic import BaseModel - -app = FastAPI() - - -class Item(BaseModel): - name: str - price: float - - -@app.get("/items/stream", response_class=EventSourceResponse) -async def stream_items() -> AsyncIterable[Item]: - yield Item(name="Plumbus", price=32.99) - yield Item(name="Portal Gun", price=999.99) -``` - -For full control over SSE fields (`event`, `id`, `retry`, `comment`), yield `ServerSentEvent` instances: - -```python -from collections.abc import AsyncIterable - -from fastapi import FastAPI -from fastapi.sse import EventSourceResponse, ServerSentEvent - -app = FastAPI() - - -@app.get("/events", response_class=EventSourceResponse) -async def stream_events() -> AsyncIterable[ServerSentEvent]: - yield ServerSentEvent(data={"status": "started"}, event="status", id="1") - yield ServerSentEvent(data={"progress": 50}, event="progress", id="2") -``` - -Use `raw_data` instead of `data` to send pre-formatted strings without JSON encoding: - -```python -yield ServerSentEvent(raw_data="plain text line", event="log") -``` - -## Stream bytes - -To stream bytes, declare a `response_class=` of `StreamingResponse` or a sub-class, and use `yield` to return the data. - -```python -from fastapi import FastAPI -from fastapi.responses import StreamingResponse -from app.utils import read_image - -app = FastAPI() - - -class PNGStreamingResponse(StreamingResponse): - media_type = "image/png" - -@app.get("/image", response_class=PNGStreamingResponse) -def stream_image_no_async_no_annotation(): - with read_image() as image_file: - yield from image_file -``` - -prefer this over returning a `StreamingResponse` directly: - -```python -# DO NOT DO THIS - -import anyio -from fastapi import FastAPI -from fastapi.responses import StreamingResponse -from app.utils import read_image - -app = FastAPI() - - -class PNGStreamingResponse(StreamingResponse): - media_type = "image/png" - - -@app.get("/") -async def main(): - return PNGStreamingResponse(read_image()) -``` diff --git a/functions/python/data-processor/fastapi/__init__.py b/functions/python/data-processor/fastapi/__init__.py deleted file mode 100644 index 38e747232..000000000 --- a/functions/python/data-processor/fastapi/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -"""FastAPI framework, high performance, easy to learn, fast to code, ready for production""" - -__version__ = "0.136.3" - -from starlette import status as status - -from .applications import FastAPI as FastAPI -from .background import BackgroundTasks as BackgroundTasks -from .datastructures import UploadFile as UploadFile -from .exceptions import HTTPException as HTTPException -from .exceptions import WebSocketException as WebSocketException -from .param_functions import Body as Body -from .param_functions import Cookie as Cookie -from .param_functions import Depends as Depends -from .param_functions import File as File -from .param_functions import Form as Form -from .param_functions import Header as Header -from .param_functions import Path as Path -from .param_functions import Query as Query -from .param_functions import Security as Security -from .requests import Request as Request -from .responses import Response as Response -from .routing import APIRouter as APIRouter -from .websockets import WebSocket as WebSocket -from .websockets import WebSocketDisconnect as WebSocketDisconnect diff --git a/functions/python/data-processor/fastapi/__main__.py b/functions/python/data-processor/fastapi/__main__.py deleted file mode 100644 index fc36465f5..000000000 --- a/functions/python/data-processor/fastapi/__main__.py +++ /dev/null @@ -1,3 +0,0 @@ -from fastapi.cli import main - -main() diff --git a/functions/python/data-processor/fastapi/_compat/__init__.py b/functions/python/data-processor/fastapi/_compat/__init__.py deleted file mode 100644 index 4581c38c8..000000000 --- a/functions/python/data-processor/fastapi/_compat/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from .shared import PYDANTIC_VERSION_MINOR_TUPLE as PYDANTIC_VERSION_MINOR_TUPLE -from .shared import annotation_is_pydantic_v1 as annotation_is_pydantic_v1 -from .shared import field_annotation_is_scalar as field_annotation_is_scalar -from .shared import ( - field_annotation_is_scalar_sequence as field_annotation_is_scalar_sequence, -) -from .shared import field_annotation_is_sequence as field_annotation_is_sequence -from .shared import ( - is_bytes_or_nonable_bytes_annotation as is_bytes_or_nonable_bytes_annotation, -) -from .shared import is_bytes_sequence_annotation as is_bytes_sequence_annotation -from .shared import is_pydantic_v1_model_instance as is_pydantic_v1_model_instance -from .shared import ( - is_uploadfile_or_nonable_uploadfile_annotation as is_uploadfile_or_nonable_uploadfile_annotation, -) -from .shared import ( - is_uploadfile_sequence_annotation as is_uploadfile_sequence_annotation, -) -from .shared import lenient_issubclass as lenient_issubclass -from .shared import sequence_types as sequence_types -from .shared import value_is_sequence as value_is_sequence -from .v2 import ModelField as ModelField -from .v2 import PydanticSchemaGenerationError as PydanticSchemaGenerationError -from .v2 import RequiredParam as RequiredParam -from .v2 import Undefined as Undefined -from .v2 import Url as Url -from .v2 import copy_field_info as copy_field_info -from .v2 import create_body_model as create_body_model -from .v2 import evaluate_forwardref as evaluate_forwardref -from .v2 import get_cached_model_fields as get_cached_model_fields -from .v2 import get_definitions as get_definitions -from .v2 import get_flat_models_from_fields as get_flat_models_from_fields -from .v2 import get_missing_field_error as get_missing_field_error -from .v2 import get_model_name_map as get_model_name_map -from .v2 import get_schema_from_model_field as get_schema_from_model_field -from .v2 import is_scalar_field as is_scalar_field -from .v2 import serialize_sequence_value as serialize_sequence_value -from .v2 import ( - with_info_plain_validator_function as with_info_plain_validator_function, -) diff --git a/functions/python/data-processor/fastapi/_compat/shared.py b/functions/python/data-processor/fastapi/_compat/shared.py deleted file mode 100644 index bd38c5534..000000000 --- a/functions/python/data-processor/fastapi/_compat/shared.py +++ /dev/null @@ -1,214 +0,0 @@ -import types -import typing -import warnings -from collections import deque -from collections.abc import Mapping, Sequence -from dataclasses import is_dataclass -from typing import ( - Annotated, - Any, - TypeGuard, - TypeVar, - Union, - get_args, - get_origin, -) - -from fastapi.types import UnionType -from pydantic import BaseModel -from pydantic.version import VERSION as PYDANTIC_VERSION -from starlette.datastructures import UploadFile - -_T = TypeVar("_T") - -# Copy from Pydantic: pydantic/_internal/_typing_extra.py -WithArgsTypes: tuple[Any, ...] = ( - typing._GenericAlias, # type: ignore[attr-defined] # ty: ignore[unresolved-attribute] - types.GenericAlias, - types.UnionType, -) # pyright: ignore[reportAttributeAccessIssue] - -PYDANTIC_VERSION_MINOR_TUPLE = tuple(int(x) for x in PYDANTIC_VERSION.split(".")[:2]) - - -sequence_annotation_to_type = { - Sequence: list, - list: list, - tuple: tuple, - set: set, - frozenset: frozenset, - deque: deque, -} - -sequence_types: tuple[type[Any], ...] = tuple(sequence_annotation_to_type.keys()) - - -# Copy of Pydantic: pydantic/_internal/_utils.py with added TypeGuard -def lenient_issubclass( - cls: Any, class_or_tuple: type[_T] | tuple[type[_T], ...] | None -) -> TypeGuard[type[_T]]: - try: - return isinstance(cls, type) and issubclass(cls, class_or_tuple) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - except TypeError: # pragma: no cover - if isinstance(cls, WithArgsTypes): - return False - raise # pragma: no cover - - -def _annotation_is_sequence(annotation: type[Any] | None) -> bool: - if lenient_issubclass(annotation, (str, bytes)): - return False - return lenient_issubclass(annotation, sequence_types) - - -def field_annotation_is_sequence(annotation: type[Any] | None) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if field_annotation_is_sequence(arg): - return True - return False - return _annotation_is_sequence(annotation) or _annotation_is_sequence( - get_origin(annotation) - ) - - -def value_is_sequence(value: Any) -> bool: - return isinstance(value, sequence_types) and not isinstance(value, (str, bytes)) - - -def _annotation_is_complex(annotation: type[Any] | None) -> bool: - return ( - lenient_issubclass(annotation, (BaseModel, Mapping, UploadFile)) - or _annotation_is_sequence(annotation) - or is_dataclass(annotation) - ) - - -def field_annotation_is_complex(annotation: type[Any] | None) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - return any(field_annotation_is_complex(arg) for arg in get_args(annotation)) - - if origin is Annotated: - return field_annotation_is_complex(get_args(annotation)[0]) - - return ( - _annotation_is_complex(annotation) - or _annotation_is_complex(origin) - or hasattr(origin, "__pydantic_core_schema__") - or hasattr(origin, "__get_pydantic_core_schema__") - ) - - -def field_annotation_is_scalar(annotation: Any) -> bool: - # handle Ellipsis here to make tuple[int, ...] work nicely - return annotation is Ellipsis or not field_annotation_is_complex(annotation) - - -def field_annotation_is_scalar_sequence(annotation: type[Any] | None) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - at_least_one_scalar_sequence = False - for arg in get_args(annotation): - if field_annotation_is_scalar_sequence(arg): - at_least_one_scalar_sequence = True - continue - elif not field_annotation_is_scalar(arg): - return False - return at_least_one_scalar_sequence - return field_annotation_is_sequence(annotation) and all( - field_annotation_is_scalar(sub_annotation) - for sub_annotation in get_args(annotation) - ) - - -def is_bytes_or_nonable_bytes_annotation(annotation: Any) -> bool: - if lenient_issubclass(annotation, bytes): - return True - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if lenient_issubclass(arg, bytes): - return True - return False - - -def is_uploadfile_or_nonable_uploadfile_annotation(annotation: Any) -> bool: - if lenient_issubclass(annotation, UploadFile): - return True - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if lenient_issubclass(arg, UploadFile): - return True - return False - - -def is_bytes_sequence_annotation(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - at_least_one = False - for arg in get_args(annotation): - if is_bytes_sequence_annotation(arg): - at_least_one = True - continue - return at_least_one - return field_annotation_is_sequence(annotation) and all( - is_bytes_or_nonable_bytes_annotation(sub_annotation) - for sub_annotation in get_args(annotation) - ) - - -def is_uploadfile_sequence_annotation(annotation: Any) -> bool: - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - at_least_one = False - for arg in get_args(annotation): - if is_uploadfile_sequence_annotation(arg): - at_least_one = True - continue - return at_least_one - return field_annotation_is_sequence(annotation) and all( - is_uploadfile_or_nonable_uploadfile_annotation(sub_annotation) - for sub_annotation in get_args(annotation) - ) - - -def is_pydantic_v1_model_instance(obj: Any) -> bool: - # TODO: remove this function once the required version of Pydantic fully - # removes pydantic.v1 - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - from pydantic import v1 - except ImportError: # pragma: no cover - return False - return isinstance(obj, v1.BaseModel) - - -def is_pydantic_v1_model_class(cls: Any) -> bool: - # TODO: remove this function once the required version of Pydantic fully - # removes pydantic.v1 - try: - with warnings.catch_warnings(): - warnings.simplefilter("ignore", UserWarning) - from pydantic import v1 - except ImportError: # pragma: no cover - return False - return lenient_issubclass(cls, v1.BaseModel) - - -def annotation_is_pydantic_v1(annotation: Any) -> bool: - if is_pydantic_v1_model_class(annotation): - return True - origin = get_origin(annotation) - if origin is Union or origin is UnionType: - for arg in get_args(annotation): - if is_pydantic_v1_model_class(arg): - return True - if field_annotation_is_sequence(annotation): - for sub_annotation in get_args(annotation): - if annotation_is_pydantic_v1(sub_annotation): - return True - return False diff --git a/functions/python/data-processor/fastapi/_compat/v2.py b/functions/python/data-processor/fastapi/_compat/v2.py deleted file mode 100644 index 7be686d86..000000000 --- a/functions/python/data-processor/fastapi/_compat/v2.py +++ /dev/null @@ -1,493 +0,0 @@ -import re -import warnings -from collections.abc import Sequence -from copy import copy -from dataclasses import dataclass, is_dataclass -from enum import Enum -from functools import lru_cache -from typing import ( - Annotated, - Any, - Literal, - Union, - cast, - get_args, - get_origin, -) - -from fastapi._compat import lenient_issubclass, shared -from fastapi.openapi.constants import REF_TEMPLATE -from fastapi.types import IncEx, ModelNameMap, UnionType -from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, create_model -from pydantic import PydanticSchemaGenerationError as PydanticSchemaGenerationError -from pydantic import PydanticUndefinedAnnotation as PydanticUndefinedAnnotation -from pydantic import ValidationError as ValidationError -from pydantic._internal import _typing_extra as _pydantic_typing_extra -from pydantic._internal._schema_generation_shared import ( # type: ignore[attr-defined] - GetJsonSchemaHandler as GetJsonSchemaHandler, -) -from pydantic.fields import FieldInfo as FieldInfo -from pydantic.json_schema import GenerateJsonSchema as _GenerateJsonSchema -from pydantic.json_schema import JsonSchemaValue as JsonSchemaValue -from pydantic_core import CoreSchema as CoreSchema -from pydantic_core import PydanticUndefined -from pydantic_core import Url as Url -from pydantic_core.core_schema import ( - with_info_plain_validator_function as with_info_plain_validator_function, -) - -RequiredParam = PydanticUndefined -Undefined = PydanticUndefined - - -def evaluate_forwardref( - value: Any, - globalns: dict[str, Any] | None = None, - localns: dict[str, Any] | None = None, -) -> Any: - # eval_type_lenient has been deprecated since Pydantic v2.10.0b1 (PR #10530) - try_eval_type = getattr(_pydantic_typing_extra, "try_eval_type", None) - if try_eval_type is not None: - return try_eval_type(value, globalns, localns)[0] - return _pydantic_typing_extra.eval_type_lenient( # ty: ignore[deprecated] - value, globalns, localns - ) - - -class GenerateJsonSchema(_GenerateJsonSchema): - # TODO: remove when this is merged (or equivalent): https://github.com/pydantic/pydantic/pull/12841 - # and dropping support for any version of Pydantic before that one (so, in a very long time) - def bytes_schema(self, schema: CoreSchema) -> JsonSchemaValue: - json_schema = {"type": "string", "contentMediaType": "application/octet-stream"} - bytes_mode = ( - self._config.ser_json_bytes - if self.mode == "serialization" - else self._config.val_json_bytes - ) - if bytes_mode == "base64": - json_schema["contentEncoding"] = "base64" - self.update_with_validations(json_schema, schema, self.ValidationsMapping.bytes) - return json_schema - - -# TODO: remove when dropping support for Pydantic < v2.12.3 -_Attrs = { - "default": ..., - "default_factory": None, - "alias": None, - "alias_priority": None, - "validation_alias": None, - "serialization_alias": None, - "title": None, - "field_title_generator": None, - "description": None, - "examples": None, - "exclude": None, - "exclude_if": None, - "discriminator": None, - "deprecated": None, - "json_schema_extra": None, - "frozen": None, - "validate_default": None, - "repr": True, - "init": None, - "init_var": None, - "kw_only": None, -} - - -# TODO: remove when dropping support for Pydantic < v2.12.3 -def asdict(field_info: FieldInfo) -> dict[str, Any]: - attributes = {} - for attr in _Attrs: - value = getattr(field_info, attr, Undefined) - if value is not Undefined: - attributes[attr] = value - return { - "annotation": field_info.annotation, - "metadata": field_info.metadata, - "attributes": attributes, - } - - -@dataclass -class ModelField: - field_info: FieldInfo - name: str - mode: Literal["validation", "serialization"] = "validation" - config: ConfigDict | None = None - - @property - def alias(self) -> str: - a = self.field_info.alias - return a if a is not None else self.name - - @property - def validation_alias(self) -> str | None: - va = self.field_info.validation_alias - if isinstance(va, str) and va: - return va - return None - - @property - def serialization_alias(self) -> str | None: - sa = self.field_info.serialization_alias - return sa or None - - @property - def default(self) -> Any: - return self.get_default() - - def __post_init__(self) -> None: - with warnings.catch_warnings(): - # Pydantic >= 2.12.0 warns about field specific metadata that is unused - # (e.g. `TypeAdapter(Annotated[int, Field(alias='b')])`). In some cases, we - # end up building the type adapter from a model field annotation so we - # need to ignore the warning: - if shared.PYDANTIC_VERSION_MINOR_TUPLE >= (2, 12): - from pydantic.warnings import UnsupportedFieldAttributeWarning - - warnings.simplefilter( - "ignore", category=UnsupportedFieldAttributeWarning - ) - # TODO: remove after setting the min Pydantic to v2.12.3 - # that adds asdict(), and use self.field_info.asdict() instead - field_dict = asdict(self.field_info) - annotated_args = ( - field_dict["annotation"], - *field_dict["metadata"], - # this FieldInfo needs to be created again so that it doesn't include - # the old field info metadata and only the rest of the attributes - Field(**field_dict["attributes"]), - ) - self._type_adapter: TypeAdapter[Any] = TypeAdapter( - Annotated[annotated_args], # ty: ignore[invalid-type-form] - config=self.config, - ) - - def get_default(self) -> Any: - if self.field_info.is_required(): - return Undefined - return self.field_info.get_default(call_default_factory=True) - - def validate( - self, - value: Any, - values: dict[str, Any] = {}, # noqa: B006 - *, - loc: tuple[int | str, ...] = (), - ) -> tuple[Any, list[dict[str, Any]]]: - try: - return ( - self._type_adapter.validate_python(value, from_attributes=True), - [], - ) - except ValidationError as exc: - return None, _regenerate_error_with_loc( - errors=exc.errors(include_url=False), loc_prefix=loc - ) - - def serialize( - self, - value: Any, - *, - mode: Literal["json", "python"] = "json", - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> Any: - # What calls this code passes a value that already called - # self._type_adapter.validate_python(value) - return self._type_adapter.dump_python( - value, - mode=mode, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - def serialize_json( - self, - value: Any, - *, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> bytes: - # What calls this code passes a value that already called - # self._type_adapter.validate_python(value) - # This uses Pydantic's dump_json() which serializes directly to JSON - # bytes in one pass (via Rust), avoiding the intermediate Python dict - # step of dump_python(mode="json") + json.dumps(). - return self._type_adapter.dump_json( - value, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - def __hash__(self) -> int: - # Each ModelField is unique for our purposes, to allow making a dict from - # ModelField to its JSON Schema. - return id(self) - - -def _has_computed_fields(field: ModelField) -> bool: - computed_fields = field._type_adapter.core_schema.get("schema", {}).get( - "computed_fields", [] - ) - return len(computed_fields) > 0 - - -def get_schema_from_model_field( - *, - field: ModelField, - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], JsonSchemaValue - ], - separate_input_output_schemas: bool = True, -) -> dict[str, Any]: - override_mode: Literal["validation"] | None = ( - None - if (separate_input_output_schemas or _has_computed_fields(field)) - else "validation" - ) - field_alias = ( - (field.validation_alias or field.alias) - if field.mode == "validation" - else (field.serialization_alias or field.alias) - ) - - # This expects that GenerateJsonSchema was already used to generate the definitions - json_schema = field_mapping[(field, override_mode or field.mode)] - if "$ref" not in json_schema: - # TODO remove when deprecating Pydantic v1 - # Ref: https://github.com/pydantic/pydantic/blob/d61792cc42c80b13b23e3ffa74bc37ec7c77f7d1/pydantic/schema.py#L207 - json_schema["title"] = field.field_info.title or field_alias.title().replace( - "_", " " - ) - return json_schema - - -def get_definitions( - *, - fields: Sequence[ModelField], - model_name_map: ModelNameMap, - separate_input_output_schemas: bool = True, -) -> tuple[ - dict[tuple[ModelField, Literal["validation", "serialization"]], JsonSchemaValue], - dict[str, dict[str, Any]], -]: - schema_generator = GenerateJsonSchema(ref_template=REF_TEMPLATE) - validation_fields = [field for field in fields if field.mode == "validation"] - serialization_fields = [field for field in fields if field.mode == "serialization"] - flat_validation_models = get_flat_models_from_fields( - validation_fields, known_models=set() - ) - flat_serialization_models = get_flat_models_from_fields( - serialization_fields, known_models=set() - ) - flat_validation_model_fields = [ - ModelField( - field_info=FieldInfo(annotation=model), - name=model.__name__, - mode="validation", - ) - for model in flat_validation_models - ] - flat_serialization_model_fields = [ - ModelField( - field_info=FieldInfo(annotation=model), - name=model.__name__, - mode="serialization", - ) - for model in flat_serialization_models - ] - flat_model_fields = flat_validation_model_fields + flat_serialization_model_fields - input_types = {f.field_info.annotation for f in fields} - unique_flat_model_fields = { - f for f in flat_model_fields if f.field_info.annotation not in input_types - } - inputs = [ - ( - field, - ( - field.mode - if (separate_input_output_schemas or _has_computed_fields(field)) - else "validation" - ), - field._type_adapter.core_schema, - ) - for field in list(fields) + list(unique_flat_model_fields) - ] - field_mapping, definitions = schema_generator.generate_definitions(inputs=inputs) - for item_def in cast(dict[str, dict[str, Any]], definitions).values(): - if "description" in item_def: - item_description = cast(str, item_def["description"]).split("\f")[0] - item_def["description"] = item_description - # definitions: dict[DefsRef, dict[str, Any]] - # but mypy complains about general str in other places that are not declared as - # DefsRef, although DefsRef is just str: - # DefsRef = NewType('DefsRef', str) - # So, a cast to simplify the types here - return field_mapping, cast(dict[str, dict[str, Any]], definitions) - - -def is_scalar_field(field: ModelField) -> bool: - from fastapi import params - - return shared.field_annotation_is_scalar( - field.field_info.annotation - ) and not isinstance(field.field_info, params.Body) - - -def copy_field_info(*, field_info: FieldInfo, annotation: Any) -> FieldInfo: - cls = type(field_info) - merged_field_info = cls.from_annotation(annotation) - new_field_info = copy(field_info) - new_field_info.metadata = merged_field_info.metadata - new_field_info.annotation = merged_field_info.annotation - return new_field_info - - -def serialize_sequence_value(*, field: ModelField, value: Any) -> Sequence[Any]: - origin_type = get_origin(field.field_info.annotation) or field.field_info.annotation - if origin_type is Union or origin_type is UnionType: # Handle optional sequences - union_args = get_args(field.field_info.annotation) - for union_arg in union_args: - if union_arg is type(None): - continue - origin_type = get_origin(union_arg) or union_arg - break - assert issubclass(origin_type, shared.sequence_types) # type: ignore[arg-type] # ty: ignore[invalid-argument-type] - return shared.sequence_annotation_to_type[origin_type](value) # type: ignore[no-any-return,index] # ty: ignore[invalid-return-type] - - -def get_missing_field_error(loc: tuple[int | str, ...]) -> dict[str, Any]: - error = ValidationError.from_exception_data( - "Field required", [{"type": "missing", "loc": loc, "input": {}}] - ).errors(include_url=False)[0] - error["input"] = None - return error # type: ignore[return-value] # ty: ignore[invalid-return-type] - - -def create_body_model( - *, fields: Sequence[ModelField], model_name: str -) -> type[BaseModel]: - field_params = {f.name: (f.field_info.annotation, f.field_info) for f in fields} - BodyModel: type[BaseModel] = create_model(model_name, **field_params) # type: ignore[call-overload] # ty: ignore[no-matching-overload] - return BodyModel - - -def get_model_fields(model: type[BaseModel]) -> list[ModelField]: - model_fields: list[ModelField] = [] - for name, field_info in model.model_fields.items(): - type_ = field_info.annotation - if lenient_issubclass(type_, (BaseModel, dict)) or is_dataclass(type_): - model_config = None - else: - model_config = model.model_config - model_fields.append( - ModelField( - field_info=field_info, - name=name, - config=model_config, - ) - ) - return model_fields - - -@lru_cache -def get_cached_model_fields(model: type[BaseModel]) -> list[ModelField]: - return get_model_fields(model) - - -# Duplicate of several schema functions from Pydantic v1 to make them compatible with -# Pydantic v2 and allow mixing the models - -TypeModelOrEnum = type["BaseModel"] | type[Enum] -TypeModelSet = set[TypeModelOrEnum] - - -def normalize_name(name: str) -> str: - return re.sub(r"[^a-zA-Z0-9.\-_]", "_", name) - - -def get_model_name_map(unique_models: TypeModelSet) -> dict[TypeModelOrEnum, str]: - name_model_map = {} - for model in unique_models: - model_name = normalize_name(model.__name__) - name_model_map[model_name] = model - return {v: k for k, v in name_model_map.items()} - - -def get_flat_models_from_model( - model: type["BaseModel"], known_models: TypeModelSet | None = None -) -> TypeModelSet: - known_models = known_models or set() - fields = get_model_fields(model) - get_flat_models_from_fields(fields, known_models=known_models) - return known_models - - -def get_flat_models_from_annotation( - annotation: Any, known_models: TypeModelSet -) -> TypeModelSet: - origin = get_origin(annotation) - if origin is not None: - for arg in get_args(annotation): - if lenient_issubclass(arg, (BaseModel, Enum)): - if arg not in known_models: - known_models.add(arg) # type: ignore[arg-type] - if lenient_issubclass(arg, BaseModel): - get_flat_models_from_model(arg, known_models=known_models) - else: - get_flat_models_from_annotation(arg, known_models=known_models) - return known_models - - -def get_flat_models_from_field( - field: ModelField, known_models: TypeModelSet -) -> TypeModelSet: - field_type = field.field_info.annotation - if lenient_issubclass(field_type, BaseModel): - if field_type in known_models: - return known_models - known_models.add(field_type) - get_flat_models_from_model(field_type, known_models=known_models) - elif lenient_issubclass(field_type, Enum): - known_models.add(field_type) - else: - get_flat_models_from_annotation(field_type, known_models=known_models) - return known_models - - -def get_flat_models_from_fields( - fields: Sequence[ModelField], known_models: TypeModelSet -) -> TypeModelSet: - for field in fields: - get_flat_models_from_field(field, known_models=known_models) - return known_models - - -def _regenerate_error_with_loc( - *, errors: Sequence[Any], loc_prefix: tuple[str | int, ...] -) -> list[dict[str, Any]]: - updated_loc_errors: list[Any] = [ - {**err, "loc": loc_prefix + err.get("loc", ())} for err in errors - ] - - return updated_loc_errors diff --git a/functions/python/data-processor/fastapi/applications.py b/functions/python/data-processor/fastapi/applications.py deleted file mode 100644 index faac6853f..000000000 --- a/functions/python/data-processor/fastapi/applications.py +++ /dev/null @@ -1,4691 +0,0 @@ -from collections.abc import Awaitable, Callable, Coroutine, Sequence -from enum import Enum -from typing import Annotated, Any, TypeVar - -from annotated_doc import Doc -from fastapi import routing -from fastapi.datastructures import Default, DefaultPlaceholder -from fastapi.exception_handlers import ( - http_exception_handler, - request_validation_exception_handler, - websocket_request_validation_exception_handler, -) -from fastapi.exceptions import RequestValidationError, WebSocketRequestValidationError -from fastapi.logger import logger -from fastapi.middleware.asyncexitstack import AsyncExitStackMiddleware -from fastapi.openapi.docs import ( - get_redoc_html, - get_swagger_ui_html, - get_swagger_ui_oauth2_redirect_html, -) -from fastapi.openapi.utils import get_openapi -from fastapi.params import Depends -from fastapi.types import DecoratedCallable, IncEx -from fastapi.utils import generate_unique_id -from starlette.applications import Starlette -from starlette.datastructures import State -from starlette.exceptions import HTTPException -from starlette.middleware import Middleware -from starlette.middleware.base import BaseHTTPMiddleware -from starlette.middleware.errors import ServerErrorMiddleware -from starlette.middleware.exceptions import ExceptionMiddleware -from starlette.requests import Request -from starlette.responses import HTMLResponse, JSONResponse, Response -from starlette.routing import BaseRoute -from starlette.types import ASGIApp, ExceptionHandler, Lifespan, Receive, Scope, Send -from typing_extensions import deprecated - -AppType = TypeVar("AppType", bound="FastAPI") - - -class FastAPI(Starlette): - """ - `FastAPI` app class, the main entrypoint to use FastAPI. - - Read more in the - [FastAPI docs for First Steps](https://fastapi.tiangolo.com/tutorial/first-steps/). - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - ``` - """ - - def __init__( - self: AppType, - *, - debug: Annotated[ - bool, - Doc( - """ - Boolean indicating if debug tracebacks should be returned on server - errors. - - Read more in the - [Starlette docs for Applications](https://www.starlette.dev/applications/#instantiating-the-application). - """ - ), - ] = False, - routes: Annotated[ - list[BaseRoute] | None, - Doc( - """ - **Note**: you probably shouldn't use this parameter, it is inherited - from Starlette and supported for compatibility. - - --- - - A list of routes to serve incoming HTTP and WebSocket requests. - """ - ), - deprecated( - """ - You normally wouldn't use this parameter with FastAPI, it is inherited - from Starlette and supported for compatibility. - - In FastAPI, you normally would use the *path operation methods*, - like `app.get()`, `app.post()`, etc. - """ - ), - ] = None, - title: Annotated[ - str, - Doc( - """ - The title of the API. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(title="ChimichangApp") - ``` - """ - ), - ] = "FastAPI", - summary: Annotated[ - str | None, - Doc( - """ - A short summary of the API. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(summary="Deadpond's favorite app. Nuff said.") - ``` - """ - ), - ] = None, - description: Annotated[ - str, - Doc( - ''' - A description of the API. Supports Markdown (using - [CommonMark syntax](https://commonmark.org/)). - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI( - description=""" - ChimichangApp API helps you do awesome stuff. 🚀 - - ## Items - - You can **read items**. - - ## Users - - You will be able to: - - * **Create users** (_not implemented_). - * **Read users** (_not implemented_). - - """ - ) - ``` - ''' - ), - ] = "", - version: Annotated[ - str, - Doc( - """ - The version of the API. - - **Note** This is the version of your application, not the version of - the OpenAPI specification nor the version of FastAPI being used. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(version="0.0.1") - ``` - """ - ), - ] = "0.1.0", - openapi_url: Annotated[ - str | None, - Doc( - """ - The URL where the OpenAPI schema will be served from. - - If you set it to `None`, no OpenAPI schema will be served publicly, and - the default automatic endpoints `/docs` and `/redoc` will also be - disabled. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#openapi-url). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(openapi_url="/api/v1/openapi.json") - ``` - """ - ), - ] = "/openapi.json", - openapi_tags: Annotated[ - list[dict[str, Any]] | None, - Doc( - """ - A list of tags used by OpenAPI, these are the same `tags` you can set - in the *path operations*, like: - - * `@app.get("/users/", tags=["users"])` - * `@app.get("/items/", tags=["items"])` - - The order of the tags can be used to specify the order shown in - tools like Swagger UI, used in the automatic path `/docs`. - - It's not required to specify all the tags used. - - The tags that are not declared MAY be organized randomly or based - on the tools' logic. Each tag name in the list MUST be unique. - - The value of each item is a `dict` containing: - - * `name`: The name of the tag. - * `description`: A short description of the tag. - [CommonMark syntax](https://commonmark.org/) MAY be used for rich - text representation. - * `externalDocs`: Additional external documentation for this tag. If - provided, it would contain a `dict` with: - * `description`: A short description of the target documentation. - [CommonMark syntax](https://commonmark.org/) MAY be used for - rich text representation. - * `url`: The URL for the target documentation. Value MUST be in - the form of a URL. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-tags). - - **Example** - - ```python - from fastapi import FastAPI - - tags_metadata = [ - { - "name": "users", - "description": "Operations with users. The **login** logic is also here.", - }, - { - "name": "items", - "description": "Manage items. So _fancy_ they have their own docs.", - "externalDocs": { - "description": "Items external docs", - "url": "https://fastapi.tiangolo.com/", - }, - }, - ] - - app = FastAPI(openapi_tags=tags_metadata) - ``` - """ - ), - ] = None, - servers: Annotated[ - list[dict[str, str | Any]] | None, - Doc( - """ - A `list` of `dict`s with connectivity information to a target server. - - You would use it, for example, if your application is served from - different domains and you want to use the same Swagger UI in the - browser to interact with each of them (instead of having multiple - browser tabs open). Or if you want to leave fixed the possible URLs. - - If the servers `list` is not provided, or is an empty `list`, the - `servers` property in the generated OpenAPI will be: - - * a `dict` with a `url` value of the application's mounting point - (`root_path`) if it's different from `/`. - * otherwise, the `servers` property will be omitted from the OpenAPI - schema. - - Each item in the `list` is a `dict` containing: - - * `url`: A URL to the target host. This URL supports Server Variables - and MAY be relative, to indicate that the host location is relative - to the location where the OpenAPI document is being served. Variable - substitutions will be made when a variable is named in `{`brackets`}`. - * `description`: An optional string describing the host designated by - the URL. [CommonMark syntax](https://commonmark.org/) MAY be used for - rich text representation. - * `variables`: A `dict` between a variable name and its value. The value - is used for substitution in the server's URL template. - - Read more in the - [FastAPI docs for Behind a Proxy](https://fastapi.tiangolo.com/advanced/behind-a-proxy/#additional-servers). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI( - servers=[ - {"url": "https://stag.example.com", "description": "Staging environment"}, - {"url": "https://prod.example.com", "description": "Production environment"}, - ] - ) - ``` - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of global dependencies, they will be applied to each - *path operation*, including in sub-routers. - - Read more about it in the - [FastAPI docs for Global Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/global-dependencies/). - - **Example** - - ```python - from fastapi import Depends, FastAPI - - from .dependencies import func_dep_1, func_dep_2 - - app = FastAPI(dependencies=[Depends(func_dep_1), Depends(func_dep_2)]) - ``` - """ - ), - ] = None, - default_response_class: Annotated[ - type[Response], - Doc( - """ - The default response class to be used. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - - **Example** - - ```python - from fastapi import FastAPI - from fastapi.responses import ORJSONResponse - - app = FastAPI(default_response_class=ORJSONResponse) - ``` - """ - ), - ] = Default(JSONResponse), - redirect_slashes: Annotated[ - bool, - Doc( - """ - Whether to detect and redirect slashes in URLs when the client doesn't - use the same format. - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(redirect_slashes=True) # the default - - @app.get("/items/") - async def read_items(): - return [{"item_id": "Foo"}] - ``` - - With this app, if a client goes to `/items` (without a trailing slash), - they will be automatically redirected with an HTTP status code of 307 - to `/items/`. - """ - ), - ] = True, - docs_url: Annotated[ - str | None, - Doc( - """ - The path to the automatic interactive API documentation. - It is handled in the browser by Swagger UI. - - The default URL is `/docs`. You can disable it by setting it to `None`. - - If `openapi_url` is set to `None`, this will be automatically disabled. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#docs-urls). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(docs_url="/documentation", redoc_url=None) - ``` - """ - ), - ] = "/docs", - redoc_url: Annotated[ - str | None, - Doc( - """ - The path to the alternative automatic interactive API documentation - provided by ReDoc. - - The default URL is `/redoc`. You can disable it by setting it to `None`. - - If `openapi_url` is set to `None`, this will be automatically disabled. - - Read more in the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#docs-urls). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(docs_url="/documentation", redoc_url="redocumentation") - ``` - """ - ), - ] = "/redoc", - swagger_ui_oauth2_redirect_url: Annotated[ - str | None, - Doc( - """ - The OAuth2 redirect endpoint for the Swagger UI. - - By default it is `/docs/oauth2-redirect`. - - This is only used if you use OAuth2 (with the "Authorize" button) - with Swagger UI. - """ - ), - ] = "/docs/oauth2-redirect", - swagger_ui_init_oauth: Annotated[ - dict[str, Any] | None, - Doc( - """ - OAuth2 configuration for the Swagger UI, by default shown at `/docs`. - - Read more about the available configuration options in the - [Swagger UI docs](https://swagger.io/docs/open-source-tools/swagger-ui/usage/oauth2/). - """ - ), - ] = None, - middleware: Annotated[ - Sequence[Middleware] | None, - Doc( - """ - List of middleware to be added when creating the application. - - In FastAPI you would normally do this with `app.add_middleware()` - instead. - - Read more in the - [FastAPI docs for Middleware](https://fastapi.tiangolo.com/tutorial/middleware/). - """ - ), - ] = None, - exception_handlers: Annotated[ - dict[ - int | type[Exception], - Callable[[Request, Any], Coroutine[Any, Any, Response]], - ] - | None, - Doc( - """ - A dictionary with handlers for exceptions. - - In FastAPI, you would normally use the decorator - `@app.exception_handler()`. - - Read more in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/). - """ - ), - ] = None, - on_startup: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of startup event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - on_shutdown: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of shutdown event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - lifespan: Annotated[ - Lifespan[AppType] | None, - Doc( - """ - A `Lifespan` context manager handler. This replaces `startup` and - `shutdown` functions with a single context manager. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - terms_of_service: Annotated[ - str | None, - Doc( - """ - A URL to the Terms of Service for your API. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more at the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - app = FastAPI(terms_of_service="http://example.com/terms/") - ``` - """ - ), - ] = None, - contact: Annotated[ - dict[str, str | Any] | None, - Doc( - """ - A dictionary with the contact information for the exposed API. - - It can contain several fields. - - * `name`: (`str`) The name of the contact person/organization. - * `url`: (`str`) A URL pointing to the contact information. MUST be in - the format of a URL. - * `email`: (`str`) The email address of the contact person/organization. - MUST be in the format of an email address. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more at the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - app = FastAPI( - contact={ - "name": "Deadpoolio the Amazing", - "url": "http://x-force.example.com/contact/", - "email": "dp@x-force.example.com", - } - ) - ``` - """ - ), - ] = None, - license_info: Annotated[ - dict[str, str | Any] | None, - Doc( - """ - A dictionary with the license information for the exposed API. - - It can contain several fields. - - * `name`: (`str`) **REQUIRED** (if a `license_info` is set). The - license name used for the API. - * `identifier`: (`str`) An [SPDX](https://spdx.dev/) license expression - for the API. The `identifier` field is mutually exclusive of the `url` - field. Available since OpenAPI 3.1.0, FastAPI 0.99.0. - * `url`: (`str`) A URL to the license used for the API. This MUST be - the format of a URL. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more at the - [FastAPI docs for Metadata and Docs URLs](https://fastapi.tiangolo.com/tutorial/metadata/#metadata-for-api). - - **Example** - - ```python - app = FastAPI( - license_info={ - "name": "Apache 2.0", - "url": "https://www.apache.org/licenses/LICENSE-2.0.html", - } - ) - ``` - """ - ), - ] = None, - openapi_prefix: Annotated[ - str, - Doc( - """ - A URL prefix for the OpenAPI URL. - """ - ), - deprecated( - """ - "openapi_prefix" has been deprecated in favor of "root_path", which - follows more closely the ASGI standard, is simpler, and more - automatic. - """ - ), - ] = "", - root_path: Annotated[ - str, - Doc( - """ - A path prefix handled by a proxy that is not seen by the application - but is seen by external clients, which affects things like Swagger UI. - - Read more about it at the - [FastAPI docs for Behind a Proxy](https://fastapi.tiangolo.com/advanced/behind-a-proxy/). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(root_path="/api/v1") - ``` - """ - ), - ] = "", - root_path_in_servers: Annotated[ - bool, - Doc( - """ - To disable automatically generating the URLs in the `servers` field - in the autogenerated OpenAPI using the `root_path`. - - Read more about it in the - [FastAPI docs for Behind a Proxy](https://fastapi.tiangolo.com/advanced/behind-a-proxy/#disable-automatic-server-from-root-path). - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI(root_path_in_servers=False) - ``` - """ - ), - ] = True, - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - OpenAPI callbacks that should apply to all *path operations*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - webhooks: Annotated[ - routing.APIRouter | None, - Doc( - """ - Add OpenAPI webhooks. This is similar to `callbacks` but it doesn't - depend on specific *path operations*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - **Note**: This is available since OpenAPI 3.1.0, FastAPI 0.99.0. - - Read more about it in the - [FastAPI docs for OpenAPI Webhooks](https://fastapi.tiangolo.com/advanced/openapi-webhooks/). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all *path operations* as deprecated. You probably don't need it, - but it's available. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#deprecate-a-path-operation). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) all the *path operations* in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - swagger_ui_parameters: Annotated[ - dict[str, Any] | None, - Doc( - """ - Parameters to configure Swagger UI, the autogenerated interactive API - documentation (by default at `/docs`). - - Read more about it in the - [FastAPI docs about how to Configure Swagger UI](https://fastapi.tiangolo.com/how-to/configure-swagger-ui/). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - separate_input_output_schemas: Annotated[ - bool, - Doc( - """ - Whether to generate separate OpenAPI schemas for request body and - response body when the results would be more precise. - - This is particularly useful when automatically generating clients. - - For example, if you have a model like: - - ```python - from pydantic import BaseModel - - class Item(BaseModel): - name: str - tags: list[str] = [] - ``` - - When `Item` is used for input, a request body, `tags` is not required, - the client doesn't have to provide it. - - But when using `Item` for output, for a response body, `tags` is always - available because it has a default value, even if it's just an empty - list. So, the client should be able to always expect it. - - In this case, there would be two different schemas, one for input and - another one for output. - - Read more about it in the - [FastAPI docs about how to separate schemas for input and output](https://fastapi.tiangolo.com/how-to/separate-openapi-schemas) - """ - ), - ] = True, - openapi_external_docs: Annotated[ - dict[str, Any] | None, - Doc( - """ - This field allows you to provide additional external documentation links. - If provided, it must be a dictionary containing: - - * `description`: A brief description of the external documentation. - * `url`: The URL pointing to the external documentation. The value **MUST** - be a valid URL format. - - **Example**: - - ```python - from fastapi import FastAPI - - external_docs = { - "description": "Detailed API Reference", - "url": "https://example.com/api-docs", - } - - app = FastAPI(openapi_external_docs=external_docs) - ``` - """ - ), - ] = None, - strict_content_type: Annotated[ - bool, - Doc( - """ - Enable strict checking for request Content-Type headers. - - When `True` (the default), requests with a body that do not include - a `Content-Type` header will **not** be parsed as JSON. - - This prevents potential cross-site request forgery (CSRF) attacks - that exploit the browser's ability to send requests without a - Content-Type header, bypassing CORS preflight checks. In particular - applicable for apps that need to be run locally (in localhost). - - When `False`, requests without a `Content-Type` header will have - their body parsed as JSON, which maintains compatibility with - certain clients that don't send `Content-Type` headers. - - Read more about it in the - [FastAPI docs for Strict Content-Type](https://fastapi.tiangolo.com/advanced/strict-content-type/). - """ - ), - ] = True, - **extra: Annotated[ - Any, - Doc( - """ - Extra keyword arguments to be stored in the app, not used by FastAPI - anywhere. - """ - ), - ], - ) -> None: - self.debug = debug - self.title = title - self.summary = summary - self.description = description - self.version = version - self.terms_of_service = terms_of_service - self.contact = contact - self.license_info = license_info - self.openapi_url = openapi_url - self.openapi_tags = openapi_tags - self.root_path_in_servers = root_path_in_servers - self.docs_url = docs_url - self.redoc_url = redoc_url - self.swagger_ui_oauth2_redirect_url = swagger_ui_oauth2_redirect_url - self.swagger_ui_init_oauth = swagger_ui_init_oauth - self.swagger_ui_parameters = swagger_ui_parameters - self.servers = servers or [] - self.separate_input_output_schemas = separate_input_output_schemas - self.openapi_external_docs = openapi_external_docs - self.extra = extra - self.openapi_version: Annotated[ - str, - Doc( - """ - The version string of OpenAPI. - - FastAPI will generate OpenAPI version 3.1.0, and will output that as - the OpenAPI version. But some tools, even though they might be - compatible with OpenAPI 3.1.0, might not recognize it as a valid. - - So you could override this value to trick those tools into using - the generated OpenAPI. Have in mind that this is a hack. But if you - avoid using features added in OpenAPI 3.1.0, it might work for your - use case. - - This is not passed as a parameter to the `FastAPI` class to avoid - giving the false idea that FastAPI would generate a different OpenAPI - schema. It is only available as an attribute. - - **Example** - - ```python - from fastapi import FastAPI - - app = FastAPI() - - app.openapi_version = "3.0.2" - ``` - """ - ), - ] = "3.1.0" - self.openapi_schema: dict[str, Any] | None = None - if self.openapi_url: - assert self.title, "A title must be provided for OpenAPI, e.g.: 'My API'" - assert self.version, "A version must be provided for OpenAPI, e.g.: '2.1.0'" - # TODO: remove when discarding the openapi_prefix parameter - if openapi_prefix: - logger.warning( - '"openapi_prefix" has been deprecated in favor of "root_path", which ' - "follows more closely the ASGI standard, is simpler, and more " - "automatic. Check the docs at " - "https://fastapi.tiangolo.com/advanced/sub-applications/" - ) - self.webhooks: Annotated[ - routing.APIRouter, - Doc( - """ - The `app.webhooks` attribute is an `APIRouter` with the *path - operations* that will be used just for documentation of webhooks. - - Read more about it in the - [FastAPI docs for OpenAPI Webhooks](https://fastapi.tiangolo.com/advanced/openapi-webhooks/). - """ - ), - ] = webhooks or routing.APIRouter() - self.root_path = root_path or openapi_prefix - self.state: Annotated[ - State, - Doc( - """ - A state object for the application. This is the same object for the - entire application, it doesn't change from request to request. - - You normally wouldn't use this in FastAPI, for most of the cases you - would instead use FastAPI dependencies. - - This is simply inherited from Starlette. - - Read more about it in the - [Starlette docs for Applications](https://www.starlette.dev/applications/#storing-state-on-the-app-instance). - """ - ), - ] = State() - self.dependency_overrides: Annotated[ - dict[Callable[..., Any], Callable[..., Any]], - Doc( - """ - A dictionary with overrides for the dependencies. - - Each key is the original dependency callable, and the value is the - actual dependency that should be called. - - This is for testing, to replace expensive dependencies with testing - versions. - - Read more about it in the - [FastAPI docs for Testing Dependencies with Overrides](https://fastapi.tiangolo.com/advanced/testing-dependencies/). - """ - ), - ] = {} - self.router: routing.APIRouter = routing.APIRouter( - routes=routes, - redirect_slashes=redirect_slashes, - dependency_overrides_provider=self, - on_startup=on_startup, - on_shutdown=on_shutdown, - lifespan=lifespan, - default_response_class=default_response_class, - dependencies=dependencies, - callbacks=callbacks, - deprecated=deprecated, - include_in_schema=include_in_schema, - responses=responses, - generate_unique_id_function=generate_unique_id_function, - strict_content_type=strict_content_type, - ) - self.exception_handlers: dict[ - Any, Callable[[Request, Any], Response | Awaitable[Response]] - ] = {} if exception_handlers is None else dict(exception_handlers) - self.exception_handlers.setdefault(HTTPException, http_exception_handler) - self.exception_handlers.setdefault( - RequestValidationError, request_validation_exception_handler - ) - - # Starlette still has incorrect type specification for the handlers - self.exception_handlers.setdefault( - WebSocketRequestValidationError, - websocket_request_validation_exception_handler, # type: ignore[arg-type] - ) # ty: ignore[no-matching-overload] - - self.user_middleware: list[Middleware] = ( - [] if middleware is None else list(middleware) - ) - self.middleware_stack: ASGIApp | None = None - self.setup() - - def build_middleware_stack(self) -> ASGIApp: - # Duplicate/override from Starlette to add AsyncExitStackMiddleware - # inside of ExceptionMiddleware, inside of custom user middlewares - debug = self.debug - error_handler = None - exception_handlers: dict[Any, ExceptionHandler] = {} - - for key, value in self.exception_handlers.items(): - if key in (500, Exception): - error_handler = value - else: - exception_handlers[key] = value - - middleware = ( - [Middleware(ServerErrorMiddleware, handler=error_handler, debug=debug)] - + self.user_middleware - + [ - Middleware( - ExceptionMiddleware, - handlers=exception_handlers, - debug=debug, - ), - # Add FastAPI-specific AsyncExitStackMiddleware for closing files. - # Before this was also used for closing dependencies with yield but - # those now have their own AsyncExitStack, to properly support - # streaming responses while keeping compatibility with the previous - # versions (as of writing 0.117.1) that allowed doing - # except HTTPException inside a dependency with yield. - # This needs to happen after user middlewares because those create a - # new contextvars context copy by using a new AnyIO task group. - # This AsyncExitStack preserves the context for contextvars, not - # strictly necessary for closing files but it was one of the original - # intentions. - # If the AsyncExitStack lived outside of the custom middlewares and - # contextvars were set, for example in a dependency with 'yield' - # in that internal contextvars context, the values would not be - # available in the outer context of the AsyncExitStack. - # By placing the middleware and the AsyncExitStack here, inside all - # user middlewares, the same context is used. - # This is currently not needed, only for closing files, but used to be - # important when dependencies with yield were closed here. - Middleware(AsyncExitStackMiddleware), - ] - ) - - app = self.router - for cls, args, kwargs in reversed(middleware): - app = cls(app, *args, **kwargs) - return app - - def openapi(self) -> dict[str, Any]: - """ - Generate the OpenAPI schema of the application. This is called by FastAPI - internally. - - The first time it is called it stores the result in the attribute - `app.openapi_schema`, and next times it is called, it just returns that same - result. To avoid the cost of generating the schema every time. - - If you need to modify the generated OpenAPI schema, you could modify it. - - Read more in the - [FastAPI docs for OpenAPI](https://fastapi.tiangolo.com/how-to/extending-openapi/). - """ - if not self.openapi_schema: - self.openapi_schema = get_openapi( - title=self.title, - version=self.version, - openapi_version=self.openapi_version, - summary=self.summary, - description=self.description, - terms_of_service=self.terms_of_service, - contact=self.contact, - license_info=self.license_info, - routes=self.routes, - webhooks=self.webhooks.routes, - tags=self.openapi_tags, - servers=self.servers, - separate_input_output_schemas=self.separate_input_output_schemas, - external_docs=self.openapi_external_docs, - ) - return self.openapi_schema - - def setup(self) -> None: - if self.openapi_url: - - async def openapi(req: Request) -> JSONResponse: - root_path = req.scope.get("root_path", "").rstrip("/") - schema = self.openapi() - if root_path and self.root_path_in_servers: - server_urls = {s.get("url") for s in schema.get("servers", [])} - if root_path not in server_urls: - schema = dict(schema) - schema["servers"] = [{"url": root_path}] + schema.get( - "servers", [] - ) - return JSONResponse(schema) - - self.add_route(self.openapi_url, openapi, include_in_schema=False) - if self.openapi_url and self.docs_url: - - async def swagger_ui_html(req: Request) -> HTMLResponse: - root_path = req.scope.get("root_path", "").rstrip("/") - openapi_url = root_path + self.openapi_url - oauth2_redirect_url = self.swagger_ui_oauth2_redirect_url - if oauth2_redirect_url: - oauth2_redirect_url = root_path + oauth2_redirect_url - return get_swagger_ui_html( - openapi_url=openapi_url, - title=f"{self.title} - Swagger UI", - oauth2_redirect_url=oauth2_redirect_url, - init_oauth=self.swagger_ui_init_oauth, - swagger_ui_parameters=self.swagger_ui_parameters, - ) - - self.add_route(self.docs_url, swagger_ui_html, include_in_schema=False) - - if self.swagger_ui_oauth2_redirect_url: - - async def swagger_ui_redirect(req: Request) -> HTMLResponse: - return get_swagger_ui_oauth2_redirect_html() - - self.add_route( - self.swagger_ui_oauth2_redirect_url, - swagger_ui_redirect, - include_in_schema=False, - ) - if self.openapi_url and self.redoc_url: - - async def redoc_html(req: Request) -> HTMLResponse: - root_path = req.scope.get("root_path", "").rstrip("/") - openapi_url = root_path + self.openapi_url - return get_redoc_html( - openapi_url=openapi_url, title=f"{self.title} - ReDoc" - ) - - self.add_route(self.redoc_url, redoc_html, include_in_schema=False) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if self.root_path: - scope["root_path"] = self.root_path - await super().__call__(scope, receive, send) - - def add_api_route( - self, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - name: str | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[routing.APIRoute], str] = Default( - generate_unique_id - ), - ) -> None: - self.router.add_api_route( - path, - endpoint=endpoint, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def api_route( - self, - path: str, - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] = Default(JSONResponse), - name: str | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[routing.APIRoute], str] = Default( - generate_unique_id - ), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.router.add_api_route( - path, - func, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - return func - - return decorator - - def add_api_websocket_route( - self, - path: str, - endpoint: Callable[..., Any], - name: str | None = None, - *, - dependencies: Sequence[Depends] | None = None, - ) -> None: - self.router.add_api_websocket_route( - path, - endpoint, - name=name, - dependencies=dependencies, - ) - - def websocket( - self, - path: Annotated[ - str, - Doc( - """ - WebSocket path. - """ - ), - ], - name: Annotated[ - str | None, - Doc( - """ - A name for the WebSocket. Only used internally. - """ - ), - ] = None, - *, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be used for this - WebSocket. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - """ - ), - ] = None, - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Decorate a WebSocket function. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - - **Example** - - ```python - from fastapi import FastAPI, WebSocket - - app = FastAPI() - - @app.websocket("/ws") - async def websocket_endpoint(websocket: WebSocket): - await websocket.accept() - while True: - data = await websocket.receive_text() - await websocket.send_text(f"Message text was: {data}") - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_api_websocket_route( - path, - func, - name=name, - dependencies=dependencies, - ) - return func - - return decorator - - def include_router( - self, - router: Annotated[routing.APIRouter, Doc("The `APIRouter` to include.")], - *, - prefix: Annotated[str, Doc("An optional path prefix for the router.")] = "", - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to all the *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to all the - *path operations* in this router. - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - - **Example** - - ```python - from fastapi import Depends, FastAPI - - from .dependencies import get_token_header - from .internal import admin - - app = FastAPI() - - app.include_router( - admin.router, - dependencies=[Depends(get_token_header)], - ) - ``` - """ - ), - ] = None, - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all the *path operations* in this router as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - **Example** - - ```python - from fastapi import FastAPI - - from .internal import old_api - - app = FastAPI() - - app.include_router( - old_api.router, - deprecated=True, - ) - ``` - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include (or not) all the *path operations* in this router in the - generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - **Example** - - ```python - from fastapi import FastAPI - - from .internal import old_api - - app = FastAPI() - - app.include_router( - old_api.router, - include_in_schema=False, - ) - ``` - """ - ), - ] = True, - default_response_class: Annotated[ - type[Response], - Doc( - """ - Default response class to be used for the *path operations* in this - router. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - - **Example** - - ```python - from fastapi import FastAPI - from fastapi.responses import ORJSONResponse - - from .internal import old_api - - app = FastAPI() - - app.include_router( - old_api.router, - default_response_class=ORJSONResponse, - ) - ``` - """ - ), - ] = Default(JSONResponse), - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> None: - """ - Include an `APIRouter` in the same app. - - Read more about it in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/). - - ## Example - - ```python - from fastapi import FastAPI - - from .users import users_router - - app = FastAPI() - - app.include_router(users_router) - ``` - """ - self.router.include_router( - router, - prefix=prefix, - tags=tags, - dependencies=dependencies, - responses=responses, - deprecated=deprecated, - include_in_schema=include_in_schema, - default_response_class=default_response_class, - callbacks=callbacks, - generate_unique_id_function=generate_unique_id_function, - ) - - def get( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP GET operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.get("/items/") - def read_items(): - return [{"name": "Empanada"}, {"name": "Arepa"}] - ``` - """ - return self.router.get( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def put( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PUT operation. - - ## Example - - ```python - from fastapi import FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - - @app.put("/items/{item_id}") - def replace_item(item_id: str, item: Item): - return {"message": "Item replaced", "id": item_id} - ``` - """ - return self.router.put( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def post( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP POST operation. - - ## Example - - ```python - from fastapi import FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - - @app.post("/items/") - def create_item(item: Item): - return {"message": "Item created"} - ``` - """ - return self.router.post( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def delete( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP DELETE operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.delete("/items/{item_id}") - def delete_item(item_id: str): - return {"message": "Item deleted"} - ``` - """ - return self.router.delete( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def options( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP OPTIONS operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.options("/items/") - def get_item_options(): - return {"additions": ["Aji", "Guacamole"]} - ``` - """ - return self.router.options( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def head( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP HEAD operation. - - ## Example - - ```python - from fastapi import FastAPI, Response - - app = FastAPI() - - @app.head("/items/", status_code=204) - def get_items_headers(response: Response): - response.headers["X-Cat-Dog"] = "Alone in the world" - ``` - """ - return self.router.head( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def patch( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PATCH operation. - - ## Example - - ```python - from fastapi import FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - - @app.patch("/items/") - def update_item(item: Item): - return {"message": "Item updated in place"} - ``` - """ - return self.router.patch( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def trace( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[routing.APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP TRACE operation. - - ## Example - - ```python - from fastapi import FastAPI - - app = FastAPI() - - @app.trace("/items/{item_id}") - def trace_item(item_id: str): - return None - ``` - """ - return self.router.trace( - path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def websocket_route( - self, path: str, name: str | None = None - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.router.add_websocket_route(path, func, name=name) - return func - - return decorator - - @deprecated( - """ - on_event is deprecated, use lifespan event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). - """ - ) - def on_event( - self, - event_type: Annotated[ - str, - Doc( - """ - The type of event. `startup` or `shutdown`. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add an event handler for the application. - - `on_event` is deprecated, use `lifespan` event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/#alternative-events-deprecated). - """ - return self.router.on_event(event_type) # ty: ignore[deprecated] - - def middleware( - self, - middleware_type: Annotated[ - str, - Doc( - """ - The type of middleware. Currently only supports `http`. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a middleware to the application. - - Read more about it in the - [FastAPI docs for Middleware](https://fastapi.tiangolo.com/tutorial/middleware/). - - ## Example - - ```python - import time - from typing import Awaitable, Callable - - from fastapi import FastAPI, Request, Response - - app = FastAPI() - - - @app.middleware("http") - async def add_process_time_header( - request: Request, call_next: Callable[[Request], Awaitable[Response]] - ) -> Response: - start_time = time.time() - response = await call_next(request) - process_time = time.time() - start_time - response.headers["X-Process-Time"] = str(process_time) - return response - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_middleware(BaseHTTPMiddleware, dispatch=func) - return func - - return decorator - - def exception_handler( - self, - exc_class_or_status_code: Annotated[ - int | type[Exception], - Doc( - """ - The Exception class this would handle, or a status code. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add an exception handler to the app. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/). - - ## Example - - ```python - from fastapi import FastAPI, Request - from fastapi.responses import JSONResponse - - - class UnicornException(Exception): - def __init__(self, name: str): - self.name = name - - - app = FastAPI() - - - @app.exception_handler(UnicornException) - async def unicorn_exception_handler(request: Request, exc: UnicornException): - return JSONResponse( - status_code=418, - content={"message": f"Oops! {exc.name} did something. There goes a rainbow..."}, - ) - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_exception_handler(exc_class_or_status_code, func) - return func - - return decorator diff --git a/functions/python/data-processor/fastapi/background.py b/functions/python/data-processor/fastapi/background.py deleted file mode 100644 index 7677058c4..000000000 --- a/functions/python/data-processor/fastapi/background.py +++ /dev/null @@ -1,61 +0,0 @@ -from collections.abc import Callable -from typing import Annotated, Any - -from annotated_doc import Doc -from starlette.background import BackgroundTasks as StarletteBackgroundTasks -from typing_extensions import ParamSpec - -P = ParamSpec("P") - - -class BackgroundTasks(StarletteBackgroundTasks): - """ - A collection of background tasks that will be called after a response has been - sent to the client. - - Read more about it in the - [FastAPI docs for Background Tasks](https://fastapi.tiangolo.com/tutorial/background-tasks/). - - ## Example - - ```python - from fastapi import BackgroundTasks, FastAPI - - app = FastAPI() - - - def write_notification(email: str, message=""): - with open("log.txt", mode="w") as email_file: - content = f"notification for {email}: {message}" - email_file.write(content) - - - @app.post("/send-notification/{email}") - async def send_notification(email: str, background_tasks: BackgroundTasks): - background_tasks.add_task(write_notification, email, message="some notification") - return {"message": "Notification sent in the background"} - ``` - """ - - def add_task( - self, - func: Annotated[ - Callable[P, Any], - Doc( - """ - The function to call after the response is sent. - - It can be a regular `def` function or an `async def` function. - """ - ), - ], - *args: P.args, - **kwargs: P.kwargs, - ) -> None: - """ - Add a function to be called in the background after the response is sent. - - Read more about it in the - [FastAPI docs for Background Tasks](https://fastapi.tiangolo.com/tutorial/background-tasks/). - """ - return super().add_task(func, *args, **kwargs) diff --git a/functions/python/data-processor/fastapi/cli.py b/functions/python/data-processor/fastapi/cli.py deleted file mode 100644 index 8d3301e9d..000000000 --- a/functions/python/data-processor/fastapi/cli.py +++ /dev/null @@ -1,13 +0,0 @@ -try: - from fastapi_cli.cli import main as cli_main - -except ImportError: # pragma: no cover - cli_main = None # type: ignore - - -def main() -> None: - if not cli_main: # type: ignore[truthy-function] - message = 'To use the fastapi command, please install "fastapi[standard]":\n\n\tpip install "fastapi[standard]"\n' - print(message) - raise RuntimeError(message) # noqa: B904 - cli_main() diff --git a/functions/python/data-processor/fastapi/concurrency.py b/functions/python/data-processor/fastapi/concurrency.py deleted file mode 100644 index 76a5a2eb1..000000000 --- a/functions/python/data-processor/fastapi/concurrency.py +++ /dev/null @@ -1,41 +0,0 @@ -from collections.abc import AsyncGenerator -from contextlib import AbstractContextManager -from contextlib import asynccontextmanager as asynccontextmanager -from typing import TypeVar - -import anyio.to_thread -from anyio import CapacityLimiter -from starlette.concurrency import iterate_in_threadpool as iterate_in_threadpool # noqa -from starlette.concurrency import run_in_threadpool as run_in_threadpool # noqa -from starlette.concurrency import ( # noqa - run_until_first_complete as run_until_first_complete, -) - -_T = TypeVar("_T") - - -@asynccontextmanager -async def contextmanager_in_threadpool( - cm: AbstractContextManager[_T], -) -> AsyncGenerator[_T, None]: - # blocking __exit__ from running waiting on a free thread - # can create race conditions/deadlocks if the context manager itself - # has its own internal pool (e.g. a database connection pool) - # to avoid this we let __exit__ run without a capacity limit - # since we're creating a new limiter for each call, any non-zero limit - # works (1 is arbitrary) - exit_limiter = CapacityLimiter(1) - try: - yield await run_in_threadpool(cm.__enter__) - except Exception as e: - ok = bool( - await anyio.to_thread.run_sync( - cm.__exit__, type(e), e, e.__traceback__, limiter=exit_limiter - ) - ) - if not ok: - raise e - else: - await anyio.to_thread.run_sync( - cm.__exit__, None, None, None, limiter=exit_limiter - ) diff --git a/functions/python/data-processor/fastapi/datastructures.py b/functions/python/data-processor/fastapi/datastructures.py deleted file mode 100644 index 1da784cf0..000000000 --- a/functions/python/data-processor/fastapi/datastructures.py +++ /dev/null @@ -1,186 +0,0 @@ -from collections.abc import Callable, Mapping -from typing import ( - Annotated, - Any, - BinaryIO, - TypeVar, - cast, -) - -from annotated_doc import Doc -from pydantic import GetJsonSchemaHandler -from starlette.datastructures import URL as URL # noqa: F401 -from starlette.datastructures import Address as Address # noqa: F401 -from starlette.datastructures import FormData as FormData # noqa: F401 -from starlette.datastructures import Headers as Headers # noqa: F401 -from starlette.datastructures import QueryParams as QueryParams # noqa: F401 -from starlette.datastructures import State as State # noqa: F401 -from starlette.datastructures import UploadFile as StarletteUploadFile - - -class UploadFile(StarletteUploadFile): - """ - A file uploaded in a request. - - Define it as a *path operation function* (or dependency) parameter. - - If you are using a regular `def` function, you can use the `upload_file.file` - attribute to access the raw standard Python file (blocking, not async), useful and - needed for non-async code. - - Read more about it in the - [FastAPI docs for Request Files](https://fastapi.tiangolo.com/tutorial/request-files/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import FastAPI, File, UploadFile - - app = FastAPI() - - - @app.post("/files/") - async def create_file(file: Annotated[bytes, File()]): - return {"file_size": len(file)} - - - @app.post("/uploadfile/") - async def create_upload_file(file: UploadFile): - return {"filename": file.filename} - ``` - """ - - file: Annotated[ - BinaryIO, - Doc("The standard Python file object (non-async)."), - ] - filename: Annotated[str | None, Doc("The original file name.")] - size: Annotated[int | None, Doc("The size of the file in bytes.")] - headers: Annotated[Headers, Doc("The headers of the request.")] - content_type: Annotated[ - str | None, Doc("The content type of the request, from the headers.") - ] - - async def write( - self, - data: Annotated[ - bytes, - Doc( - """ - The bytes to write to the file. - """ - ), - ], - ) -> None: - """ - Write some bytes to the file. - - You normally wouldn't use this from a file you read in a request. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().write(data) - - async def read( - self, - size: Annotated[ - int, - Doc( - """ - The number of bytes to read from the file. - """ - ), - ] = -1, - ) -> bytes: - """ - Read some bytes from the file. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().read(size) - - async def seek( - self, - offset: Annotated[ - int, - Doc( - """ - The position in bytes to seek to in the file. - """ - ), - ], - ) -> None: - """ - Move to a position in the file. - - Any next read or write will be done from that position. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().seek(offset) - - async def close(self) -> None: - """ - Close the file. - - To be awaitable, compatible with async, this is run in threadpool. - """ - return await super().close() - - @classmethod - def _validate(cls, __input_value: Any, _: Any) -> "UploadFile": - if not isinstance(__input_value, StarletteUploadFile): - raise ValueError(f"Expected UploadFile, received: {type(__input_value)}") - return cast(UploadFile, __input_value) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: Mapping[str, Any], handler: GetJsonSchemaHandler - ) -> dict[str, Any]: - return {"type": "string", "contentMediaType": "application/octet-stream"} - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: Callable[[Any], Mapping[str, Any]] - ) -> Mapping[str, Any]: - from ._compat.v2 import with_info_plain_validator_function - - return with_info_plain_validator_function(cls._validate) - - -class DefaultPlaceholder: - """ - You shouldn't use this class directly. - - It's used internally to recognize when a default value has been overwritten, even - if the overridden default value was truthy. - """ - - def __init__(self, value: Any): - self.value = value - - def __bool__(self) -> bool: - return bool(self.value) - - def __eq__(self, o: object) -> bool: - return isinstance(o, DefaultPlaceholder) and o.value == self.value - - -DefaultType = TypeVar("DefaultType") - - -def Default(value: DefaultType) -> DefaultType: - """ - You shouldn't use this function directly. - - It's used internally to recognize when a default value has been overwritten, even - if the overridden default value was truthy. - """ - return DefaultPlaceholder(value) # type: ignore - - -# Sentinel for "parameter not provided" in Param/FieldInfo. -# Typed as None to satisfy ty -_Unset = Default(None) diff --git a/functions/python/data-processor/fastapi/dependencies/__init__.py b/functions/python/data-processor/fastapi/dependencies/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/fastapi/dependencies/models.py b/functions/python/data-processor/fastapi/dependencies/models.py deleted file mode 100644 index 25ffb0d2d..000000000 --- a/functions/python/data-processor/fastapi/dependencies/models.py +++ /dev/null @@ -1,193 +0,0 @@ -import inspect -import sys -from collections.abc import Callable -from dataclasses import dataclass, field -from functools import cached_property, partial -from typing import Any, Literal - -from fastapi._compat import ModelField -from fastapi.security.base import SecurityBase -from fastapi.types import DependencyCacheKey - -if sys.version_info >= (3, 13): # pragma: no cover - from inspect import iscoroutinefunction -else: # pragma: no cover - from asyncio import iscoroutinefunction - - -def _unwrapped_call(call: Callable[..., Any] | None) -> Any: - if call is None: - return call # pragma: no cover - unwrapped = inspect.unwrap(_impartial(call)) - return unwrapped - - -def _impartial(func: Callable[..., Any]) -> Callable[..., Any]: - while isinstance(func, partial): - func = func.func - return func - - -@dataclass -class Dependant: - path_params: list[ModelField] = field(default_factory=list) - query_params: list[ModelField] = field(default_factory=list) - header_params: list[ModelField] = field(default_factory=list) - cookie_params: list[ModelField] = field(default_factory=list) - body_params: list[ModelField] = field(default_factory=list) - dependencies: list["Dependant"] = field(default_factory=list) - name: str | None = None - call: Callable[..., Any] | None = None - request_param_name: str | None = None - websocket_param_name: str | None = None - http_connection_param_name: str | None = None - response_param_name: str | None = None - background_tasks_param_name: str | None = None - security_scopes_param_name: str | None = None - own_oauth_scopes: list[str] | None = None - parent_oauth_scopes: list[str] | None = None - use_cache: bool = True - path: str | None = None - scope: Literal["function", "request"] | None = None - - @cached_property - def oauth_scopes(self) -> list[str]: - scopes = self.parent_oauth_scopes.copy() if self.parent_oauth_scopes else [] - # This doesn't use a set to preserve order, just in case - for scope in self.own_oauth_scopes or []: - if scope not in scopes: - scopes.append(scope) - return scopes - - @cached_property - def cache_key(self) -> DependencyCacheKey: - scopes_for_cache = ( - tuple(sorted(set(self.oauth_scopes or []))) if self._uses_scopes else () - ) - return ( - self.call, - scopes_for_cache, - self.computed_scope or "", - ) - - @cached_property - def _uses_scopes(self) -> bool: - if self.own_oauth_scopes: - return True - if self.security_scopes_param_name is not None: - return True - if self._is_security_scheme: - return True - for sub_dep in self.dependencies: - if sub_dep._uses_scopes: - return True - return False - - @cached_property - def _is_security_scheme(self) -> bool: - if self.call is None: - return False # pragma: no cover - unwrapped = _unwrapped_call(self.call) - return isinstance(unwrapped, SecurityBase) - - # Mainly to get the type of SecurityBase, but it's the same self.call - @cached_property - def _security_scheme(self) -> SecurityBase: - unwrapped = _unwrapped_call(self.call) - assert isinstance(unwrapped, SecurityBase) - return unwrapped - - @cached_property - def _security_dependencies(self) -> list["Dependant"]: - security_deps = [dep for dep in self.dependencies if dep._is_security_scheme] - return security_deps - - @cached_property - def is_gen_callable(self) -> bool: - if self.call is None: - return False # pragma: no cover - if inspect.isgeneratorfunction( - _impartial(self.call) - ) or inspect.isgeneratorfunction(_unwrapped_call(self.call)): - return True - if inspect.isclass(_unwrapped_call(self.call)): - return False - dunder_call = getattr(_impartial(self.call), "__call__", None) # noqa: B004 - if dunder_call is None: - return False # pragma: no cover - if inspect.isgeneratorfunction( - _impartial(dunder_call) - ) or inspect.isgeneratorfunction(_unwrapped_call(dunder_call)): - return True - dunder_unwrapped_call = getattr(_unwrapped_call(self.call), "__call__", None) # noqa: B004 - if dunder_unwrapped_call is None: - return False # pragma: no cover - if inspect.isgeneratorfunction( - _impartial(dunder_unwrapped_call) - ) or inspect.isgeneratorfunction(_unwrapped_call(dunder_unwrapped_call)): - return True - return False - - @cached_property - def is_async_gen_callable(self) -> bool: - if self.call is None: - return False # pragma: no cover - if inspect.isasyncgenfunction( - _impartial(self.call) - ) or inspect.isasyncgenfunction(_unwrapped_call(self.call)): - return True - if inspect.isclass(_unwrapped_call(self.call)): - return False - dunder_call = getattr(_impartial(self.call), "__call__", None) # noqa: B004 - if dunder_call is None: - return False # pragma: no cover - if inspect.isasyncgenfunction( - _impartial(dunder_call) - ) or inspect.isasyncgenfunction(_unwrapped_call(dunder_call)): - return True - dunder_unwrapped_call = getattr(_unwrapped_call(self.call), "__call__", None) # noqa: B004 - if dunder_unwrapped_call is None: - return False # pragma: no cover - if inspect.isasyncgenfunction( - _impartial(dunder_unwrapped_call) - ) or inspect.isasyncgenfunction(_unwrapped_call(dunder_unwrapped_call)): - return True - return False - - @cached_property - def is_coroutine_callable(self) -> bool: - if self.call is None: - return False # pragma: no cover - if inspect.isroutine(_impartial(self.call)) and iscoroutinefunction( - _impartial(self.call) - ): - return True - if inspect.isroutine(_unwrapped_call(self.call)) and iscoroutinefunction( - _unwrapped_call(self.call) - ): - return True - if inspect.isclass(_unwrapped_call(self.call)): - return False - dunder_call = getattr(_impartial(self.call), "__call__", None) # noqa: B004 - if dunder_call is None: - return False # pragma: no cover - if iscoroutinefunction(_impartial(dunder_call)) or iscoroutinefunction( - _unwrapped_call(dunder_call) - ): - return True - dunder_unwrapped_call = getattr(_unwrapped_call(self.call), "__call__", None) # noqa: B004 - if dunder_unwrapped_call is None: - return False # pragma: no cover - if iscoroutinefunction( - _impartial(dunder_unwrapped_call) - ) or iscoroutinefunction(_unwrapped_call(dunder_unwrapped_call)): - return True - return False - - @cached_property - def computed_scope(self) -> str | None: - if self.scope: - return self.scope - if self.is_gen_callable or self.is_async_gen_callable: - return "request" - return None diff --git a/functions/python/data-processor/fastapi/dependencies/utils.py b/functions/python/data-processor/fastapi/dependencies/utils.py deleted file mode 100644 index 40dffba64..000000000 --- a/functions/python/data-processor/fastapi/dependencies/utils.py +++ /dev/null @@ -1,1061 +0,0 @@ -import dataclasses -import inspect -import sys -from collections.abc import ( - AsyncGenerator, - AsyncIterable, - AsyncIterator, - Callable, - Generator, - Iterable, - Iterator, - Mapping, - Sequence, -) -from contextlib import AsyncExitStack, contextmanager -from copy import copy, deepcopy -from dataclasses import dataclass -from typing import ( - Annotated, - Any, - ForwardRef, - Literal, - Union, - cast, - get_args, - get_origin, -) - -from fastapi import params -from fastapi._compat import ( - ModelField, - RequiredParam, - Undefined, - copy_field_info, - create_body_model, - evaluate_forwardref, - field_annotation_is_scalar, - field_annotation_is_scalar_sequence, - field_annotation_is_sequence, - get_cached_model_fields, - get_missing_field_error, - is_bytes_or_nonable_bytes_annotation, - is_bytes_sequence_annotation, - is_scalar_field, - is_uploadfile_or_nonable_uploadfile_annotation, - is_uploadfile_sequence_annotation, - lenient_issubclass, - sequence_types, - serialize_sequence_value, - value_is_sequence, -) -from fastapi.background import BackgroundTasks -from fastapi.concurrency import ( - asynccontextmanager, - contextmanager_in_threadpool, -) -from fastapi.dependencies.models import Dependant -from fastapi.exceptions import DependencyScopeError -from fastapi.logger import logger -from fastapi.security.oauth2 import SecurityScopes -from fastapi.types import DependencyCacheKey -from fastapi.utils import create_model_field, get_path_param_names -from pydantic import BaseModel, Json -from pydantic.fields import FieldInfo -from starlette.background import BackgroundTasks as StarletteBackgroundTasks -from starlette.concurrency import run_in_threadpool -from starlette.datastructures import ( - FormData, - Headers, - ImmutableMultiDict, - QueryParams, - UploadFile, -) -from starlette.requests import HTTPConnection, Request -from starlette.responses import Response -from starlette.websockets import WebSocket -from typing_inspection.typing_objects import is_typealiastype - -multipart_not_installed_error = ( - 'Form data requires "python-multipart" to be installed. \n' - 'You can install "python-multipart" with: \n\n' - "pip install python-multipart\n" -) -multipart_incorrect_install_error = ( - 'Form data requires "python-multipart" to be installed. ' - 'It seems you installed "multipart" instead. \n' - 'You can remove "multipart" with: \n\n' - "pip uninstall multipart\n\n" - 'And then install "python-multipart" with: \n\n' - "pip install python-multipart\n" -) - - -def ensure_multipart_is_installed() -> None: - try: - from python_multipart import __version__ - - # Import an attribute that can be mocked/deleted in testing - assert __version__ > "0.0.12" - except (ImportError, AssertionError): - try: - # __version__ is available in both multiparts, and can be mocked - from multipart import ( # type: ignore[no-redef,import-untyped] - __version__, - ) - - assert __version__ - try: - # parse_options_header is only available in the right multipart - from multipart.multipart import ( # type: ignore[import-untyped] - parse_options_header, - ) - - assert parse_options_header - except ImportError: - logger.error(multipart_incorrect_install_error) - raise RuntimeError(multipart_incorrect_install_error) from None - except ImportError: - logger.error(multipart_not_installed_error) - raise RuntimeError(multipart_not_installed_error) from None - - -def get_parameterless_sub_dependant(*, depends: params.Depends, path: str) -> Dependant: - assert callable(depends.dependency), ( - "A parameter-less dependency must have a callable dependency" - ) - own_oauth_scopes: list[str] = [] - if isinstance(depends, params.Security) and depends.scopes: - own_oauth_scopes.extend(depends.scopes) - return get_dependant( - path=path, - call=depends.dependency, - scope=depends.scope, - own_oauth_scopes=own_oauth_scopes, - ) - - -def get_flat_dependant( - dependant: Dependant, - *, - skip_repeats: bool = False, - visited: list[DependencyCacheKey] | None = None, - parent_oauth_scopes: list[str] | None = None, -) -> Dependant: - if visited is None: - visited = [] - visited.append(dependant.cache_key) - use_parent_oauth_scopes = (parent_oauth_scopes or []) + ( - dependant.oauth_scopes or [] - ) - - flat_dependant = Dependant( - path_params=dependant.path_params.copy(), - query_params=dependant.query_params.copy(), - header_params=dependant.header_params.copy(), - cookie_params=dependant.cookie_params.copy(), - body_params=dependant.body_params.copy(), - name=dependant.name, - call=dependant.call, - request_param_name=dependant.request_param_name, - websocket_param_name=dependant.websocket_param_name, - http_connection_param_name=dependant.http_connection_param_name, - response_param_name=dependant.response_param_name, - background_tasks_param_name=dependant.background_tasks_param_name, - security_scopes_param_name=dependant.security_scopes_param_name, - own_oauth_scopes=dependant.own_oauth_scopes, - parent_oauth_scopes=use_parent_oauth_scopes, - use_cache=dependant.use_cache, - path=dependant.path, - scope=dependant.scope, - ) - for sub_dependant in dependant.dependencies: - if skip_repeats and sub_dependant.cache_key in visited: - continue - flat_sub = get_flat_dependant( - sub_dependant, - skip_repeats=skip_repeats, - visited=visited, - parent_oauth_scopes=flat_dependant.oauth_scopes, - ) - flat_dependant.dependencies.append(flat_sub) - flat_dependant.path_params.extend(flat_sub.path_params) - flat_dependant.query_params.extend(flat_sub.query_params) - flat_dependant.header_params.extend(flat_sub.header_params) - flat_dependant.cookie_params.extend(flat_sub.cookie_params) - flat_dependant.body_params.extend(flat_sub.body_params) - flat_dependant.dependencies.extend(flat_sub.dependencies) - - return flat_dependant - - -def _get_flat_fields_from_params(fields: list[ModelField]) -> list[ModelField]: - if not fields: - return fields - first_field = fields[0] - if len(fields) == 1 and lenient_issubclass( - first_field.field_info.annotation, BaseModel - ): - fields_to_extract = get_cached_model_fields(first_field.field_info.annotation) - return fields_to_extract - return fields - - -def get_flat_params(dependant: Dependant) -> list[ModelField]: - flat_dependant = get_flat_dependant(dependant, skip_repeats=True) - path_params = _get_flat_fields_from_params(flat_dependant.path_params) - query_params = _get_flat_fields_from_params(flat_dependant.query_params) - header_params = _get_flat_fields_from_params(flat_dependant.header_params) - cookie_params = _get_flat_fields_from_params(flat_dependant.cookie_params) - return path_params + query_params + header_params + cookie_params - - -def _get_signature(call: Callable[..., Any]) -> inspect.Signature: - try: - signature = inspect.signature(call, eval_str=True) - except NameError: - # Handle type annotations with if TYPE_CHECKING, not used by FastAPI - # e.g. dependency return types - if sys.version_info >= (3, 14): - from annotationlib import Format - - signature = inspect.signature(call, annotation_format=Format.FORWARDREF) - else: - signature = inspect.signature(call) - return signature - - -def get_typed_signature(call: Callable[..., Any]) -> inspect.Signature: - signature = _get_signature(call) - unwrapped = inspect.unwrap(call) - globalns = getattr(unwrapped, "__globals__", {}) - typed_params = [ - inspect.Parameter( - name=param.name, - kind=param.kind, - default=param.default, - annotation=get_typed_annotation(param.annotation, globalns), - ) - for param in signature.parameters.values() - ] - typed_signature = inspect.Signature(typed_params) - return typed_signature - - -def get_typed_annotation(annotation: Any, globalns: dict[str, Any]) -> Any: - if isinstance(annotation, str): - annotation = ForwardRef(annotation) - annotation = evaluate_forwardref(annotation, globalns, globalns) - if annotation is type(None): - return None - return annotation - - -def get_typed_return_annotation(call: Callable[..., Any]) -> Any: - signature = _get_signature(call) - unwrapped = inspect.unwrap(call) - annotation = signature.return_annotation - - if annotation is inspect.Signature.empty: - return None - - globalns = getattr(unwrapped, "__globals__", {}) - return get_typed_annotation(annotation, globalns) - - -_STREAM_ORIGINS = { - AsyncIterable, - AsyncIterator, - AsyncGenerator, - Iterable, - Iterator, - Generator, -} - - -def get_stream_item_type(annotation: Any) -> Any | None: - origin = get_origin(annotation) - if origin is not None and origin in _STREAM_ORIGINS: - type_args = get_args(annotation) - if type_args: - return type_args[0] - return Any - return None - - -def get_dependant( - *, - path: str, - call: Callable[..., Any], - name: str | None = None, - own_oauth_scopes: list[str] | None = None, - parent_oauth_scopes: list[str] | None = None, - use_cache: bool = True, - scope: Literal["function", "request"] | None = None, -) -> Dependant: - dependant = Dependant( - call=call, - name=name, - path=path, - use_cache=use_cache, - scope=scope, - own_oauth_scopes=own_oauth_scopes, - parent_oauth_scopes=parent_oauth_scopes, - ) - current_scopes = (parent_oauth_scopes or []) + (own_oauth_scopes or []) - path_param_names = get_path_param_names(path) - endpoint_signature = get_typed_signature(call) - signature_params = endpoint_signature.parameters - for param_name, param in signature_params.items(): - is_path_param = param_name in path_param_names - param_details = analyze_param( - param_name=param_name, - annotation=param.annotation, - value=param.default, - is_path_param=is_path_param, - ) - if param_details.depends is not None: - assert param_details.depends.dependency - if ( - (dependant.is_gen_callable or dependant.is_async_gen_callable) - and dependant.computed_scope == "request" - and param_details.depends.scope == "function" - ): - assert dependant.call - call_name = getattr(dependant.call, "__name__", "") - raise DependencyScopeError( - f'The dependency "{call_name}" has a scope of ' - '"request", it cannot depend on dependencies with scope "function".' - ) - sub_own_oauth_scopes: list[str] = [] - if isinstance(param_details.depends, params.Security): - if param_details.depends.scopes: - sub_own_oauth_scopes = list(param_details.depends.scopes) - sub_dependant = get_dependant( - path=path, - call=param_details.depends.dependency, - name=param_name, - own_oauth_scopes=sub_own_oauth_scopes, - parent_oauth_scopes=current_scopes, - use_cache=param_details.depends.use_cache, - scope=param_details.depends.scope, - ) - dependant.dependencies.append(sub_dependant) - continue - if add_non_field_param_to_dependency( - param_name=param_name, - type_annotation=param_details.type_annotation, - dependant=dependant, - ): - assert param_details.field is None, ( - f"Cannot specify multiple FastAPI annotations for {param_name!r}" - ) - continue - assert param_details.field is not None - if isinstance(param_details.field.field_info, params.Body): - dependant.body_params.append(param_details.field) - else: - add_param_to_fields(field=param_details.field, dependant=dependant) - return dependant - - -def add_non_field_param_to_dependency( - *, param_name: str, type_annotation: Any, dependant: Dependant -) -> bool | None: - if lenient_issubclass(type_annotation, Request): - dependant.request_param_name = param_name - return True - elif lenient_issubclass(type_annotation, WebSocket): - dependant.websocket_param_name = param_name - return True - elif lenient_issubclass(type_annotation, HTTPConnection): - dependant.http_connection_param_name = param_name - return True - elif lenient_issubclass(type_annotation, Response): - dependant.response_param_name = param_name - return True - elif lenient_issubclass(type_annotation, StarletteBackgroundTasks): - dependant.background_tasks_param_name = param_name - return True - elif lenient_issubclass(type_annotation, SecurityScopes): - dependant.security_scopes_param_name = param_name - return True - return None - - -@dataclass -class ParamDetails: - type_annotation: Any - depends: params.Depends | None - field: ModelField | None - - -def analyze_param( - *, - param_name: str, - annotation: Any, - value: Any, - is_path_param: bool, -) -> ParamDetails: - field_info = None - depends = None - type_annotation: Any = Any - use_annotation: Any = Any - if is_typealiastype(annotation): - # unpack in case PEP 695 type syntax is used - annotation = annotation.__value__ - if annotation is not inspect.Signature.empty: - use_annotation = annotation - type_annotation = annotation - # Extract Annotated info - if get_origin(use_annotation) is Annotated: - annotated_args = get_args(annotation) - type_annotation = annotated_args[0] - fastapi_annotations = [ - arg - for arg in annotated_args[1:] - if isinstance(arg, (FieldInfo, params.Depends)) - ] - fastapi_specific_annotations = [ - arg - for arg in fastapi_annotations - if isinstance( - arg, - ( - params.Param, - params.Body, - params.Depends, - ), - ) - ] - if fastapi_specific_annotations: - fastapi_annotation: FieldInfo | params.Depends | None = ( - fastapi_specific_annotations[-1] - ) - else: - fastapi_annotation = None - # Set default for Annotated FieldInfo - if isinstance(fastapi_annotation, FieldInfo): - # Copy `field_info` because we mutate `field_info.default` below. - field_info = copy_field_info( - field_info=fastapi_annotation, - annotation=use_annotation, - ) - assert ( - field_info.default == Undefined or field_info.default == RequiredParam - ), ( - f"`{field_info.__class__.__name__}` default value cannot be set in" - f" `Annotated` for {param_name!r}. Set the default value with `=` instead." - ) - if value is not inspect.Signature.empty: - assert not is_path_param, "Path parameters cannot have default values" - field_info.default = value - else: - field_info.default = RequiredParam - # Get Annotated Depends - elif isinstance(fastapi_annotation, params.Depends): - depends = fastapi_annotation - # Get Depends from default value - if isinstance(value, params.Depends): - assert depends is None, ( - "Cannot specify `Depends` in `Annotated` and default value" - f" together for {param_name!r}" - ) - assert field_info is None, ( - "Cannot specify a FastAPI annotation in `Annotated` and `Depends` as a" - f" default value together for {param_name!r}" - ) - depends = value - # Get FieldInfo from default value - elif isinstance(value, FieldInfo): - assert field_info is None, ( - "Cannot specify FastAPI annotations in `Annotated` and default value" - f" together for {param_name!r}" - ) - field_info = value - if isinstance(field_info, FieldInfo): - field_info.annotation = type_annotation - - # Get Depends from type annotation - if depends is not None and depends.dependency is None: - # Copy `depends` before mutating it - depends = copy(depends) - depends = dataclasses.replace(depends, dependency=type_annotation) - - # Handle non-param type annotations like Request - # Only apply special handling when there's no explicit Depends - if there's a Depends, - # the dependency will be called and its return value used instead of the special injection - if depends is None and lenient_issubclass( - type_annotation, - ( - Request, - WebSocket, - HTTPConnection, - Response, - StarletteBackgroundTasks, - SecurityScopes, - ), - ): - assert field_info is None, ( - f"Cannot specify FastAPI annotation for type {type_annotation!r}" - ) - # Handle default assignations, neither field_info nor depends was not found in Annotated nor default value - elif field_info is None and depends is None: - default_value = value if value is not inspect.Signature.empty else RequiredParam - if is_path_param: - # We might check here that `default_value is RequiredParam`, but the fact is that the same - # parameter might sometimes be a path parameter and sometimes not. See - # `tests/test_infer_param_optionality.py` for an example. - field_info = params.Path(annotation=use_annotation) - elif is_uploadfile_or_nonable_uploadfile_annotation( - type_annotation - ) or is_uploadfile_sequence_annotation(type_annotation): - field_info = params.File(annotation=use_annotation, default=default_value) - elif not field_annotation_is_scalar(annotation=type_annotation): - field_info = params.Body(annotation=use_annotation, default=default_value) - else: - field_info = params.Query(annotation=use_annotation, default=default_value) - - field = None - # It's a field_info, not a dependency - if field_info is not None: - # Handle field_info.in_ - if is_path_param: - assert isinstance(field_info, params.Path), ( - f"Cannot use `{field_info.__class__.__name__}` for path param" - f" {param_name!r}" - ) - elif ( - isinstance(field_info, params.Param) - and getattr(field_info, "in_", None) is None - ): - field_info.in_ = params.ParamTypes.query - use_annotation_from_field_info = use_annotation - if isinstance(field_info, params.Form): - ensure_multipart_is_installed() - if not field_info.alias and getattr(field_info, "convert_underscores", None): - alias = param_name.replace("_", "-") - else: - alias = field_info.alias or param_name - field_info.alias = alias - field = create_model_field( - name=param_name, - type_=use_annotation_from_field_info, - default=field_info.default, - alias=alias, - field_info=field_info, - ) - if is_path_param: - assert is_scalar_field(field=field), ( - "Path params must be of one of the supported types" - ) - elif isinstance(field_info, params.Query): - assert ( - is_scalar_field(field) - or field_annotation_is_scalar_sequence(field.field_info.annotation) - or lenient_issubclass(field.field_info.annotation, BaseModel) - ), f"Query parameter {param_name!r} must be one of the supported types" - - return ParamDetails(type_annotation=type_annotation, depends=depends, field=field) - - -def add_param_to_fields(*, field: ModelField, dependant: Dependant) -> None: - field_info = field.field_info - field_info_in = getattr(field_info, "in_", None) - if field_info_in == params.ParamTypes.path: - dependant.path_params.append(field) - elif field_info_in == params.ParamTypes.query: - dependant.query_params.append(field) - elif field_info_in == params.ParamTypes.header: - dependant.header_params.append(field) - else: - assert field_info_in == params.ParamTypes.cookie, ( - f"non-body parameters must be in path, query, header or cookie: {field.name}" - ) - dependant.cookie_params.append(field) - - -async def _solve_generator( - *, dependant: Dependant, stack: AsyncExitStack, sub_values: dict[str, Any] -) -> Any: - assert dependant.call - if dependant.is_async_gen_callable: - cm = asynccontextmanager(dependant.call)(**sub_values) - elif dependant.is_gen_callable: - cm = contextmanager_in_threadpool(contextmanager(dependant.call)(**sub_values)) - return await stack.enter_async_context(cm) - - -@dataclass -class SolvedDependency: - values: dict[str, Any] - errors: list[Any] - background_tasks: StarletteBackgroundTasks | None - response: Response - dependency_cache: dict[DependencyCacheKey, Any] - - -async def solve_dependencies( - *, - request: Request | WebSocket, - dependant: Dependant, - body: dict[str, Any] | FormData | bytes | None = None, - background_tasks: StarletteBackgroundTasks | None = None, - response: Response | None = None, - dependency_overrides_provider: Any | None = None, - dependency_cache: dict[DependencyCacheKey, Any] | None = None, - # TODO: remove this parameter later, no longer used, not removing it yet as some - # people might be monkey patching this function (although that's not supported) - async_exit_stack: AsyncExitStack, - embed_body_fields: bool, -) -> SolvedDependency: - request_astack = request.scope.get("fastapi_inner_astack") - assert isinstance(request_astack, AsyncExitStack), ( - "fastapi_inner_astack not found in request scope" - ) - function_astack = request.scope.get("fastapi_function_astack") - assert isinstance(function_astack, AsyncExitStack), ( - "fastapi_function_astack not found in request scope" - ) - values: dict[str, Any] = {} - errors: list[Any] = [] - if response is None: - response = Response() - del response.headers["content-length"] - response.status_code = None # type: ignore - if dependency_cache is None: - dependency_cache = {} - for sub_dependant in dependant.dependencies: - sub_dependant.call = cast(Callable[..., Any], sub_dependant.call) - call = sub_dependant.call - use_sub_dependant = sub_dependant - if ( - dependency_overrides_provider - and dependency_overrides_provider.dependency_overrides - ): - original_call = sub_dependant.call - call = getattr( - dependency_overrides_provider, "dependency_overrides", {} - ).get(original_call, original_call) - use_path: str = sub_dependant.path # type: ignore - use_sub_dependant = get_dependant( - path=use_path, - call=call, - name=sub_dependant.name, - parent_oauth_scopes=sub_dependant.oauth_scopes, - scope=sub_dependant.scope, - ) - - solved_result = await solve_dependencies( - request=request, - dependant=use_sub_dependant, - body=body, - background_tasks=background_tasks, - response=response, - dependency_overrides_provider=dependency_overrides_provider, - dependency_cache=dependency_cache, - async_exit_stack=async_exit_stack, - embed_body_fields=embed_body_fields, - ) - background_tasks = solved_result.background_tasks - if solved_result.errors: - errors.extend(solved_result.errors) - continue - if sub_dependant.use_cache and sub_dependant.cache_key in dependency_cache: - solved = dependency_cache[sub_dependant.cache_key] - elif ( - use_sub_dependant.is_gen_callable or use_sub_dependant.is_async_gen_callable - ): - use_astack = request_astack - if sub_dependant.scope == "function": - use_astack = function_astack - solved = await _solve_generator( - dependant=use_sub_dependant, - stack=use_astack, - sub_values=solved_result.values, - ) - elif use_sub_dependant.is_coroutine_callable: - solved = await call(**solved_result.values) - else: - solved = await run_in_threadpool(call, **solved_result.values) - if sub_dependant.name is not None: - values[sub_dependant.name] = solved - if sub_dependant.cache_key not in dependency_cache: - dependency_cache[sub_dependant.cache_key] = solved - path_values, path_errors = request_params_to_args( - dependant.path_params, request.path_params - ) - query_values, query_errors = request_params_to_args( - dependant.query_params, request.query_params - ) - header_values, header_errors = request_params_to_args( - dependant.header_params, request.headers - ) - cookie_values, cookie_errors = request_params_to_args( - dependant.cookie_params, request.cookies - ) - values.update(path_values) - values.update(query_values) - values.update(header_values) - values.update(cookie_values) - errors += path_errors + query_errors + header_errors + cookie_errors - if dependant.body_params: - ( - body_values, - body_errors, - ) = await request_body_to_args( # body_params checked above - body_fields=dependant.body_params, - received_body=body, - embed_body_fields=embed_body_fields, - ) - values.update(body_values) - errors.extend(body_errors) - if dependant.http_connection_param_name: - values[dependant.http_connection_param_name] = request - if dependant.request_param_name and isinstance(request, Request): - values[dependant.request_param_name] = request - elif dependant.websocket_param_name and isinstance(request, WebSocket): - values[dependant.websocket_param_name] = request - if dependant.background_tasks_param_name: - if background_tasks is None: - background_tasks = BackgroundTasks() - values[dependant.background_tasks_param_name] = background_tasks - if dependant.response_param_name: - values[dependant.response_param_name] = response - if dependant.security_scopes_param_name: - values[dependant.security_scopes_param_name] = SecurityScopes( - scopes=dependant.oauth_scopes - ) - return SolvedDependency( - values=values, - errors=errors, - background_tasks=background_tasks, - response=response, - dependency_cache=dependency_cache, - ) - - -def _validate_value_with_model_field( - *, field: ModelField, value: Any, values: dict[str, Any], loc: tuple[str, ...] -) -> tuple[Any, list[Any]]: - if value is None: - if field.field_info.is_required(): - return None, [get_missing_field_error(loc=loc)] - else: - return deepcopy(field.default), [] - return field.validate(value, values, loc=loc) - - -def _is_json_field(field: ModelField) -> bool: - return any(type(item) is Json for item in field.field_info.metadata) - - -def _get_multidict_value( - field: ModelField, values: Mapping[str, Any], alias: str | None = None -) -> Any: - alias = alias or get_validation_alias(field) - if ( - (not _is_json_field(field)) - and field_annotation_is_sequence(field.field_info.annotation) - and isinstance(values, (ImmutableMultiDict, Headers)) - ): - value = values.getlist(alias) - else: - value = values.get(alias, None) - if ( - value is None - or ( - isinstance(field.field_info, params.Form) - and isinstance(value, str) # For type checks - and value == "" - ) - or ( - field_annotation_is_sequence(field.field_info.annotation) - and len(value) == 0 - ) - ): - if field.field_info.is_required(): - return - else: - return deepcopy(field.default) - return value - - -def request_params_to_args( - fields: Sequence[ModelField], - received_params: Mapping[str, Any] | QueryParams | Headers, -) -> tuple[dict[str, Any], list[Any]]: - values: dict[str, Any] = {} - errors: list[dict[str, Any]] = [] - - if not fields: - return values, errors - - first_field = fields[0] - fields_to_extract = fields - single_not_embedded_field = False - default_convert_underscores = True - if len(fields) == 1 and lenient_issubclass( - first_field.field_info.annotation, BaseModel - ): - fields_to_extract = get_cached_model_fields(first_field.field_info.annotation) - single_not_embedded_field = True - # If headers are in a Pydantic model, the way to disable convert_underscores - # would be with Header(convert_underscores=False) at the Pydantic model level - default_convert_underscores = getattr( - first_field.field_info, "convert_underscores", True - ) - - params_to_process: dict[str, Any] = {} - - processed_keys = set() - - for field in fields_to_extract: - alias = None - if isinstance(received_params, Headers): - # Handle fields extracted from a Pydantic Model for a header, each field - # doesn't have a FieldInfo of type Header with the default convert_underscores=True - convert_underscores = getattr( - field.field_info, "convert_underscores", default_convert_underscores - ) - if convert_underscores: - alias = get_validation_alias(field) - if alias == field.name: - alias = alias.replace("_", "-") - value = _get_multidict_value(field, received_params, alias=alias) - if value is not None: - params_to_process[get_validation_alias(field)] = value - processed_keys.add(alias or get_validation_alias(field)) - # For headers with convert_underscores=True, mark both the converted - # header name and the original field alias as processed to avoid - # accepting the original alias as an extra header. - processed_keys.add(get_validation_alias(field)) - - for key in received_params.keys(): - if key not in processed_keys: - if isinstance(received_params, (ImmutableMultiDict, Headers)): - value = received_params.getlist(key) - if isinstance(value, list) and (len(value) == 1): - params_to_process[key] = value[0] - else: - params_to_process[key] = value - else: - params_to_process[key] = received_params.get(key) - - if single_not_embedded_field: - field_info = first_field.field_info - assert isinstance(field_info, params.Param), ( - "Params must be subclasses of Param" - ) - loc: tuple[str, ...] = (field_info.in_.value,) - v_, errors_ = _validate_value_with_model_field( - field=first_field, value=params_to_process, values=values, loc=loc - ) - return {first_field.name: v_}, errors_ - - for field in fields: - value = _get_multidict_value(field, received_params) - field_info = field.field_info - assert isinstance(field_info, params.Param), ( - "Params must be subclasses of Param" - ) - loc = (field_info.in_.value, get_validation_alias(field)) - v_, errors_ = _validate_value_with_model_field( - field=field, value=value, values=values, loc=loc - ) - if errors_: - errors.extend(errors_) - else: - values[field.name] = v_ - return values, errors - - -def is_union_of_base_models(field_type: Any) -> bool: - """Check if field type is a Union where all members are BaseModel subclasses.""" - from fastapi.types import UnionType - - origin = get_origin(field_type) - - # Check if it's a Union type (covers both typing.Union and types.UnionType in Python 3.10+) - if origin is not Union and origin is not UnionType: - return False - - union_args = get_args(field_type) - - for arg in union_args: - if not lenient_issubclass(arg, BaseModel): - return False - - return True - - -def _should_embed_body_fields(fields: list[ModelField]) -> bool: - if not fields: - return False - # More than one dependency could have the same field, it would show up as multiple - # fields but it's the same one, so count them by name - body_param_names_set = {field.name for field in fields} - # A top level field has to be a single field, not multiple - if len(body_param_names_set) > 1: - return True - first_field = fields[0] - # If it explicitly specifies it is embedded, it has to be embedded - if getattr(first_field.field_info, "embed", None): - return True - # If it's a Form (or File) field, it has to be a BaseModel (or a union of BaseModels) to be top level - # otherwise it has to be embedded, so that the key value pair can be extracted - if ( - isinstance(first_field.field_info, params.Form) - and not lenient_issubclass(first_field.field_info.annotation, BaseModel) - and not is_union_of_base_models(first_field.field_info.annotation) - ): - return True - return False - - -async def _extract_form_body( - body_fields: list[ModelField], - received_body: FormData, -) -> dict[str, Any]: - values = {} - - for field in body_fields: - value = _get_multidict_value(field, received_body) - field_info = field.field_info - if ( - isinstance(field_info, params.File) - and is_bytes_or_nonable_bytes_annotation(field.field_info.annotation) - and isinstance(value, UploadFile) - ): - value = await value.read() - elif ( - is_bytes_sequence_annotation(field.field_info.annotation) - and isinstance(field_info, params.File) - and value_is_sequence(value) - ): - # For types - assert isinstance(value, sequence_types) - results: list[bytes | str] = [] - for sub_value in value: - results.append(await sub_value.read()) - value = serialize_sequence_value(field=field, value=results) - if value is not None: - values[get_validation_alias(field)] = value - field_aliases = {get_validation_alias(field) for field in body_fields} - for key in received_body.keys(): - if key not in field_aliases: - param_values = received_body.getlist(key) - if len(param_values) == 1: - values[key] = param_values[0] - else: - values[key] = param_values - return values - - -async def request_body_to_args( - body_fields: list[ModelField], - received_body: dict[str, Any] | FormData | bytes | None, - embed_body_fields: bool, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - values: dict[str, Any] = {} - errors: list[dict[str, Any]] = [] - assert body_fields, "request_body_to_args() should be called with fields" - single_not_embedded_field = len(body_fields) == 1 and not embed_body_fields - first_field = body_fields[0] - body_to_process = received_body - - fields_to_extract: list[ModelField] = body_fields - - if ( - single_not_embedded_field - and lenient_issubclass(first_field.field_info.annotation, BaseModel) - and isinstance(received_body, FormData) - ): - fields_to_extract = get_cached_model_fields(first_field.field_info.annotation) - - if isinstance(received_body, FormData): - body_to_process = await _extract_form_body(fields_to_extract, received_body) - - if single_not_embedded_field: - loc: tuple[str, ...] = ("body",) - v_, errors_ = _validate_value_with_model_field( - field=first_field, value=body_to_process, values=values, loc=loc - ) - return {first_field.name: v_}, errors_ - for field in body_fields: - loc = ("body", get_validation_alias(field)) - value: Any | None = None - if body_to_process is not None and not isinstance(body_to_process, bytes): - try: - value = body_to_process.get(get_validation_alias(field)) - # If the received body is a list, not a dict - except AttributeError: - errors.append(get_missing_field_error(loc)) - continue - v_, errors_ = _validate_value_with_model_field( - field=field, value=value, values=values, loc=loc - ) - if errors_: - errors.extend(errors_) - else: - values[field.name] = v_ - return values, errors - - -def get_body_field( - *, flat_dependant: Dependant, name: str, embed_body_fields: bool -) -> ModelField | None: - """ - Get a ModelField representing the request body for a path operation, combining - all body parameters into a single field if necessary. - - Used to check if it's form data (with `isinstance(body_field, params.Form)`) - or JSON and to generate the JSON Schema for a request body. - - This is **not** used to validate/parse the request body, that's done with each - individual body parameter. - """ - if not flat_dependant.body_params: - return None - first_param = flat_dependant.body_params[0] - if not embed_body_fields: - return first_param - model_name = "Body_" + name - BodyModel = create_body_model( - fields=flat_dependant.body_params, model_name=model_name - ) - required = any( - True for f in flat_dependant.body_params if f.field_info.is_required() - ) - BodyFieldInfo_kwargs: dict[str, Any] = { - "annotation": BodyModel, - "alias": "body", - } - if not required: - BodyFieldInfo_kwargs["default"] = None - if any(isinstance(f.field_info, params.File) for f in flat_dependant.body_params): - BodyFieldInfo: type[params.Body] = params.File - elif any(isinstance(f.field_info, params.Form) for f in flat_dependant.body_params): - BodyFieldInfo = params.Form - else: - BodyFieldInfo = params.Body - - body_param_media_types = [ - f.field_info.media_type - for f in flat_dependant.body_params - if isinstance(f.field_info, params.Body) - ] - if len(set(body_param_media_types)) == 1: - BodyFieldInfo_kwargs["media_type"] = body_param_media_types[0] - final_field = create_model_field( - name="body", - type_=BodyModel, - alias="body", - field_info=BodyFieldInfo(**BodyFieldInfo_kwargs), - ) - return final_field - - -def get_validation_alias(field: ModelField) -> str: - va = getattr(field, "validation_alias", None) - return va or field.alias diff --git a/functions/python/data-processor/fastapi/encoders.py b/functions/python/data-processor/fastapi/encoders.py deleted file mode 100644 index c9f882d2b..000000000 --- a/functions/python/data-processor/fastapi/encoders.py +++ /dev/null @@ -1,364 +0,0 @@ -import dataclasses -import datetime -from collections import defaultdict, deque -from collections.abc import Callable -from decimal import Decimal -from enum import Enum -from ipaddress import ( - IPv4Address, - IPv4Interface, - IPv4Network, - IPv6Address, - IPv6Interface, - IPv6Network, -) -from pathlib import Path, PurePath -from re import Pattern -from types import GeneratorType -from typing import Annotated, Any -from uuid import UUID - -from annotated_doc import Doc -from fastapi.exceptions import PydanticV1NotSupportedError -from fastapi.types import IncEx -from pydantic import BaseModel -from pydantic.networks import AnyUrl, NameEmail -from pydantic.types import SecretBytes, SecretStr -from pydantic_core import PydanticUndefinedType - -from ._compat import ( - Url, - is_pydantic_v1_model_instance, -) - -try: - # pydantic.color.Color is deprecated since v2.0b3, but supporting for bwd-compat - from pydantic.color import Color # ty: ignore[deprecated] -except ImportError: # pragma: no cover - - class Color: # type: ignore[no-redef] - pass - - -try: - # Supporting the new Color format for newer versions of Pydantic - from pydantic_extra_types.color import Color as PyExtraColor -except ImportError: # pragma: no cover - - class PyExtraColor: # type: ignore[no-redef] - pass - - -# Taken from Pydantic v1 as is -def isoformat(o: datetime.date | datetime.time) -> str: - return o.isoformat() - - -# Adapted from Pydantic v1 -# TODO: pv2 should this return strings instead? -def decimal_encoder(dec_value: Decimal) -> int | float: - """ - Encodes a Decimal as int if there's no exponent, otherwise float - - This is useful when we use ConstrainedDecimal to represent Numeric(x,0) - where an integer (but not int typed) is used. Encoding this as a float - results in failed round-tripping between encode and parse. - Our Id type is a prime example of this. - - >>> decimal_encoder(Decimal("1.0")) - 1.0 - - >>> decimal_encoder(Decimal("1")) - 1 - - >>> decimal_encoder(Decimal("NaN")) - nan - """ - exponent = dec_value.as_tuple().exponent - if isinstance(exponent, int) and exponent >= 0: - return int(dec_value) - else: - return float(dec_value) - - -ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { - bytes: lambda o: o.decode(), - Color: str, - PyExtraColor: str, - datetime.date: isoformat, - datetime.datetime: isoformat, - datetime.time: isoformat, - datetime.timedelta: lambda td: td.total_seconds(), - Decimal: decimal_encoder, - Enum: lambda o: o.value, - frozenset: list, - deque: list, - GeneratorType: list, - IPv4Address: str, - IPv4Interface: str, - IPv4Network: str, - IPv6Address: str, - IPv6Interface: str, - IPv6Network: str, - NameEmail: str, - Path: str, - Pattern: lambda o: o.pattern, - SecretBytes: str, - SecretStr: str, - set: list, - UUID: str, - Url: str, - AnyUrl: str, -} - - -def generate_encoders_by_class_tuples( - type_encoder_map: dict[Any, Callable[[Any], Any]], -) -> dict[Callable[[Any], Any], tuple[Any, ...]]: - encoders_by_class_tuples: dict[Callable[[Any], Any], tuple[Any, ...]] = defaultdict( - tuple - ) - for type_, encoder in type_encoder_map.items(): - encoders_by_class_tuples[encoder] += (type_,) - return encoders_by_class_tuples - - -encoders_by_class_tuples = generate_encoders_by_class_tuples(ENCODERS_BY_TYPE) - - -def jsonable_encoder( - obj: Annotated[ - Any, - Doc( - """ - The input object to convert to JSON. - """ - ), - ], - include: Annotated[ - IncEx | None, - Doc( - """ - Pydantic's `include` parameter, passed to Pydantic models to set the - fields to include. - """ - ), - ] = None, - exclude: Annotated[ - IncEx | None, - Doc( - """ - Pydantic's `exclude` parameter, passed to Pydantic models to set the - fields to exclude. - """ - ), - ] = None, - by_alias: Annotated[ - bool, - Doc( - """ - Pydantic's `by_alias` parameter, passed to Pydantic models to define if - the output should use the alias names (when provided) or the Python - attribute names. In an API, if you set an alias, it's probably because you - want to use it in the result, so you probably want to leave this set to - `True`. - """ - ), - ] = True, - exclude_unset: Annotated[ - bool, - Doc( - """ - Pydantic's `exclude_unset` parameter, passed to Pydantic models to define - if it should exclude from the output the fields that were not explicitly - set (and that only had their default values). - """ - ), - ] = False, - exclude_defaults: Annotated[ - bool, - Doc( - """ - Pydantic's `exclude_defaults` parameter, passed to Pydantic models to define - if it should exclude from the output the fields that had the same default - value, even when they were explicitly set. - """ - ), - ] = False, - exclude_none: Annotated[ - bool, - Doc( - """ - Pydantic's `exclude_none` parameter, passed to Pydantic models to define - if it should exclude from the output any fields that have a `None` value. - """ - ), - ] = False, - custom_encoder: Annotated[ - dict[Any, Callable[[Any], Any]] | None, - Doc( - """ - Pydantic's `custom_encoder` parameter, passed to Pydantic models to define - a custom encoder. - """ - ), - ] = None, - sqlalchemy_safe: Annotated[ - bool, - Doc( - """ - Exclude from the output any fields that start with the name `_sa`. - - This is mainly a hack for compatibility with SQLAlchemy objects, they - store internal SQLAlchemy-specific state in attributes named with `_sa`, - and those objects can't (and shouldn't be) serialized to JSON. - """ - ), - ] = True, -) -> Any: - """ - Convert any object to something that can be encoded in JSON. - - This is used internally by FastAPI to make sure anything you return can be - encoded as JSON before it is sent to the client. - - You can also use it yourself, for example to convert objects before saving them - in a database that supports only JSON. - - Read more about it in the - [FastAPI docs for JSON Compatible Encoder](https://fastapi.tiangolo.com/tutorial/encoder/). - """ - custom_encoder = custom_encoder or {} - if custom_encoder: - if type(obj) in custom_encoder: - return custom_encoder[type(obj)](obj) - else: - for encoder_type, encoder_instance in custom_encoder.items(): - if isinstance(obj, encoder_type): - return encoder_instance(obj) - if include is not None and not isinstance(include, (set, dict)): - include = set(include) # type: ignore[assignment] # ty: ignore[invalid-assignment] - if exclude is not None and not isinstance(exclude, (set, dict)): - exclude = set(exclude) # type: ignore[assignment] # ty: ignore[invalid-assignment] - if isinstance(obj, BaseModel): - obj_dict = obj.model_dump( - mode="json", - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_none=exclude_none, - exclude_defaults=exclude_defaults, - ) - return jsonable_encoder( - obj_dict, - exclude_none=exclude_none, - exclude_defaults=exclude_defaults, - sqlalchemy_safe=sqlalchemy_safe, - ) - if dataclasses.is_dataclass(obj): - assert not isinstance(obj, type) - obj_dict = dataclasses.asdict(obj) - return jsonable_encoder( - obj_dict, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - if isinstance(obj, Enum): - return obj.value - if isinstance(obj, PurePath): - return str(obj) - if isinstance(obj, (str, int, float, type(None))): - return obj - if isinstance(obj, PydanticUndefinedType): - return None - if isinstance(obj, dict): - encoded_dict = {} - allowed_keys = set(obj.keys()) - if include is not None: - allowed_keys &= set(include) - if exclude is not None: - allowed_keys -= set(exclude) - for key, value in obj.items(): - if ( - ( - not sqlalchemy_safe - or (not isinstance(key, str)) - or (not key.startswith("_sa")) - ) - and (value is not None or not exclude_none) - and key in allowed_keys - ): - encoded_key = jsonable_encoder( - key, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - encoded_value = jsonable_encoder( - value, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - encoded_dict[encoded_key] = encoded_value - return encoded_dict - if isinstance(obj, (list, set, frozenset, GeneratorType, tuple, deque)): - encoded_list = [] - for item in obj: - encoded_list.append( - jsonable_encoder( - item, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) - ) - return encoded_list - - if type(obj) in ENCODERS_BY_TYPE: - return ENCODERS_BY_TYPE[type(obj)](obj) - for encoder, classes_tuple in encoders_by_class_tuples.items(): - if isinstance(obj, classes_tuple): - return encoder(obj) - if is_pydantic_v1_model_instance(obj): - raise PydanticV1NotSupportedError( - "pydantic.v1 models are no longer supported by FastAPI." - f" Please update the model {obj!r}." - ) - try: - data = dict(obj) - except Exception as e: - errors: list[Exception] = [] - errors.append(e) - try: - data = vars(obj) - except Exception as e: - errors.append(e) - raise ValueError(errors) from e - return jsonable_encoder( - data, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - custom_encoder=custom_encoder, - sqlalchemy_safe=sqlalchemy_safe, - ) diff --git a/functions/python/data-processor/fastapi/exception_handlers.py b/functions/python/data-processor/fastapi/exception_handlers.py deleted file mode 100644 index 475dd7bdd..000000000 --- a/functions/python/data-processor/fastapi/exception_handlers.py +++ /dev/null @@ -1,34 +0,0 @@ -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import RequestValidationError, WebSocketRequestValidationError -from fastapi.utils import is_body_allowed_for_status_code -from fastapi.websockets import WebSocket -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.responses import JSONResponse, Response -from starlette.status import WS_1008_POLICY_VIOLATION - - -async def http_exception_handler(request: Request, exc: HTTPException) -> Response: - headers = getattr(exc, "headers", None) - if not is_body_allowed_for_status_code(exc.status_code): - return Response(status_code=exc.status_code, headers=headers) - return JSONResponse( - {"detail": exc.detail}, status_code=exc.status_code, headers=headers - ) - - -async def request_validation_exception_handler( - request: Request, exc: RequestValidationError -) -> JSONResponse: - return JSONResponse( - status_code=422, - content={"detail": jsonable_encoder(exc.errors())}, - ) - - -async def websocket_request_validation_exception_handler( - websocket: WebSocket, exc: WebSocketRequestValidationError -) -> None: - await websocket.close( - code=WS_1008_POLICY_VIOLATION, reason=jsonable_encoder(exc.errors()) - ) diff --git a/functions/python/data-processor/fastapi/exceptions.py b/functions/python/data-processor/fastapi/exceptions.py deleted file mode 100644 index d7065c52f..000000000 --- a/functions/python/data-processor/fastapi/exceptions.py +++ /dev/null @@ -1,256 +0,0 @@ -from collections.abc import Mapping, Sequence -from typing import Annotated, Any, TypedDict - -from annotated_doc import Doc -from pydantic import BaseModel, create_model -from starlette.exceptions import HTTPException as StarletteHTTPException -from starlette.exceptions import WebSocketException as StarletteWebSocketException - - -class EndpointContext(TypedDict, total=False): - function: str - path: str - file: str - line: int - - -class HTTPException(StarletteHTTPException): - """ - An HTTP exception you can raise in your own code to show errors to the client. - - This is for client errors, invalid authentication, invalid data, etc. Not for server - errors in your code. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/). - - ## Example - - ```python - from fastapi import FastAPI, HTTPException - - app = FastAPI() - - items = {"foo": "The Foo Wrestlers"} - - - @app.get("/items/{item_id}") - async def read_item(item_id: str): - if item_id not in items: - raise HTTPException(status_code=404, detail="Item not found") - return {"item": items[item_id]} - ``` - """ - - def __init__( - self, - status_code: Annotated[ - int, - Doc( - """ - HTTP status code to send to the client. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/#use-httpexception) - """ - ), - ], - detail: Annotated[ - Any, - Doc( - """ - Any data to be sent to the client in the `detail` key of the JSON - response. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/#use-httpexception) - """ - ), - ] = None, - headers: Annotated[ - Mapping[str, str] | None, - Doc( - """ - Any headers to send to the client in the response. - - Read more about it in the - [FastAPI docs for Handling Errors](https://fastapi.tiangolo.com/tutorial/handling-errors/#add-custom-headers) - - """ - ), - ] = None, - ) -> None: - super().__init__(status_code=status_code, detail=detail, headers=headers) - - -class WebSocketException(StarletteWebSocketException): - """ - A WebSocket exception you can raise in your own code to show errors to the client. - - This is for client errors, invalid authentication, invalid data, etc. Not for server - errors in your code. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import ( - Cookie, - FastAPI, - WebSocket, - WebSocketException, - status, - ) - - app = FastAPI() - - @app.websocket("/items/{item_id}/ws") - async def websocket_endpoint( - *, - websocket: WebSocket, - session: Annotated[str | None, Cookie()] = None, - item_id: str, - ): - if session is None: - raise WebSocketException(code=status.WS_1008_POLICY_VIOLATION) - await websocket.accept() - while True: - data = await websocket.receive_text() - await websocket.send_text(f"Session cookie is: {session}") - await websocket.send_text(f"Message text was: {data}, for item ID: {item_id}") - ``` - """ - - def __init__( - self, - code: Annotated[ - int, - Doc( - """ - A closing code from the - [valid codes defined in the specification](https://datatracker.ietf.org/doc/html/rfc6455#section-7.4.1). - """ - ), - ], - reason: Annotated[ - str | None, - Doc( - """ - The reason to close the WebSocket connection. - - It is UTF-8-encoded data. The interpretation of the reason is up to the - application, it is not specified by the WebSocket specification. - - It could contain text that could be human-readable or interpretable - by the client code, etc. - """ - ), - ] = None, - ) -> None: - super().__init__(code=code, reason=reason) - - -RequestErrorModel: type[BaseModel] = create_model("Request") -WebSocketErrorModel: type[BaseModel] = create_model("WebSocket") - - -class FastAPIError(RuntimeError): - """ - A generic, FastAPI-specific error. - """ - - -class DependencyScopeError(FastAPIError): - """ - A dependency declared that it depends on another dependency with an invalid - (narrower) scope. - """ - - -class ValidationException(Exception): - def __init__( - self, - errors: Sequence[Any], - *, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - self._errors = errors - self.endpoint_ctx = endpoint_ctx - - ctx = endpoint_ctx or {} - self.endpoint_function = ctx.get("function") - self.endpoint_path = ctx.get("path") - self.endpoint_file = ctx.get("file") - self.endpoint_line = ctx.get("line") - - def errors(self) -> Sequence[Any]: - return self._errors - - def _format_endpoint_context(self) -> str: - if not (self.endpoint_file and self.endpoint_line and self.endpoint_function): - if self.endpoint_path: - return f"\n Endpoint: {self.endpoint_path}" - return "" - - context = f'\n File "{self.endpoint_file}", line {self.endpoint_line}, in {self.endpoint_function}' - if self.endpoint_path: - context += f"\n {self.endpoint_path}" - return context - - def __str__(self) -> str: - message = f"{len(self._errors)} validation error{'s' if len(self._errors) != 1 else ''}:\n" - for err in self._errors: - message += f" {err}\n" - message += self._format_endpoint_context() - return message.rstrip() - - -class RequestValidationError(ValidationException): - def __init__( - self, - errors: Sequence[Any], - *, - body: Any = None, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - super().__init__(errors, endpoint_ctx=endpoint_ctx) - self.body = body - - -class WebSocketRequestValidationError(ValidationException): - def __init__( - self, - errors: Sequence[Any], - *, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - super().__init__(errors, endpoint_ctx=endpoint_ctx) - - -class ResponseValidationError(ValidationException): - def __init__( - self, - errors: Sequence[Any], - *, - body: Any = None, - endpoint_ctx: EndpointContext | None = None, - ) -> None: - super().__init__(errors, endpoint_ctx=endpoint_ctx) - self.body = body - - -class PydanticV1NotSupportedError(FastAPIError): - """ - A pydantic.v1 model is used, which is no longer supported. - """ - - -class FastAPIDeprecationWarning(UserWarning): - """ - A custom deprecation warning as DeprecationWarning is ignored - Ref: https://sethmlarson.dev/deprecations-via-warnings-dont-work-for-python-libraries - """ diff --git a/functions/python/data-processor/fastapi/logger.py b/functions/python/data-processor/fastapi/logger.py deleted file mode 100644 index 5b2c4ad52..000000000 --- a/functions/python/data-processor/fastapi/logger.py +++ /dev/null @@ -1,3 +0,0 @@ -import logging - -logger = logging.getLogger("fastapi") diff --git a/functions/python/data-processor/fastapi/middleware/__init__.py b/functions/python/data-processor/fastapi/middleware/__init__.py deleted file mode 100644 index 620296d5a..000000000 --- a/functions/python/data-processor/fastapi/middleware/__init__.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.middleware import Middleware as Middleware diff --git a/functions/python/data-processor/fastapi/middleware/asyncexitstack.py b/functions/python/data-processor/fastapi/middleware/asyncexitstack.py deleted file mode 100644 index 4ce3f5a62..000000000 --- a/functions/python/data-processor/fastapi/middleware/asyncexitstack.py +++ /dev/null @@ -1,18 +0,0 @@ -from contextlib import AsyncExitStack - -from starlette.types import ASGIApp, Receive, Scope, Send - - -# Used mainly to close files after the request is done, dependencies are closed -# in their own AsyncExitStack -class AsyncExitStackMiddleware: - def __init__( - self, app: ASGIApp, context_name: str = "fastapi_middleware_astack" - ) -> None: - self.app = app - self.context_name = context_name - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - async with AsyncExitStack() as stack: - scope[self.context_name] = stack - await self.app(scope, receive, send) diff --git a/functions/python/data-processor/fastapi/middleware/cors.py b/functions/python/data-processor/fastapi/middleware/cors.py deleted file mode 100644 index 8dfaad0db..000000000 --- a/functions/python/data-processor/fastapi/middleware/cors.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.middleware.cors import CORSMiddleware as CORSMiddleware # noqa diff --git a/functions/python/data-processor/fastapi/middleware/gzip.py b/functions/python/data-processor/fastapi/middleware/gzip.py deleted file mode 100644 index bbeb2cc78..000000000 --- a/functions/python/data-processor/fastapi/middleware/gzip.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.middleware.gzip import GZipMiddleware as GZipMiddleware # noqa diff --git a/functions/python/data-processor/fastapi/middleware/httpsredirect.py b/functions/python/data-processor/fastapi/middleware/httpsredirect.py deleted file mode 100644 index b7a3d8e07..000000000 --- a/functions/python/data-processor/fastapi/middleware/httpsredirect.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.middleware.httpsredirect import ( # noqa - HTTPSRedirectMiddleware as HTTPSRedirectMiddleware, -) diff --git a/functions/python/data-processor/fastapi/middleware/trustedhost.py b/functions/python/data-processor/fastapi/middleware/trustedhost.py deleted file mode 100644 index 08d7e0353..000000000 --- a/functions/python/data-processor/fastapi/middleware/trustedhost.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.middleware.trustedhost import ( # noqa - TrustedHostMiddleware as TrustedHostMiddleware, -) diff --git a/functions/python/data-processor/fastapi/middleware/wsgi.py b/functions/python/data-processor/fastapi/middleware/wsgi.py deleted file mode 100644 index 69e4dcab9..000000000 --- a/functions/python/data-processor/fastapi/middleware/wsgi.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.middleware.wsgi import ( - WSGIMiddleware as WSGIMiddleware, -) # pragma: no cover # noqa diff --git a/functions/python/data-processor/fastapi/openapi/__init__.py b/functions/python/data-processor/fastapi/openapi/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/fastapi/openapi/constants.py b/functions/python/data-processor/fastapi/openapi/constants.py deleted file mode 100644 index d724ee3cf..000000000 --- a/functions/python/data-processor/fastapi/openapi/constants.py +++ /dev/null @@ -1,3 +0,0 @@ -METHODS_WITH_BODY = {"GET", "HEAD", "POST", "PUT", "DELETE", "PATCH"} -REF_PREFIX = "#/components/schemas/" -REF_TEMPLATE = "#/components/schemas/{model}" diff --git a/functions/python/data-processor/fastapi/openapi/docs.py b/functions/python/data-processor/fastapi/openapi/docs.py deleted file mode 100644 index 0d9242f9f..000000000 --- a/functions/python/data-processor/fastapi/openapi/docs.py +++ /dev/null @@ -1,389 +0,0 @@ -import json -from typing import Annotated, Any - -from annotated_doc import Doc -from fastapi.encoders import jsonable_encoder -from starlette.responses import HTMLResponse - - -def _html_safe_json(value: Any) -> str: - """Serialize a value to JSON with HTML special characters escaped. - - This prevents injection when the JSON is embedded inside a - - - - - """ - return HTMLResponse(html) - - -def get_redoc_html( - *, - openapi_url: Annotated[ - str, - Doc( - """ - The OpenAPI URL that ReDoc should load and use. - - This is normally done automatically by FastAPI using the default URL - `/openapi.json`. - - Read more about it in the - [FastAPI docs for Conditional OpenAPI](https://fastapi.tiangolo.com/how-to/conditional-openapi/#conditional-openapi-from-settings-and-env-vars) - """ - ), - ], - title: Annotated[ - str, - Doc( - """ - The HTML `` content, normally shown in the browser tab. - - Read more about it in the - [FastAPI docs for Custom Docs UI Static Assets](https://fastapi.tiangolo.com/how-to/custom-docs-ui-assets/) - """ - ), - ], - redoc_js_url: Annotated[ - str, - Doc( - """ - The URL to use to load the ReDoc JavaScript. - - It is normally set to a CDN URL. - - Read more about it in the - [FastAPI docs for Custom Docs UI Static Assets](https://fastapi.tiangolo.com/how-to/custom-docs-ui-assets/) - """ - ), - ] = "https://cdn.jsdelivr.net/npm/redoc@2/bundles/redoc.standalone.js", - redoc_favicon_url: Annotated[ - str, - Doc( - """ - The URL of the favicon to use. It is normally shown in the browser tab. - """ - ), - ] = "https://fastapi.tiangolo.com/img/favicon.png", - with_google_fonts: Annotated[ - bool, - Doc( - """ - Load and use Google Fonts. - """ - ), - ] = True, -) -> HTMLResponse: - """ - Generate and return the HTML response that loads ReDoc for the alternative - API docs (normally served at `/redoc`). - - You would only call this function yourself if you needed to override some parts, - for example the URLs to use to load ReDoc's JavaScript and CSS. - - Read more about it in the - [FastAPI docs for Custom Docs UI Static Assets (Self-Hosting)](https://fastapi.tiangolo.com/how-to/custom-docs-ui-assets/). - """ - html = f""" - <!DOCTYPE html> - <html> - <head> - <title>{title} - - - - """ - if with_google_fonts: - html += """ - - """ - html += f""" - - - - - - - - - - - """ - return HTMLResponse(html) - - -def get_swagger_ui_oauth2_redirect_html() -> HTMLResponse: - """ - Generate the HTML response with the OAuth2 redirection for Swagger UI. - - You normally don't need to use or change this. - """ - # copied from https://github.com/swagger-api/swagger-ui/blob/v4.14.0/dist/oauth2-redirect.html - html = """ - - - - Swagger UI: OAuth2 Redirect - - - - - - """ - return HTMLResponse(content=html) diff --git a/functions/python/data-processor/fastapi/openapi/models.py b/functions/python/data-processor/fastapi/openapi/models.py deleted file mode 100644 index ca26bf931..000000000 --- a/functions/python/data-processor/fastapi/openapi/models.py +++ /dev/null @@ -1,435 +0,0 @@ -from collections.abc import Callable, Iterable, Mapping -from enum import Enum -from typing import Annotated, Any, Literal, Optional, Union - -from fastapi._compat import with_info_plain_validator_function -from fastapi.logger import logger -from pydantic import ( - AnyUrl, - BaseModel, - Field, - GetJsonSchemaHandler, -) -from typing_extensions import TypedDict -from typing_extensions import deprecated as typing_deprecated - -try: - import email_validator - - assert email_validator # make autoflake ignore the unused import - from pydantic import EmailStr -except ImportError: # pragma: no cover - - class EmailStr(str): # type: ignore[no-redef] - @classmethod - def __get_validators__(cls) -> Iterable[Callable[..., Any]]: - yield cls.validate - - @classmethod - def validate(cls, v: Any) -> str: - logger.warning( - "email-validator not installed, email fields will be treated as str.\n" - "To install, run: pip install email-validator" - ) - return str(v) - - @classmethod - def _validate(cls, __input_value: Any, _: Any) -> str: - logger.warning( - "email-validator not installed, email fields will be treated as str.\n" - "To install, run: pip install email-validator" - ) - return str(__input_value) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: Mapping[str, Any], handler: GetJsonSchemaHandler - ) -> dict[str, Any]: - return {"type": "string", "format": "email"} - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: Callable[[Any], Mapping[str, Any]] - ) -> Mapping[str, Any]: - return with_info_plain_validator_function(cls._validate) - - -class BaseModelWithConfig(BaseModel): - model_config = {"extra": "allow"} - - -class Contact(BaseModelWithConfig): - name: str | None = None - url: AnyUrl | None = None - email: EmailStr | None = None - - -class License(BaseModelWithConfig): - name: str - identifier: str | None = None - url: AnyUrl | None = None - - -class Info(BaseModelWithConfig): - title: str - summary: str | None = None - description: str | None = None - termsOfService: str | None = None - contact: Contact | None = None - license: License | None = None - version: str - - -class ServerVariable(BaseModelWithConfig): - enum: Annotated[list[str] | None, Field(min_length=1)] = None - default: str - description: str | None = None - - -class Server(BaseModelWithConfig): - url: AnyUrl | str - description: str | None = None - variables: dict[str, ServerVariable] | None = None - - -class Reference(BaseModel): - ref: str = Field(alias="$ref") - - -class Discriminator(BaseModel): - propertyName: str - mapping: dict[str, str] | None = None - - -class XML(BaseModelWithConfig): - name: str | None = None - namespace: str | None = None - prefix: str | None = None - attribute: bool | None = None - wrapped: bool | None = None - - -class ExternalDocumentation(BaseModelWithConfig): - description: str | None = None - url: AnyUrl - - -# Ref JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation#name-type -SchemaType = Literal[ - "array", "boolean", "integer", "null", "number", "object", "string" -] - - -class Schema(BaseModelWithConfig): - # Ref: JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-core.html#name-the-json-schema-core-vocabu - # Core Vocabulary - schema_: str | None = Field(default=None, alias="$schema") - vocabulary: str | None = Field(default=None, alias="$vocabulary") - id: str | None = Field(default=None, alias="$id") - anchor: str | None = Field(default=None, alias="$anchor") - dynamicAnchor: str | None = Field(default=None, alias="$dynamicAnchor") - ref: str | None = Field(default=None, alias="$ref") - dynamicRef: str | None = Field(default=None, alias="$dynamicRef") - defs: dict[str, "SchemaOrBool"] | None = Field(default=None, alias="$defs") - comment: str | None = Field(default=None, alias="$comment") - # Ref: JSON Schema 2020-12: https://json-schema.org/draft/2020-12/json-schema-core.html#name-a-vocabulary-for-applying-s - # A Vocabulary for Applying Subschemas - allOf: list["SchemaOrBool"] | None = None - anyOf: list["SchemaOrBool"] | None = None - oneOf: list["SchemaOrBool"] | None = None - not_: Optional["SchemaOrBool"] = Field(default=None, alias="not") - if_: Optional["SchemaOrBool"] = Field(default=None, alias="if") - then: Optional["SchemaOrBool"] = None - else_: Optional["SchemaOrBool"] = Field(default=None, alias="else") - dependentSchemas: dict[str, "SchemaOrBool"] | None = None - prefixItems: list["SchemaOrBool"] | None = None - items: Optional["SchemaOrBool"] = None - contains: Optional["SchemaOrBool"] = None - properties: dict[str, "SchemaOrBool"] | None = None - patternProperties: dict[str, "SchemaOrBool"] | None = None - additionalProperties: Optional["SchemaOrBool"] = None - propertyNames: Optional["SchemaOrBool"] = None - unevaluatedItems: Optional["SchemaOrBool"] = None - unevaluatedProperties: Optional["SchemaOrBool"] = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-a-vocabulary-for-structural - # A Vocabulary for Structural Validation - type: SchemaType | list[SchemaType] | None = None - enum: list[Any] | None = None - const: Any | None = None - multipleOf: float | None = Field(default=None, gt=0) - maximum: float | None = None - exclusiveMaximum: float | None = None - minimum: float | None = None - exclusiveMinimum: float | None = None - maxLength: int | None = Field(default=None, ge=0) - minLength: int | None = Field(default=None, ge=0) - pattern: str | None = None - maxItems: int | None = Field(default=None, ge=0) - minItems: int | None = Field(default=None, ge=0) - uniqueItems: bool | None = None - maxContains: int | None = Field(default=None, ge=0) - minContains: int | None = Field(default=None, ge=0) - maxProperties: int | None = Field(default=None, ge=0) - minProperties: int | None = Field(default=None, ge=0) - required: list[str] | None = None - dependentRequired: dict[str, set[str]] | None = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-vocabularies-for-semantic-c - # Vocabularies for Semantic Content With "format" - format: str | None = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-a-vocabulary-for-the-conten - # A Vocabulary for the Contents of String-Encoded Data - contentEncoding: str | None = None - contentMediaType: str | None = None - contentSchema: Optional["SchemaOrBool"] = None - # Ref: JSON Schema Validation 2020-12: https://json-schema.org/draft/2020-12/json-schema-validation.html#name-a-vocabulary-for-basic-meta - # A Vocabulary for Basic Meta-Data Annotations - title: str | None = None - description: str | None = None - default: Any | None = None - deprecated: bool | None = None - readOnly: bool | None = None - writeOnly: bool | None = None - examples: list[Any] | None = None - # Ref: OpenAPI 3.1.0: https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#schema-object - # Schema Object - discriminator: Discriminator | None = None - xml: XML | None = None - externalDocs: ExternalDocumentation | None = None - example: Annotated[ - Any | None, - typing_deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = None - - -# Ref: https://json-schema.org/draft/2020-12/json-schema-core.html#name-json-schema-documents -# A JSON Schema MUST be an object or a boolean. -SchemaOrBool = Schema | bool - - -class Example(TypedDict, total=False): - summary: str | None - description: str | None - value: Any | None - externalValue: AnyUrl | None - - __pydantic_config__ = {"extra": "allow"} # type: ignore[misc] # ty: ignore[invalid-typed-dict-statement] - - -class ParameterInType(Enum): - query = "query" - header = "header" - path = "path" - cookie = "cookie" - - -class Encoding(BaseModelWithConfig): - contentType: str | None = None - headers: dict[str, Union["Header", Reference]] | None = None - style: str | None = None - explode: bool | None = None - allowReserved: bool | None = None - - -class MediaType(BaseModelWithConfig): - schema_: Schema | Reference | None = Field(default=None, alias="schema") - example: Any | None = None - examples: dict[str, Example | Reference] | None = None - encoding: dict[str, Encoding] | None = None - - -class ParameterBase(BaseModelWithConfig): - description: str | None = None - required: bool | None = None - deprecated: bool | None = None - # Serialization rules for simple scenarios - style: str | None = None - explode: bool | None = None - allowReserved: bool | None = None - schema_: Schema | Reference | None = Field(default=None, alias="schema") - example: Any | None = None - examples: dict[str, Example | Reference] | None = None - # Serialization rules for more complex scenarios - content: dict[str, MediaType] | None = None - - -class Parameter(ParameterBase): - name: str - in_: ParameterInType = Field(alias="in") - - -class Header(ParameterBase): - pass - - -class RequestBody(BaseModelWithConfig): - description: str | None = None - content: dict[str, MediaType] - required: bool | None = None - - -class Link(BaseModelWithConfig): - operationRef: str | None = None - operationId: str | None = None - parameters: dict[str, Any | str] | None = None - requestBody: Any | str | None = None - description: str | None = None - server: Server | None = None - - -class Response(BaseModelWithConfig): - description: str - headers: dict[str, Header | Reference] | None = None - content: dict[str, MediaType] | None = None - links: dict[str, Link | Reference] | None = None - - -class Operation(BaseModelWithConfig): - tags: list[str] | None = None - summary: str | None = None - description: str | None = None - externalDocs: ExternalDocumentation | None = None - operationId: str | None = None - parameters: list[Parameter | Reference] | None = None - requestBody: RequestBody | Reference | None = None - # Using Any for Specification Extensions - responses: dict[str, Response | Any] | None = None - callbacks: dict[str, dict[str, "PathItem"] | Reference] | None = None - deprecated: bool | None = None - security: list[dict[str, list[str]]] | None = None - servers: list[Server] | None = None - - -class PathItem(BaseModelWithConfig): - ref: str | None = Field(default=None, alias="$ref") - summary: str | None = None - description: str | None = None - get: Operation | None = None - put: Operation | None = None - post: Operation | None = None - delete: Operation | None = None - options: Operation | None = None - head: Operation | None = None - patch: Operation | None = None - trace: Operation | None = None - servers: list[Server] | None = None - parameters: list[Parameter | Reference] | None = None - - -class SecuritySchemeType(Enum): - apiKey = "apiKey" - http = "http" - oauth2 = "oauth2" - openIdConnect = "openIdConnect" - - -class SecurityBase(BaseModelWithConfig): - type_: SecuritySchemeType = Field(alias="type") - description: str | None = None - - -class APIKeyIn(Enum): - query = "query" - header = "header" - cookie = "cookie" - - -class APIKey(SecurityBase): - type_: SecuritySchemeType = Field(default=SecuritySchemeType.apiKey, alias="type") - in_: APIKeyIn = Field(alias="in") - name: str - - -class HTTPBase(SecurityBase): - type_: SecuritySchemeType = Field(default=SecuritySchemeType.http, alias="type") - scheme: str - - -class HTTPBearer(HTTPBase): - scheme: Literal["bearer"] = "bearer" - bearerFormat: str | None = None - - -class OAuthFlow(BaseModelWithConfig): - refreshUrl: str | None = None - scopes: dict[str, str] = {} - - -class OAuthFlowImplicit(OAuthFlow): - authorizationUrl: str - - -class OAuthFlowPassword(OAuthFlow): - tokenUrl: str - - -class OAuthFlowClientCredentials(OAuthFlow): - tokenUrl: str - - -class OAuthFlowAuthorizationCode(OAuthFlow): - authorizationUrl: str - tokenUrl: str - - -class OAuthFlows(BaseModelWithConfig): - implicit: OAuthFlowImplicit | None = None - password: OAuthFlowPassword | None = None - clientCredentials: OAuthFlowClientCredentials | None = None - authorizationCode: OAuthFlowAuthorizationCode | None = None - - -class OAuth2(SecurityBase): - type_: SecuritySchemeType = Field(default=SecuritySchemeType.oauth2, alias="type") - flows: OAuthFlows - - -class OpenIdConnect(SecurityBase): - type_: SecuritySchemeType = Field( - default=SecuritySchemeType.openIdConnect, alias="type" - ) - openIdConnectUrl: str - - -SecurityScheme = APIKey | HTTPBase | OAuth2 | OpenIdConnect | HTTPBearer - - -class Components(BaseModelWithConfig): - schemas: dict[str, Schema | Reference] | None = None - responses: dict[str, Response | Reference] | None = None - parameters: dict[str, Parameter | Reference] | None = None - examples: dict[str, Example | Reference] | None = None - requestBodies: dict[str, RequestBody | Reference] | None = None - headers: dict[str, Header | Reference] | None = None - securitySchemes: dict[str, SecurityScheme | Reference] | None = None - links: dict[str, Link | Reference] | None = None - # Using Any for Specification Extensions - callbacks: dict[str, dict[str, PathItem] | Reference | Any] | None = None - pathItems: dict[str, PathItem | Reference] | None = None - - -class Tag(BaseModelWithConfig): - name: str - description: str | None = None - externalDocs: ExternalDocumentation | None = None - - -class OpenAPI(BaseModelWithConfig): - openapi: str - info: Info - jsonSchemaDialect: str | None = None - servers: list[Server] | None = None - # Using Any for Specification Extensions - paths: dict[str, PathItem | Any] | None = None - webhooks: dict[str, PathItem | Reference] | None = None - components: Components | None = None - security: list[dict[str, list[str]]] | None = None - tags: list[Tag] | None = None - externalDocs: ExternalDocumentation | None = None - - -Schema.model_rebuild() -Operation.model_rebuild() -Encoding.model_rebuild() diff --git a/functions/python/data-processor/fastapi/openapi/utils.py b/functions/python/data-processor/fastapi/openapi/utils.py deleted file mode 100644 index 1c7a17c4c..000000000 --- a/functions/python/data-processor/fastapi/openapi/utils.py +++ /dev/null @@ -1,606 +0,0 @@ -import copy -import http.client -import inspect -import warnings -from collections.abc import Sequence -from typing import Any, Literal, cast - -from fastapi import routing -from fastapi._compat import ( - ModelField, - get_definitions, - get_flat_models_from_fields, - get_model_name_map, - get_schema_from_model_field, - lenient_issubclass, -) -from fastapi.datastructures import DefaultPlaceholder, _Unset -from fastapi.dependencies.models import Dependant -from fastapi.dependencies.utils import ( - _get_flat_fields_from_params, - get_flat_dependant, - get_flat_params, - get_validation_alias, -) -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import FastAPIDeprecationWarning -from fastapi.openapi.constants import METHODS_WITH_BODY, REF_PREFIX -from fastapi.openapi.models import OpenAPI -from fastapi.params import Body, ParamTypes -from fastapi.responses import Response -from fastapi.sse import _SSE_EVENT_SCHEMA -from fastapi.types import ModelNameMap -from fastapi.utils import ( - deep_dict_update, - generate_operation_id_for_path, - is_body_allowed_for_status_code, -) -from pydantic import BaseModel -from starlette.responses import JSONResponse -from starlette.routing import BaseRoute - -validation_error_definition = { - "title": "ValidationError", - "type": "object", - "properties": { - "loc": { - "title": "Location", - "type": "array", - "items": {"anyOf": [{"type": "string"}, {"type": "integer"}]}, - }, - "msg": {"title": "Message", "type": "string"}, - "type": {"title": "Error Type", "type": "string"}, - "input": {"title": "Input"}, - "ctx": {"title": "Context", "type": "object"}, - }, - "required": ["loc", "msg", "type"], -} - -validation_error_response_definition = { - "title": "HTTPValidationError", - "type": "object", - "properties": { - "detail": { - "title": "Detail", - "type": "array", - "items": {"$ref": REF_PREFIX + "ValidationError"}, - } - }, -} - -status_code_ranges: dict[str, str] = { - "1XX": "Information", - "2XX": "Success", - "3XX": "Redirection", - "4XX": "Client Error", - "5XX": "Server Error", - "DEFAULT": "Default Response", -} - - -def get_openapi_security_definitions( - flat_dependant: Dependant, -) -> tuple[dict[str, Any], list[dict[str, Any]]]: - security_definitions = {} - # Use a dict to merge scopes for same security scheme - operation_security_dict: dict[str, list[str]] = {} - for security_dependency in flat_dependant._security_dependencies: - security_definition = jsonable_encoder( - security_dependency._security_scheme.model, - by_alias=True, - exclude_none=True, - ) - security_name = security_dependency._security_scheme.scheme_name - security_definitions[security_name] = security_definition - # Merge scopes for the same security scheme - if security_name not in operation_security_dict: - operation_security_dict[security_name] = [] - for scope in security_dependency.oauth_scopes or []: - if scope not in operation_security_dict[security_name]: - operation_security_dict[security_name].append(scope) - operation_security = [ - {name: scopes} for name, scopes in operation_security_dict.items() - ] - return security_definitions, operation_security - - -def _get_openapi_operation_parameters( - *, - dependant: Dependant, - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], dict[str, Any] - ], - separate_input_output_schemas: bool = True, -) -> list[dict[str, Any]]: - parameters = [] - flat_dependant = get_flat_dependant(dependant, skip_repeats=True) - path_params = _get_flat_fields_from_params(flat_dependant.path_params) - query_params = _get_flat_fields_from_params(flat_dependant.query_params) - header_params = _get_flat_fields_from_params(flat_dependant.header_params) - cookie_params = _get_flat_fields_from_params(flat_dependant.cookie_params) - parameter_groups = [ - (ParamTypes.path, path_params), - (ParamTypes.query, query_params), - (ParamTypes.header, header_params), - (ParamTypes.cookie, cookie_params), - ] - default_convert_underscores = True - if len(flat_dependant.header_params) == 1: - first_field = flat_dependant.header_params[0] - if lenient_issubclass(first_field.field_info.annotation, BaseModel): - default_convert_underscores = getattr( - first_field.field_info, "convert_underscores", True - ) - for param_type, param_group in parameter_groups: - for param in param_group: - field_info = param.field_info - # field_info = cast(Param, field_info) - if not getattr(field_info, "include_in_schema", True): - continue - param_schema = get_schema_from_model_field( - field=param, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - name = get_validation_alias(param) - convert_underscores = getattr( - param.field_info, - "convert_underscores", - default_convert_underscores, - ) - if ( - param_type == ParamTypes.header - and name == param.name - and convert_underscores - ): - name = param.name.replace("_", "-") - - parameter = { - "name": name, - "in": param_type.value, - "required": param.field_info.is_required(), - "schema": param_schema, - } - if field_info.description: - parameter["description"] = field_info.description - openapi_examples = getattr(field_info, "openapi_examples", None) - example = getattr(field_info, "example", None) - if openapi_examples: - parameter["examples"] = jsonable_encoder(openapi_examples) - elif example is not _Unset: - parameter["example"] = jsonable_encoder(example) - if getattr(field_info, "deprecated", None): - parameter["deprecated"] = True - parameters.append(parameter) - return parameters - - -def get_openapi_operation_request_body( - *, - body_field: ModelField | None, - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], dict[str, Any] - ], - separate_input_output_schemas: bool = True, -) -> dict[str, Any] | None: - if not body_field: - return None - assert isinstance(body_field, ModelField) - body_schema = get_schema_from_model_field( - field=body_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - field_info = cast(Body, body_field.field_info) - request_media_type = field_info.media_type - required = body_field.field_info.is_required() - request_body_oai: dict[str, Any] = {} - if required: - request_body_oai["required"] = required - request_media_content: dict[str, Any] = {"schema": body_schema} - if field_info.openapi_examples: - request_media_content["examples"] = jsonable_encoder( - field_info.openapi_examples - ) - elif field_info.example is not _Unset: - request_media_content["example"] = jsonable_encoder(field_info.example) - request_body_oai["content"] = {request_media_type: request_media_content} - return request_body_oai - - -def generate_operation_id( - *, route: routing.APIRoute, method: str -) -> str: # pragma: nocover - warnings.warn( - message="fastapi.openapi.utils.generate_operation_id() was deprecated, " - "it is not used internally, and will be removed soon", - category=FastAPIDeprecationWarning, - stacklevel=2, - ) - if route.operation_id: - return route.operation_id - path: str = route.path_format - return generate_operation_id_for_path(name=route.name, path=path, method=method) - - -def generate_operation_summary(*, route: routing.APIRoute, method: str) -> str: - if route.summary: - return route.summary - return route.name.replace("_", " ").title() - - -def get_openapi_operation_metadata( - *, route: routing.APIRoute, method: str, operation_ids: set[str] -) -> dict[str, Any]: - operation: dict[str, Any] = {} - if route.tags: - operation["tags"] = route.tags - operation["summary"] = generate_operation_summary(route=route, method=method) - if route.description: - operation["description"] = route.description - operation_id = route.operation_id or route.unique_id - if operation_id in operation_ids: - endpoint_name = getattr(route.endpoint, "__name__", "") - message = f"Duplicate Operation ID {operation_id} for function {endpoint_name}" - file_name = getattr(route.endpoint, "__globals__", {}).get("__file__") - if file_name: - message += f" at {file_name}" - warnings.warn(message, stacklevel=1) - operation_ids.add(operation_id) - operation["operationId"] = operation_id - if route.deprecated: - operation["deprecated"] = route.deprecated - return operation - - -def get_openapi_path( - *, - route: routing.APIRoute, - operation_ids: set[str], - model_name_map: ModelNameMap, - field_mapping: dict[ - tuple[ModelField, Literal["validation", "serialization"]], dict[str, Any] - ], - separate_input_output_schemas: bool = True, -) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]: - path = {} - security_schemes: dict[str, Any] = {} - definitions: dict[str, Any] = {} - assert route.methods is not None, "Methods must be a list" - if isinstance(route.response_class, DefaultPlaceholder): - current_response_class: type[Response] = route.response_class.value - else: - current_response_class = route.response_class - assert current_response_class, "A response class is needed to generate OpenAPI" - route_response_media_type: str | None = current_response_class.media_type - if route.include_in_schema: - for method in route.methods: - operation = get_openapi_operation_metadata( - route=route, method=method, operation_ids=operation_ids - ) - parameters: list[dict[str, Any]] = [] - flat_dependant = get_flat_dependant(route.dependant, skip_repeats=True) - security_definitions, operation_security = get_openapi_security_definitions( - flat_dependant=flat_dependant - ) - if operation_security: - operation.setdefault("security", []).extend(operation_security) - if security_definitions: - security_schemes.update(security_definitions) - operation_parameters = _get_openapi_operation_parameters( - dependant=route.dependant, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - parameters.extend(operation_parameters) - if parameters: - all_parameters = { - (param["in"], param["name"]): param for param in parameters - } - required_parameters = { - (param["in"], param["name"]): param - for param in parameters - if param.get("required") - } - # Make sure required definitions of the same parameter take precedence - # over non-required definitions - all_parameters.update(required_parameters) - operation["parameters"] = list(all_parameters.values()) - if method in METHODS_WITH_BODY: - request_body_oai = get_openapi_operation_request_body( - body_field=route.body_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - if request_body_oai: - operation["requestBody"] = request_body_oai - if route.callbacks: - callbacks = {} - for callback in route.callbacks: - if isinstance(callback, routing.APIRoute): - ( - cb_path, - cb_security_schemes, - cb_definitions, - ) = get_openapi_path( - route=callback, - operation_ids=operation_ids, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - callbacks[callback.name] = {callback.path: cb_path} - operation["callbacks"] = callbacks - if route.status_code is not None: - status_code = str(route.status_code) - else: - # It would probably make more sense for all response classes to have an - # explicit default status_code, and to extract it from them, instead of - # doing this inspection tricks, that would probably be in the future - # TODO: probably make status_code a default class attribute for all - # responses in Starlette - response_signature = inspect.signature(current_response_class.__init__) - status_code_param = response_signature.parameters.get("status_code") - if status_code_param is not None: - if isinstance(status_code_param.default, int): - status_code = str(status_code_param.default) - operation.setdefault("responses", {}).setdefault(status_code, {})[ - "description" - ] = route.response_description - if is_body_allowed_for_status_code(route.status_code): - # Check for JSONL streaming (generator endpoints) - if route.is_json_stream: - jsonl_content: dict[str, Any] = {} - if route.stream_item_field: - item_schema = get_schema_from_model_field( - field=route.stream_item_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - jsonl_content["itemSchema"] = item_schema - else: - jsonl_content["itemSchema"] = {} - operation.setdefault("responses", {}).setdefault( - status_code, {} - ).setdefault("content", {})["application/jsonl"] = jsonl_content - elif route.is_sse_stream: - sse_content: dict[str, Any] = {} - item_schema = copy.deepcopy(_SSE_EVENT_SCHEMA) - if route.stream_item_field: - content_schema = get_schema_from_model_field( - field=route.stream_item_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - item_schema["required"] = ["data"] - item_schema["properties"]["data"] = { - "type": "string", - "contentMediaType": "application/json", - "contentSchema": content_schema, - } - sse_content["itemSchema"] = item_schema - operation.setdefault("responses", {}).setdefault( - status_code, {} - ).setdefault("content", {})["text/event-stream"] = sse_content - elif route_response_media_type: - response_schema = {"type": "string"} - if lenient_issubclass(current_response_class, JSONResponse): - if route.response_field: - response_schema = get_schema_from_model_field( - field=route.response_field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - else: - response_schema = {} - operation.setdefault("responses", {}).setdefault( - status_code, {} - ).setdefault("content", {}).setdefault( - route_response_media_type, {} - )["schema"] = response_schema - if route.responses: - operation_responses = operation.setdefault("responses", {}) - for ( - additional_status_code, - additional_response, - ) in route.responses.items(): - process_response = copy.deepcopy(additional_response) - process_response.pop("model", None) - status_code_key = str(additional_status_code).upper() - if status_code_key == "DEFAULT": - status_code_key = "default" - openapi_response = operation_responses.setdefault( - status_code_key, {} - ) - assert isinstance(process_response, dict), ( - "An additional response must be a dict" - ) - field = route.response_fields.get(additional_status_code) - additional_field_schema: dict[str, Any] | None = None - if field: - additional_field_schema = get_schema_from_model_field( - field=field, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - media_type = route_response_media_type or "application/json" - additional_schema = ( - process_response.setdefault("content", {}) - .setdefault(media_type, {}) - .setdefault("schema", {}) - ) - deep_dict_update(additional_schema, additional_field_schema) - status_text: str | None = status_code_ranges.get( - str(additional_status_code).upper() - ) or http.client.responses.get(int(additional_status_code)) - description = ( - process_response.get("description") - or openapi_response.get("description") - or status_text - or "Additional Response" - ) - deep_dict_update(openapi_response, process_response) - openapi_response["description"] = description - http422 = "422" - all_route_params = get_flat_params(route.dependant) - if (all_route_params or route.body_field) and not any( - status in operation["responses"] - for status in [http422, "4XX", "default"] - ): - operation["responses"][http422] = { - "description": "Validation Error", - "content": { - "application/json": { - "schema": {"$ref": REF_PREFIX + "HTTPValidationError"} - } - }, - } - if "ValidationError" not in definitions: - definitions.update( - { - "ValidationError": validation_error_definition, - "HTTPValidationError": validation_error_response_definition, - } - ) - if route.openapi_extra: - deep_dict_update(operation, route.openapi_extra) - path[method.lower()] = operation - return path, security_schemes, definitions - - -def get_fields_from_routes( - routes: Sequence[BaseRoute], -) -> list[ModelField]: - body_fields_from_routes: list[ModelField] = [] - responses_from_routes: list[ModelField] = [] - request_fields_from_routes: list[ModelField] = [] - callback_flat_models: list[ModelField] = [] - for route in routes: - if not isinstance(route, routing.APIRoute): - continue - if route.include_in_schema: - if route.body_field: - assert isinstance(route.body_field, ModelField), ( - "A request body must be a Pydantic Field" - ) - body_fields_from_routes.append(route.body_field) - if route.response_field: - responses_from_routes.append(route.response_field) - if route.response_fields: - responses_from_routes.extend(route.response_fields.values()) - if route.stream_item_field: - responses_from_routes.append(route.stream_item_field) - if route.callbacks: - callback_flat_models.extend(get_fields_from_routes(route.callbacks)) - params = get_flat_params(route.dependant) - request_fields_from_routes.extend(params) - - flat_models = callback_flat_models + list( - body_fields_from_routes + responses_from_routes + request_fields_from_routes - ) - return flat_models - - -def get_openapi( - *, - title: str, - version: str, - openapi_version: str = "3.1.0", - summary: str | None = None, - description: str | None = None, - routes: Sequence[BaseRoute], - webhooks: Sequence[BaseRoute] | None = None, - tags: list[dict[str, Any]] | None = None, - servers: list[dict[str, str | Any]] | None = None, - terms_of_service: str | None = None, - contact: dict[str, str | Any] | None = None, - license_info: dict[str, str | Any] | None = None, - separate_input_output_schemas: bool = True, - external_docs: dict[str, Any] | None = None, -) -> dict[str, Any]: - info: dict[str, Any] = {"title": title, "version": version} - if summary: - info["summary"] = summary - if description: - info["description"] = description - if terms_of_service: - info["termsOfService"] = terms_of_service - if contact: - info["contact"] = contact - if license_info: - info["license"] = license_info - output: dict[str, Any] = {"openapi": openapi_version, "info": info} - if servers: - output["servers"] = servers - components: dict[str, dict[str, Any]] = {} - paths: dict[str, dict[str, Any]] = {} - webhook_paths: dict[str, dict[str, Any]] = {} - operation_ids: set[str] = set() - all_fields = get_fields_from_routes(list(routes or []) + list(webhooks or [])) - flat_models = get_flat_models_from_fields(all_fields, known_models=set()) - model_name_map = get_model_name_map(flat_models) - field_mapping, definitions = get_definitions( - fields=all_fields, - model_name_map=model_name_map, - separate_input_output_schemas=separate_input_output_schemas, - ) - for route in routes or []: - if isinstance(route, routing.APIRoute): - result = get_openapi_path( - route=route, - operation_ids=operation_ids, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - if result: - path, security_schemes, path_definitions = result - if path: - paths.setdefault(route.path_format, {}).update(path) - if security_schemes: - components.setdefault("securitySchemes", {}).update( - security_schemes - ) - if path_definitions: - definitions.update(path_definitions) - for webhook in webhooks or []: - if isinstance(webhook, routing.APIRoute): - result = get_openapi_path( - route=webhook, - operation_ids=operation_ids, - model_name_map=model_name_map, - field_mapping=field_mapping, - separate_input_output_schemas=separate_input_output_schemas, - ) - if result: - path, security_schemes, path_definitions = result - if path: - webhook_paths.setdefault(webhook.path_format, {}).update(path) - if security_schemes: - components.setdefault("securitySchemes", {}).update( - security_schemes - ) - if path_definitions: - definitions.update(path_definitions) - if definitions: - components["schemas"] = {k: definitions[k] for k in sorted(definitions)} - if components: - output["components"] = components - output["paths"] = paths - if webhook_paths: - output["webhooks"] = webhook_paths - if tags: - output["tags"] = tags - if external_docs: - output["externalDocs"] = external_docs - return jsonable_encoder(OpenAPI(**output), by_alias=True, exclude_none=True) # type: ignore[no-any-return] diff --git a/functions/python/data-processor/fastapi/param_functions.py b/functions/python/data-processor/fastapi/param_functions.py deleted file mode 100644 index 1856178fc..000000000 --- a/functions/python/data-processor/fastapi/param_functions.py +++ /dev/null @@ -1,2460 +0,0 @@ -from collections.abc import Callable, Sequence -from typing import Annotated, Any, Literal - -from annotated_doc import Doc -from fastapi import params -from fastapi._compat import Undefined -from fastapi.datastructures import _Unset -from fastapi.openapi.models import Example -from pydantic import AliasChoices, AliasPath -from typing_extensions import deprecated - - -def Path( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = ..., - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - - Read more about it in the - [FastAPI docs for Path Parameters and Numeric Validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#declare-metadata) - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - """ - Declare a path parameter for a *path operation*. - - Read more about it in the - [FastAPI docs for Path Parameters and Numeric Validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/). - - ```python - from typing import Annotated - - from fastapi import FastAPI, Path - - app = FastAPI() - - - @app.get("/items/{item_id}") - async def read_items( - item_id: Annotated[int, Path(title="The ID of the item to get")], - ): - return {"item_id": item_id} - ``` - """ - return params.Path( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Query( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#alternative-old-query-as-the-default-value) - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#alias-parameters) - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#declare-more-metadata) - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#declare-more-metadata) - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - - Read more about it in the - [FastAPI docs about Path parameters numeric validations](https://fastapi.tiangolo.com/tutorial/path-params-numeric-validations/#number-validations-greater-than-and-less-than-or-equal) - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/) - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/) - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#add-regular-expressions - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#deprecating-parameters) - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs about Query parameters](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Query( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Header( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - convert_underscores: Annotated[ - bool, - Doc( - """ - Automatically convert underscores to hyphens in the parameter field name. - - Read more about it in the - [FastAPI docs for Header Parameters](https://fastapi.tiangolo.com/tutorial/header-params/#automatic-conversion) - """ - ), - ] = True, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Header( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - convert_underscores=convert_underscores, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Cookie( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Cookie( - default=default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Body( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - embed: Annotated[ - bool | None, - Doc( - """ - When `embed` is `True`, the parameter will be expected in a JSON body as a - key instead of being the JSON body itself. - - This happens automatically when more than one `Body` parameter is declared. - - Read more about it in the - [FastAPI docs for Body - Multiple Parameters](https://fastapi.tiangolo.com/tutorial/body-multiple-params/#embed-a-single-body-parameter). - """ - ), - ] = None, - media_type: Annotated[ - str, - Doc( - """ - The media type of this parameter field. Changing it would affect the - generated OpenAPI, but currently it doesn't affect the parsing of the data. - """ - ), - ] = "application/json", - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Body( - default=default, - default_factory=default_factory, - embed=embed, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Form( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - media_type: Annotated[ - str, - Doc( - """ - The media type of this parameter field. Changing it would affect the - generated OpenAPI, but currently it doesn't affect the parsing of the data. - """ - ), - ] = "application/x-www-form-urlencoded", - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.Form( - default=default, - default_factory=default_factory, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def File( # noqa: N802 - default: Annotated[ - Any, - Doc( - """ - Default value if the parameter field is not set. - """ - ), - ] = Undefined, - *, - default_factory: Annotated[ - Callable[[], Any] | None, - Doc( - """ - A callable to generate the default value. - - This doesn't affect `Path` parameters as the value is always required. - The parameter is available only for compatibility. - """ - ), - ] = _Unset, - media_type: Annotated[ - str, - Doc( - """ - The media type of this parameter field. Changing it would affect the - generated OpenAPI, but currently it doesn't affect the parsing of the data. - """ - ), - ] = "multipart/form-data", - alias: Annotated[ - str | None, - Doc( - """ - An alternative name for the parameter field. - - This will be used to extract the data and for the generated OpenAPI. - It is particularly useful when you can't use the name you want because it - is a Python reserved keyword or similar. - """ - ), - ] = None, - alias_priority: Annotated[ - int | None, - Doc( - """ - Priority of the alias. This affects whether an alias generator is used. - """ - ), - ] = _Unset, - validation_alias: Annotated[ - str | AliasPath | AliasChoices | None, - Doc( - """ - 'Whitelist' validation step. The parameter field will be the single one - allowed by the alias or set of aliases defined. - """ - ), - ] = None, - serialization_alias: Annotated[ - str | None, - Doc( - """ - 'Blacklist' validation step. The vanilla parameter field will be the - single one of the alias' or set of aliases' fields and all the other - fields will be ignored at serialization time. - """ - ), - ] = None, - title: Annotated[ - str | None, - Doc( - """ - Human-readable title. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Human-readable description. - """ - ), - ] = None, - gt: Annotated[ - float | None, - Doc( - """ - Greater than. If set, value must be greater than this. Only applicable to - numbers. - """ - ), - ] = None, - ge: Annotated[ - float | None, - Doc( - """ - Greater than or equal. If set, value must be greater than or equal to - this. Only applicable to numbers. - """ - ), - ] = None, - lt: Annotated[ - float | None, - Doc( - """ - Less than. If set, value must be less than this. Only applicable to numbers. - """ - ), - ] = None, - le: Annotated[ - float | None, - Doc( - """ - Less than or equal. If set, value must be less than or equal to this. - Only applicable to numbers. - """ - ), - ] = None, - min_length: Annotated[ - int | None, - Doc( - """ - Minimum length for strings. - """ - ), - ] = None, - max_length: Annotated[ - int | None, - Doc( - """ - Maximum length for strings. - """ - ), - ] = None, - pattern: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - ] = None, - regex: Annotated[ - str | None, - Doc( - """ - RegEx pattern for strings. - """ - ), - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: Annotated[ - str | None, - Doc( - """ - Parameter field name for discriminating the type in a tagged union. - """ - ), - ] = None, - strict: Annotated[ - bool | None, - Doc( - """ - If `True`, strict validation is applied to the field. - """ - ), - ] = _Unset, - multiple_of: Annotated[ - float | None, - Doc( - """ - Value must be a multiple of this. Only applicable to numbers. - """ - ), - ] = _Unset, - allow_inf_nan: Annotated[ - bool | None, - Doc( - """ - Allow `inf`, `-inf`, `nan`. Only applicable to numbers. - """ - ), - ] = _Unset, - max_digits: Annotated[ - int | None, - Doc( - """ - Maximum number of digits allowed for decimal values. - """ - ), - ] = _Unset, - decimal_places: Annotated[ - int | None, - Doc( - """ - Maximum number of decimal places allowed for decimal values. - """ - ), - ] = _Unset, - examples: Annotated[ - list[Any] | None, - Doc( - """ - Example values for this field. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/) - """ - ), - ] = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: Annotated[ - dict[str, Example] | None, - Doc( - """ - OpenAPI-specific examples. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Swagger UI (that provides the `/docs` interface) has better support for the - OpenAPI-specific examples than the JSON Schema `examples`, that's the main - use case for this. - - Read more about it in the - [FastAPI docs for Declare Request Example Data](https://fastapi.tiangolo.com/tutorial/schema-extra-example/#using-the-openapi_examples-parameter). - """ - ), - ] = None, - deprecated: Annotated[ - deprecated | str | bool | None, - Doc( - """ - Mark this parameter field as deprecated. - - It will affect the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) this parameter field in the generated OpenAPI. - You probably don't need it, but it's available. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - json_schema_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Any additional JSON schema data. - """ - ), - ] = None, - **extra: Annotated[ - Any, - Doc( - """ - Include extra fields used by the JSON Schema. - """ - ), - deprecated( - """ - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - """ - ), - ], -) -> Any: - return params.File( - default=default, - default_factory=default_factory, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - example=example, - examples=examples, - openapi_examples=openapi_examples, - deprecated=deprecated, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -def Depends( # noqa: N802 - dependency: Annotated[ - Callable[..., Any] | None, - Doc( - """ - A "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you, just pass the object - directly. - - Read more about it in the - [FastAPI docs for Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/) - """ - ), - ] = None, - *, - use_cache: Annotated[ - bool, - Doc( - """ - By default, after a dependency is called the first time in a request, if - the dependency is declared again for the rest of the request (for example - if the dependency is needed by several dependencies), the value will be - re-used for the rest of the request. - - Set `use_cache` to `False` to disable this behavior and ensure the - dependency is called again (if declared more than once) in the same request. - - Read more about it in the - [FastAPI docs about sub-dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/sub-dependencies/#using-the-same-dependency-multiple-times) - """ - ), - ] = True, - scope: Annotated[ - Literal["function", "request"] | None, - Doc( - """ - Mainly for dependencies with `yield`, define when the dependency function - should start (the code before `yield`) and when it should end (the code - after `yield`). - - * `"function"`: start the dependency before the *path operation function* - that handles the request, end the dependency after the *path operation - function* ends, but **before** the response is sent back to the client. - So, the dependency function will be executed **around** the *path operation - **function***. - * `"request"`: start the dependency before the *path operation function* - that handles the request (similar to when using `"function"`), but end - **after** the response is sent back to the client. So, the dependency - function will be executed **around** the **request** and response cycle. - - Read more about it in the - [FastAPI docs for FastAPI Dependencies with yield](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-with-yield/#early-exit-and-scope) - """ - ), - ] = None, -) -> Any: - """ - Declare a FastAPI dependency. - - It takes a single "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you. - - Read more about it in the - [FastAPI docs for Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/). - - **Example** - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - - app = FastAPI() - - - async def common_parameters(q: str | None = None, skip: int = 0, limit: int = 100): - return {"q": q, "skip": skip, "limit": limit} - - - @app.get("/items/") - async def read_items(commons: Annotated[dict, Depends(common_parameters)]): - return commons - ``` - """ - return params.Depends(dependency=dependency, use_cache=use_cache, scope=scope) - - -def Security( # noqa: N802 - dependency: Annotated[ - Callable[..., Any] | None, - Doc( - """ - A "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you, just pass the object - directly. - - Read more about it in the - [FastAPI docs for Dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/) - """ - ), - ] = None, - *, - scopes: Annotated[ - Sequence[str] | None, - Doc( - """ - OAuth2 scopes required for the *path operation* that uses this Security - dependency. - - The term "scope" comes from the OAuth2 specification, it seems to be - intentionally vague and interpretable. It normally refers to permissions, - in cases to roles. - - These scopes are integrated with OpenAPI (and the API docs at `/docs`). - So they are visible in the OpenAPI specification. - - Read more about it in the - [FastAPI docs about OAuth2 scopes](https://fastapi.tiangolo.com/advanced/security/oauth2-scopes/) - """ - ), - ] = None, - use_cache: Annotated[ - bool, - Doc( - """ - By default, after a dependency is called the first time in a request, if - the dependency is declared again for the rest of the request (for example - if the dependency is needed by several dependencies), the value will be - re-used for the rest of the request. - - Set `use_cache` to `False` to disable this behavior and ensure the - dependency is called again (if declared more than once) in the same request. - - Read more about it in the - [FastAPI docs about sub-dependencies](https://fastapi.tiangolo.com/tutorial/dependencies/sub-dependencies/#using-the-same-dependency-multiple-times) - """ - ), - ] = True, -) -> Any: - """ - Declare a FastAPI Security dependency. - - The only difference with a regular dependency is that it can declare OAuth2 - scopes that will be integrated with OpenAPI and the automatic UI docs (by default - at `/docs`). - - It takes a single "dependable" callable (like a function). - - Don't call it directly, FastAPI will call it for you. - - Read more about it in the - [FastAPI docs for Security](https://fastapi.tiangolo.com/tutorial/security/) and - in the - [FastAPI docs for OAuth2 scopes](https://fastapi.tiangolo.com/advanced/security/oauth2-scopes/). - - **Example** - - ```python - from typing import Annotated - - from fastapi import Security, FastAPI - - from .db import User - from .security import get_current_active_user - - app = FastAPI() - - @app.get("/users/me/items/") - async def read_own_items( - current_user: Annotated[User, Security(get_current_active_user, scopes=["items"])] - ): - return [{"item_id": "Foo", "owner": current_user.username}] - ``` - """ - return params.Security(dependency=dependency, scopes=scopes, use_cache=use_cache) diff --git a/functions/python/data-processor/fastapi/params.py b/functions/python/data-processor/fastapi/params.py deleted file mode 100644 index d3f2ae175..000000000 --- a/functions/python/data-processor/fastapi/params.py +++ /dev/null @@ -1,754 +0,0 @@ -import warnings -from collections.abc import Callable, Sequence -from dataclasses import dataclass -from enum import Enum -from typing import Annotated, Any, Literal - -from fastapi.exceptions import FastAPIDeprecationWarning -from fastapi.openapi.models import Example -from pydantic import AliasChoices, AliasPath -from pydantic.fields import FieldInfo -from typing_extensions import deprecated - -from ._compat import ( - Undefined, -) -from .datastructures import _Unset - - -class ParamTypes(Enum): - query = "query" - header = "header" - path = "path" - cookie = "cookie" - - -class Param(FieldInfo): # type: ignore[misc] # ty: ignore[subclass-of-final-class] - in_: ParamTypes - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - if example is not _Unset: - warnings.warn( - "`example` has been deprecated, please use `examples` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - self.example = example - self.include_in_schema = include_in_schema - self.openapi_examples = openapi_examples - kwargs = dict( - default=default, - default_factory=default_factory, - alias=alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - discriminator=discriminator, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - **extra, - ) - if examples is not None: - kwargs["examples"] = examples - if regex is not None: - warnings.warn( - "`regex` has been deprecated, please use `pattern` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - current_json_schema_extra = json_schema_extra or extra - kwargs["deprecated"] = deprecated - - if serialization_alias in (_Unset, None) and isinstance(alias, str): - serialization_alias = alias - if validation_alias in (_Unset, None): - validation_alias = alias - kwargs.update( - { - "annotation": annotation, - "alias_priority": alias_priority, - "validation_alias": validation_alias, - "serialization_alias": serialization_alias, - "strict": strict, - "json_schema_extra": current_json_schema_extra, - } - ) - kwargs["pattern"] = pattern or regex - - use_kwargs = {k: v for k, v in kwargs.items() if v is not _Unset} - - super().__init__(**use_kwargs) # ty: ignore[invalid-argument-type] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.default})" - - -class Path(Param): # type: ignore[misc] - in_ = ParamTypes.path - - def __init__( - self, - default: Any = ..., - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - assert default is ..., "Path parameters cannot have a default value" - self.in_ = self.in_ - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Query(Param): # type: ignore[misc] - in_ = ParamTypes.query - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Header(Param): # type: ignore[misc] - in_ = ParamTypes.header - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - convert_underscores: bool = True, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - self.convert_underscores = convert_underscores - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Cookie(Param): # type: ignore[misc] - in_ = ParamTypes.cookie - - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class Body(FieldInfo): # type: ignore[misc] # ty: ignore[subclass-of-final-class] - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - embed: bool | None = None, - media_type: str = "application/json", - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - self.embed = embed - self.media_type = media_type - if example is not _Unset: - warnings.warn( - "`example` has been deprecated, please use `examples` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - self.example = example - self.include_in_schema = include_in_schema - self.openapi_examples = openapi_examples - kwargs = dict( - default=default, - default_factory=default_factory, - alias=alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - discriminator=discriminator, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - **extra, - ) - if examples is not None: - kwargs["examples"] = examples - if regex is not None: - warnings.warn( - "`regex` has been deprecated, please use `pattern` instead", - category=FastAPIDeprecationWarning, - stacklevel=4, - ) - current_json_schema_extra = json_schema_extra or extra - kwargs["deprecated"] = deprecated - if serialization_alias in (_Unset, None) and isinstance(alias, str): - serialization_alias = alias - if validation_alias in (_Unset, None): - validation_alias = alias - kwargs.update( - { - "annotation": annotation, - "alias_priority": alias_priority, - "validation_alias": validation_alias, - "serialization_alias": serialization_alias, - "strict": strict, - "json_schema_extra": current_json_schema_extra, - } - ) - kwargs["pattern"] = pattern or regex - - use_kwargs = {k: v for k, v in kwargs.items() if v is not _Unset} - - super().__init__(**use_kwargs) # ty: ignore[invalid-argument-type] - - def __repr__(self) -> str: - return f"{self.__class__.__name__}({self.default})" - - -class Form(Body): # type: ignore[misc] - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - media_type: str = "application/x-www-form-urlencoded", - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -class File(Form): # type: ignore[misc] - def __init__( - self, - default: Any = Undefined, - *, - default_factory: Callable[[], Any] | None = _Unset, - annotation: Any | None = None, - media_type: str = "multipart/form-data", - alias: str | None = None, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = None, - serialization_alias: str | None = None, - title: str | None = None, - description: str | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | None = None, - regex: Annotated[ - str | None, - deprecated( - "Deprecated in FastAPI 0.100.0 and Pydantic v2, use `pattern` instead." - ), - ] = None, - discriminator: str | None = None, - strict: bool | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - examples: list[Any] | None = None, - example: Annotated[ - Any | None, - deprecated( - "Deprecated in OpenAPI 3.1.0 that now uses JSON Schema 2020-12, " - "although still supported. Use examples instead." - ), - ] = _Unset, - openapi_examples: dict[str, Example] | None = None, - deprecated: deprecated | str | bool | None = None, - include_in_schema: bool = True, - json_schema_extra: dict[str, Any] | None = None, - **extra: Any, - ): - super().__init__( - default=default, - default_factory=default_factory, - annotation=annotation, - media_type=media_type, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - description=description, - gt=gt, - ge=ge, - lt=lt, - le=le, - min_length=min_length, - max_length=max_length, - pattern=pattern, - regex=regex, - discriminator=discriminator, - strict=strict, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - deprecated=deprecated, - example=example, - examples=examples, - openapi_examples=openapi_examples, - include_in_schema=include_in_schema, - json_schema_extra=json_schema_extra, - **extra, - ) - - -@dataclass(frozen=True) -class Depends: - dependency: Callable[..., Any] | None = None - use_cache: bool = True - scope: Literal["function", "request"] | None = None - - -@dataclass(frozen=True) -class Security(Depends): - scopes: Sequence[str] | None = None diff --git a/functions/python/data-processor/fastapi/py.typed b/functions/python/data-processor/fastapi/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/fastapi/requests.py b/functions/python/data-processor/fastapi/requests.py deleted file mode 100644 index d16552c0a..000000000 --- a/functions/python/data-processor/fastapi/requests.py +++ /dev/null @@ -1,2 +0,0 @@ -from starlette.requests import HTTPConnection as HTTPConnection # noqa: F401 -from starlette.requests import Request as Request # noqa: F401 diff --git a/functions/python/data-processor/fastapi/responses.py b/functions/python/data-processor/fastapi/responses.py deleted file mode 100644 index 29df4b7a6..000000000 --- a/functions/python/data-processor/fastapi/responses.py +++ /dev/null @@ -1,98 +0,0 @@ -import importlib -from typing import Any, Protocol, cast - -from fastapi.exceptions import FastAPIDeprecationWarning -from fastapi.sse import EventSourceResponse as EventSourceResponse # noqa -from starlette.responses import FileResponse as FileResponse # noqa -from starlette.responses import HTMLResponse as HTMLResponse # noqa -from starlette.responses import JSONResponse as JSONResponse # noqa -from starlette.responses import PlainTextResponse as PlainTextResponse # noqa -from starlette.responses import RedirectResponse as RedirectResponse # noqa -from starlette.responses import Response as Response # noqa -from starlette.responses import StreamingResponse as StreamingResponse # noqa -from typing_extensions import deprecated - - -class _UjsonModule(Protocol): - def dumps(self, __obj: Any, *, ensure_ascii: bool = ...) -> str: ... - - -class _OrjsonModule(Protocol): - OPT_NON_STR_KEYS: int - OPT_SERIALIZE_NUMPY: int - - def dumps(self, __obj: Any, *, option: int = ...) -> bytes: ... - - -try: - ujson = cast(_UjsonModule, importlib.import_module("ujson")) -except ModuleNotFoundError: # pragma: nocover - ujson = None # type: ignore[assignment] - - -try: - orjson = cast(_OrjsonModule, importlib.import_module("orjson")) -except ModuleNotFoundError: # pragma: nocover - orjson = None # type: ignore[assignment] - - -@deprecated( - "UJSONResponse is deprecated, FastAPI now serializes data directly to JSON " - "bytes via Pydantic when a return type or response model is set, which is " - "faster and doesn't need a custom response class. Read more in the FastAPI " - "docs: https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model " - "and https://fastapi.tiangolo.com/tutorial/response-model/", - category=FastAPIDeprecationWarning, - stacklevel=2, -) -class UJSONResponse(JSONResponse): - """JSON response using the ujson library to serialize data to JSON. - - **Deprecated**: `UJSONResponse` is deprecated. FastAPI now serializes data - directly to JSON bytes via Pydantic when a return type or response model is - set, which is faster and doesn't need a custom response class. - - Read more in the - [FastAPI docs for Custom Response](https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model) - and the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - - **Note**: `ujson` is not included with FastAPI and must be installed - separately, e.g. `pip install ujson`. - """ - - def render(self, content: Any) -> bytes: - assert ujson is not None, "ujson must be installed to use UJSONResponse" - return ujson.dumps(content, ensure_ascii=False).encode("utf-8") - - -@deprecated( - "ORJSONResponse is deprecated, FastAPI now serializes data directly to JSON " - "bytes via Pydantic when a return type or response model is set, which is " - "faster and doesn't need a custom response class. Read more in the FastAPI " - "docs: https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model " - "and https://fastapi.tiangolo.com/tutorial/response-model/", - category=FastAPIDeprecationWarning, - stacklevel=2, -) -class ORJSONResponse(JSONResponse): - """JSON response using the orjson library to serialize data to JSON. - - **Deprecated**: `ORJSONResponse` is deprecated. FastAPI now serializes data - directly to JSON bytes via Pydantic when a return type or response model is - set, which is faster and doesn't need a custom response class. - - Read more in the - [FastAPI docs for Custom Response](https://fastapi.tiangolo.com/advanced/custom-response/#orjson-or-response-model) - and the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - - **Note**: `orjson` is not included with FastAPI and must be installed - separately, e.g. `pip install orjson`. - """ - - def render(self, content: Any) -> bytes: - assert orjson is not None, "orjson must be installed to use ORJSONResponse" - return orjson.dumps( - content, option=orjson.OPT_NON_STR_KEYS | orjson.OPT_SERIALIZE_NUMPY - ) diff --git a/functions/python/data-processor/fastapi/routing.py b/functions/python/data-processor/fastapi/routing.py deleted file mode 100644 index 21a1385a2..000000000 --- a/functions/python/data-processor/fastapi/routing.py +++ /dev/null @@ -1,4956 +0,0 @@ -import contextlib -import email.message -import functools -import inspect -import json -import types -from collections.abc import ( - AsyncIterator, - Awaitable, - Callable, - Collection, - Coroutine, - Generator, - Iterator, - Mapping, - Sequence, -) -from contextlib import ( - AbstractAsyncContextManager, - AbstractContextManager, - AsyncExitStack, - asynccontextmanager, -) -from enum import Enum, IntEnum -from typing import ( - Annotated, - Any, - TypeVar, - cast, -) - -import anyio -from annotated_doc import Doc -from anyio.abc import ObjectReceiveStream -from fastapi import params -from fastapi._compat import ( - ModelField, - Undefined, - lenient_issubclass, -) -from fastapi.datastructures import Default, DefaultPlaceholder -from fastapi.dependencies.models import Dependant -from fastapi.dependencies.utils import ( - _should_embed_body_fields, - get_body_field, - get_dependant, - get_flat_dependant, - get_parameterless_sub_dependant, - get_stream_item_type, - get_typed_return_annotation, - solve_dependencies, -) -from fastapi.encoders import jsonable_encoder -from fastapi.exceptions import ( - EndpointContext, - FastAPIError, - RequestValidationError, - ResponseValidationError, - WebSocketRequestValidationError, -) -from fastapi.sse import ( - _PING_INTERVAL, - KEEPALIVE_COMMENT, - EventSourceResponse, - ServerSentEvent, - format_sse_event, -) -from fastapi.types import DecoratedCallable, IncEx -from fastapi.utils import ( - create_model_field, - generate_unique_id, - get_value_or_default, - is_body_allowed_for_status_code, -) -from starlette import routing -from starlette._exception_handler import wrap_app_handling_exceptions -from starlette._utils import is_async_callable -from starlette.concurrency import iterate_in_threadpool, run_in_threadpool -from starlette.datastructures import FormData -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.responses import JSONResponse, Response, StreamingResponse -from starlette.routing import ( - BaseRoute, - Match, - compile_path, - get_name, -) -from starlette.routing import Mount as Mount # noqa -from starlette.types import AppType, ASGIApp, Lifespan, Receive, Scope, Send -from starlette.websockets import WebSocket -from typing_extensions import deprecated - - -# Copy of starlette.routing.request_response modified to include the -# dependencies' AsyncExitStack -def request_response( - func: Callable[[Request], Awaitable[Response] | Response], -) -> ASGIApp: - """ - Takes a function or coroutine `func(request) -> response`, - and returns an ASGI application. - """ - f: Callable[[Request], Awaitable[Response]] = ( - func # type: ignore[assignment] - if is_async_callable(func) - else functools.partial(run_in_threadpool, func) # type: ignore[call-arg] - ) # ty: ignore[invalid-assignment] - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - request = Request(scope, receive, send) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - # Starts customization - response_awaited = False - async with AsyncExitStack() as request_stack: - scope["fastapi_inner_astack"] = request_stack - async with AsyncExitStack() as function_stack: - scope["fastapi_function_astack"] = function_stack - response = await f(request) - await response(scope, receive, send) - # Continues customization - response_awaited = True - if not response_awaited: - raise FastAPIError( - "Response not awaited. There's a high chance that the " - "application code is raising an exception and a dependency with yield " - "has a block with a bare except, or a block with except Exception, " - "and is not raising the exception again. Read more about it in the " - "docs: https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-with-yield/#dependencies-with-yield-and-except" - ) - - # Same as in Starlette - await wrap_app_handling_exceptions(app, request)(scope, receive, send) - - return app - - -# Copy of starlette.routing.websocket_session modified to include the -# dependencies' AsyncExitStack -def websocket_session( - func: Callable[[WebSocket], Awaitable[None]], -) -> ASGIApp: - """ - Takes a coroutine `func(session)`, and returns an ASGI application. - """ - # assert asyncio.iscoroutinefunction(func), "WebSocket endpoints must be async" - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - session = WebSocket(scope, receive=receive, send=send) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - async with AsyncExitStack() as request_stack: - scope["fastapi_inner_astack"] = request_stack - async with AsyncExitStack() as function_stack: - scope["fastapi_function_astack"] = function_stack - await func(session) - - # Same as in Starlette - await wrap_app_handling_exceptions(app, session)(scope, receive, send) - - return app - - -_T = TypeVar("_T") - - -# Vendored from starlette.routing to avoid importing private symbols -class _AsyncLiftContextManager(AbstractAsyncContextManager[_T]): - """ - Wraps a synchronous context manager to make it async. - - This is vendored from Starlette to avoid importing private symbols. - """ - - def __init__(self, cm: AbstractContextManager[_T]) -> None: - self._cm = cm - - async def __aenter__(self) -> _T: - return self._cm.__enter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: types.TracebackType | None, - ) -> bool | None: - return self._cm.__exit__(exc_type, exc_value, traceback) - - -# Vendored from starlette.routing to avoid importing private symbols -def _wrap_gen_lifespan_context( - lifespan_context: Callable[[Any], Generator[Any, Any, Any]], -) -> Callable[[Any], AbstractAsyncContextManager[Any]]: - """ - Wrap a generator-based lifespan context into an async context manager. - - This is vendored from Starlette to avoid importing private symbols. - """ - cmgr = contextlib.contextmanager(lifespan_context) - - @functools.wraps(cmgr) - def wrapper(app: Any) -> _AsyncLiftContextManager[Any]: - return _AsyncLiftContextManager(cmgr(app)) - - return wrapper - - -def _merge_lifespan_context( - original_context: Lifespan[Any], nested_context: Lifespan[Any] -) -> Lifespan[Any]: - @asynccontextmanager - async def merged_lifespan( - app: AppType, - ) -> AsyncIterator[Mapping[str, Any] | None]: - async with original_context(app) as maybe_original_state: - async with nested_context(app) as maybe_nested_state: - if maybe_nested_state is None and maybe_original_state is None: - yield None # old ASGI compatibility - else: - yield {**(maybe_nested_state or {}), **(maybe_original_state or {})} - - return merged_lifespan # type: ignore[return-value] # ty: ignore[invalid-return-type] - - -class _DefaultLifespan: - """ - Default lifespan context manager that runs on_startup and on_shutdown handlers. - - This is a copy of the Starlette _DefaultLifespan class that was removed - in Starlette. FastAPI keeps it to maintain backward compatibility with - on_startup and on_shutdown event handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - - def __init__(self, router: "APIRouter") -> None: - self._router = router - - async def __aenter__(self) -> None: - await self._router._startup() - - async def __aexit__(self, *exc_info: object) -> None: - await self._router._shutdown() - - def __call__(self: _T, app: object) -> _T: - return self - - -# Cache for endpoint context to avoid re-extracting on every request -_endpoint_context_cache: dict[int, EndpointContext] = {} - - -def _extract_endpoint_context(func: Any) -> EndpointContext: - """Extract endpoint context with caching to avoid repeated file I/O.""" - func_id = id(func) - - if func_id in _endpoint_context_cache: - return _endpoint_context_cache[func_id] - - try: - ctx: EndpointContext = {} - - if (source_file := inspect.getsourcefile(func)) is not None: - ctx["file"] = source_file - if (line_number := inspect.getsourcelines(func)[1]) is not None: - ctx["line"] = line_number - if (func_name := getattr(func, "__name__", None)) is not None: - ctx["function"] = func_name - except Exception: - ctx = EndpointContext() - - _endpoint_context_cache[func_id] = ctx - return ctx - - -async def serialize_response( - *, - field: ModelField | None = None, - response_content: Any, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = True, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - is_coroutine: bool = True, - endpoint_ctx: EndpointContext | None = None, - dump_json: bool = False, -) -> Any: - if field: - if is_coroutine: - value, errors = field.validate(response_content, {}, loc=("response",)) - else: - value, errors = await run_in_threadpool( - field.validate, response_content, {}, loc=("response",) - ) - if errors: - ctx = endpoint_ctx or EndpointContext() - raise ResponseValidationError( - errors=errors, - body=response_content, - endpoint_ctx=ctx, - ) - serializer = field.serialize_json if dump_json else field.serialize - return serializer( - value, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - else: - return jsonable_encoder(response_content) - - -async def run_endpoint_function( - *, dependant: Dependant, values: dict[str, Any], is_coroutine: bool -) -> Any: - # Only called by get_request_handler. Has been split into its own function to - # facilitate profiling endpoints, since inner functions are harder to profile. - assert dependant.call is not None, "dependant.call must be a function" - - if is_coroutine: - return await dependant.call(**values) - else: - return await run_in_threadpool(dependant.call, **values) - - -def _build_response_args( - *, status_code: int | None, solved_result: Any -) -> dict[str, Any]: - response_args: dict[str, Any] = { - "background": solved_result.background_tasks, - } - # If status_code was set, use it, otherwise use the default from the - # response class, in the case of redirect it's 307 - current_status_code = ( - status_code if status_code else solved_result.response.status_code - ) - if current_status_code is not None: - response_args["status_code"] = current_status_code - if solved_result.response.status_code: - response_args["status_code"] = solved_result.response.status_code - return response_args - - -def get_request_handler( - dependant: Dependant, - body_field: ModelField | None = None, - status_code: int | None = None, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - response_field: ModelField | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - dependency_overrides_provider: Any | None = None, - embed_body_fields: bool = False, - strict_content_type: bool | DefaultPlaceholder = Default(True), - stream_item_field: ModelField | None = None, - is_json_stream: bool = False, -) -> Callable[[Request], Coroutine[Any, Any, Response]]: - assert dependant.call is not None, "dependant.call must be a function" - is_coroutine = dependant.is_coroutine_callable - is_body_form = body_field and isinstance(body_field.field_info, params.Form) - if isinstance(response_class, DefaultPlaceholder): - actual_response_class: type[Response] = response_class.value - else: - actual_response_class = response_class - is_sse_stream = lenient_issubclass(actual_response_class, EventSourceResponse) - if isinstance(strict_content_type, DefaultPlaceholder): - actual_strict_content_type: bool = strict_content_type.value - else: - actual_strict_content_type = strict_content_type - - async def app(request: Request) -> Response: - response: Response | None = None - file_stack = request.scope.get("fastapi_middleware_astack") - assert isinstance(file_stack, AsyncExitStack), ( - "fastapi_middleware_astack not found in request scope" - ) - - # Extract endpoint context for error messages - endpoint_ctx = ( - _extract_endpoint_context(dependant.call) - if dependant.call - else EndpointContext() - ) - - if dependant.path: - # For mounted sub-apps, include the mount path prefix - mount_path = request.scope.get("root_path", "").rstrip("/") - endpoint_ctx["path"] = f"{request.method} {mount_path}{dependant.path}" - - # Read body and auto-close files - try: - body: Any = None - if body_field: - if is_body_form: - body = await request.form() - file_stack.push_async_callback(body.close) - else: - body_bytes = await request.body() - if body_bytes: - json_body: Any = Undefined - content_type_value = request.headers.get("content-type") - if not content_type_value: - if not actual_strict_content_type: - json_body = await request.json() - else: - message = email.message.Message() - message["content-type"] = content_type_value - if message.get_content_maintype() == "application": - subtype = message.get_content_subtype() - if subtype == "json" or subtype.endswith("+json"): - json_body = await request.json() - if json_body != Undefined: - body = json_body - else: - body = body_bytes - except json.JSONDecodeError as e: - validation_error = RequestValidationError( - [ - { - "type": "json_invalid", - "loc": ("body", e.pos), - "msg": "JSON decode error", - "input": {}, - "ctx": {"error": e.msg}, - } - ], - body=e.doc, - endpoint_ctx=endpoint_ctx, - ) - raise validation_error from e - except HTTPException: - # If a middleware raises an HTTPException, it should be raised again - raise - except Exception as e: - http_error = HTTPException( - status_code=400, detail="There was an error parsing the body" - ) - raise http_error from e - - # Solve dependencies and run path operation function, auto-closing dependencies - errors: list[Any] = [] - async_exit_stack = request.scope.get("fastapi_inner_astack") - assert isinstance(async_exit_stack, AsyncExitStack), ( - "fastapi_inner_astack not found in request scope" - ) - solved_result = await solve_dependencies( - request=request, - dependant=dependant, - body=cast(dict[str, Any] | FormData | bytes | None, body), - dependency_overrides_provider=dependency_overrides_provider, - async_exit_stack=async_exit_stack, - embed_body_fields=embed_body_fields, - ) - errors = solved_result.errors - assert dependant.call # For types - if not errors: - # Shared serializer for stream items (JSONL and SSE). - # Validates against stream_item_field when set, then - # serializes to JSON bytes. - def _serialize_data(data: Any) -> bytes: - if stream_item_field: - value, errors_ = stream_item_field.validate( - data, {}, loc=("response",) - ) - if errors_: - ctx = endpoint_ctx or EndpointContext() - raise ResponseValidationError( - errors=errors_, - body=data, - endpoint_ctx=ctx, - ) - return stream_item_field.serialize_json( - value, - include=response_model_include, - exclude=response_model_exclude, - by_alias=response_model_by_alias, - exclude_unset=response_model_exclude_unset, - exclude_defaults=response_model_exclude_defaults, - exclude_none=response_model_exclude_none, - ) - else: - data = jsonable_encoder(data) - return json.dumps(data).encode("utf-8") - - if is_sse_stream: - # Generator endpoint: stream as Server-Sent Events - gen = dependant.call(**solved_result.values) - - def _serialize_sse_item(item: Any) -> bytes: - if isinstance(item, ServerSentEvent): - # User controls the event structure. - # Serialize the data payload if present. - # For ServerSentEvent items we skip stream_item_field - # validation (the user may mix types intentionally). - if item.raw_data is not None: - data_str: str | None = item.raw_data - elif item.data is not None: - if hasattr(item.data, "model_dump_json"): - data_str = item.data.model_dump_json() - else: - data_str = json.dumps(jsonable_encoder(item.data)) - else: - data_str = None - return format_sse_event( - data_str=data_str, - event=item.event, - id=item.id, - retry=item.retry, - comment=item.comment, - ) - else: - # Plain object: validate + serialize via - # stream_item_field (if set) and wrap in data field - return format_sse_event( - data_str=_serialize_data(item).decode("utf-8") - ) - - if dependant.is_async_gen_callable: - sse_aiter: AsyncIterator[Any] = gen.__aiter__() - else: - sse_aiter = iterate_in_threadpool(gen) - - @asynccontextmanager - async def _sse_producer_cm() -> AsyncIterator[ - ObjectReceiveStream[bytes] - ]: - # Use a memory stream to decouple generator iteration - # from the keepalive timer. A producer task pulls items - # from the generator independently, so - # `anyio.fail_after` never wraps the generator's - # `__anext__` directly - avoiding CancelledError that - # would finalize the generator and also working for sync - # generators running in a thread pool. - # - # This context manager is entered on the request-scoped - # AsyncExitStack so its __aexit__ (which cancels the - # task group) is called by the exit stack after the - # streaming response completes — not by async generator - # finalization via GeneratorExit. - # Ref: https://peps.python.org/pep-0789/ - send_stream, receive_stream = anyio.create_memory_object_stream[ - bytes - ](max_buffer_size=1) - - async def _producer() -> None: - async with send_stream: - async for raw_item in sse_aiter: - await send_stream.send(_serialize_sse_item(raw_item)) - - send_keepalive, receive_keepalive = ( - anyio.create_memory_object_stream[bytes](max_buffer_size=1) - ) - - async def _keepalive_inserter() -> None: - """Read from the producer and forward to the output, - inserting keepalive comments on timeout.""" - async with send_keepalive, receive_stream: - try: - while True: - try: - with anyio.fail_after(_PING_INTERVAL): - data = await receive_stream.receive() - await send_keepalive.send(data) - except TimeoutError: - await send_keepalive.send(KEEPALIVE_COMMENT) - except anyio.EndOfStream: - pass - - async with anyio.create_task_group() as tg: - tg.start_soon(_producer) - tg.start_soon(_keepalive_inserter) - yield receive_keepalive - tg.cancel_scope.cancel() - - # Enter the SSE context manager on the request-scoped - # exit stack. The stack outlives the streaming response, - # so __aexit__ runs via proper structured teardown, not - # via GeneratorExit thrown into an async generator. - sse_receive_stream = await async_exit_stack.enter_async_context( - _sse_producer_cm() - ) - # Ensure the receive stream is closed when the exit stack - # unwinds, preventing ResourceWarning from __del__. - async_exit_stack.push_async_callback(sse_receive_stream.aclose) - - async def _sse_with_checkpoints( - stream: ObjectReceiveStream[bytes], - ) -> AsyncIterator[bytes]: - async for data in stream: - yield data - # Guarantee a checkpoint so cancellation can be - # delivered even when the producer is faster than - # the consumer and receive() never suspends. - await anyio.sleep(0) - - sse_stream_content: AsyncIterator[bytes] | Iterator[bytes] = ( - _sse_with_checkpoints(sse_receive_stream) - ) - - response = StreamingResponse( - sse_stream_content, - media_type="text/event-stream", - background=solved_result.background_tasks, - ) - response.headers["Cache-Control"] = "no-cache" - # For Nginx proxies to not buffer server sent events - response.headers["X-Accel-Buffering"] = "no" - response.headers.raw.extend(solved_result.response.headers.raw) - elif is_json_stream: - # Generator endpoint: stream as JSONL - gen = dependant.call(**solved_result.values) - - def _serialize_item(item: Any) -> bytes: - return _serialize_data(item) + b"\n" - - if dependant.is_async_gen_callable: - - async def _async_stream_jsonl() -> AsyncIterator[bytes]: - async for item in gen: - yield _serialize_item(item) - # To allow for cancellation to trigger - # Ref: https://github.com/fastapi/fastapi/issues/14680 - await anyio.sleep(0) - - jsonl_stream_content: AsyncIterator[bytes] | Iterator[bytes] = ( - _async_stream_jsonl() - ) - else: - - def _sync_stream_jsonl() -> Iterator[bytes]: - for item in gen: # ty: ignore[not-iterable] - yield _serialize_item(item) - - jsonl_stream_content = _sync_stream_jsonl() - - response = StreamingResponse( - jsonl_stream_content, - media_type="application/jsonl", - background=solved_result.background_tasks, - ) - response.headers.raw.extend(solved_result.response.headers.raw) - elif dependant.is_async_gen_callable or dependant.is_gen_callable: - # Raw streaming with explicit response_class (e.g. StreamingResponse) - gen = dependant.call(**solved_result.values) - if dependant.is_async_gen_callable: - - async def _async_stream_raw( - async_gen: AsyncIterator[Any], - ) -> AsyncIterator[Any]: - async for chunk in async_gen: - yield chunk - # To allow for cancellation to trigger - # Ref: https://github.com/fastapi/fastapi/issues/14680 - await anyio.sleep(0) - - gen = _async_stream_raw(gen) - response_args = _build_response_args( - status_code=status_code, solved_result=solved_result - ) - response = actual_response_class(content=gen, **response_args) - response.headers.raw.extend(solved_result.response.headers.raw) - else: - raw_response = await run_endpoint_function( - dependant=dependant, - values=solved_result.values, - is_coroutine=is_coroutine, - ) - if isinstance(raw_response, Response): - if raw_response.background is None: - raw_response.background = solved_result.background_tasks - response = raw_response - else: - response_args = _build_response_args( - status_code=status_code, solved_result=solved_result - ) - # Use the fast path (dump_json) when no custom response - # class was set and a response field with a TypeAdapter - # exists. Serializes directly to JSON bytes via Pydantic's - # Rust core, skipping the intermediate Python dict + - # json.dumps() step. - use_dump_json = response_field is not None and isinstance( - response_class, DefaultPlaceholder - ) - content = await serialize_response( - field=response_field, - response_content=raw_response, - include=response_model_include, - exclude=response_model_exclude, - by_alias=response_model_by_alias, - exclude_unset=response_model_exclude_unset, - exclude_defaults=response_model_exclude_defaults, - exclude_none=response_model_exclude_none, - is_coroutine=is_coroutine, - endpoint_ctx=endpoint_ctx, - dump_json=use_dump_json, - ) - if use_dump_json: - response = Response( - content=content, - media_type="application/json", - **response_args, - ) - else: - response = actual_response_class(content, **response_args) - if not is_body_allowed_for_status_code(response.status_code): - response.body = b"" - response.headers.raw.extend(solved_result.response.headers.raw) - if errors: - validation_error = RequestValidationError( - errors, body=body, endpoint_ctx=endpoint_ctx - ) - raise validation_error - - # Return response - assert response - return response - - return app - - -def get_websocket_app( - dependant: Dependant, - dependency_overrides_provider: Any | None = None, - embed_body_fields: bool = False, -) -> Callable[[WebSocket], Coroutine[Any, Any, Any]]: - async def app(websocket: WebSocket) -> None: - endpoint_ctx = ( - _extract_endpoint_context(dependant.call) - if dependant.call - else EndpointContext() - ) - if dependant.path: - # For mounted sub-apps, include the mount path prefix - mount_path = websocket.scope.get("root_path", "").rstrip("/") - endpoint_ctx["path"] = f"WS {mount_path}{dependant.path}" - async_exit_stack = websocket.scope.get("fastapi_inner_astack") - assert isinstance(async_exit_stack, AsyncExitStack), ( - "fastapi_inner_astack not found in request scope" - ) - solved_result = await solve_dependencies( - request=websocket, - dependant=dependant, - dependency_overrides_provider=dependency_overrides_provider, - async_exit_stack=async_exit_stack, - embed_body_fields=embed_body_fields, - ) - if solved_result.errors: - raise WebSocketRequestValidationError( - solved_result.errors, - endpoint_ctx=endpoint_ctx, - ) - assert dependant.call is not None, "dependant.call must be a function" - await dependant.call(**solved_result.values) - - return app - - -class APIWebSocketRoute(routing.WebSocketRoute): - def __init__( - self, - path: str, - endpoint: Callable[..., Any], - *, - name: str | None = None, - dependencies: Sequence[params.Depends] | None = None, - dependency_overrides_provider: Any | None = None, - ) -> None: - self.path = path - self.endpoint = endpoint - self.name = get_name(endpoint) if name is None else name - self.dependencies = list(dependencies or []) - self.path_regex, self.path_format, self.param_convertors = compile_path(path) - self.dependant = get_dependant( - path=self.path_format, call=self.endpoint, scope="function" - ) - for depends in self.dependencies[::-1]: - self.dependant.dependencies.insert( - 0, - get_parameterless_sub_dependant(depends=depends, path=self.path_format), - ) - self._flat_dependant = get_flat_dependant(self.dependant) - self._embed_body_fields = _should_embed_body_fields( - self._flat_dependant.body_params - ) - self.app = websocket_session( - get_websocket_app( - dependant=self.dependant, - dependency_overrides_provider=dependency_overrides_provider, - embed_body_fields=self._embed_body_fields, - ) - ) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - match, child_scope = super().matches(scope) - if match != Match.NONE: - child_scope["route"] = self - return match, child_scope - - -class APIRoute(routing.Route): - def __init__( - self, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - name: str | None = None, - methods: set[str] | list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - dependency_overrides_provider: Any | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[["APIRoute"], str] - | DefaultPlaceholder = Default(generate_unique_id), - strict_content_type: bool | DefaultPlaceholder = Default(True), - ) -> None: - self.path = path - self.endpoint = endpoint - self.stream_item_type: Any | None = None - if isinstance(response_model, DefaultPlaceholder): - return_annotation = get_typed_return_annotation(endpoint) - if lenient_issubclass(return_annotation, Response): - response_model = None - else: - stream_item = get_stream_item_type(return_annotation) - if stream_item is not None: - # Extract item type for JSONL or SSE streaming when - # response_class is DefaultPlaceholder (JSONL) or - # EventSourceResponse (SSE). - # ServerSentEvent is excluded: it's a transport - # wrapper, not a data model, so it shouldn't feed - # into validation or OpenAPI schema generation. - if ( - isinstance(response_class, DefaultPlaceholder) - or lenient_issubclass(response_class, EventSourceResponse) - ) and not lenient_issubclass(stream_item, ServerSentEvent): - self.stream_item_type = stream_item - response_model = None - else: - response_model = return_annotation - self.response_model = response_model - self.summary = summary - self.response_description = response_description - self.deprecated = deprecated - self.operation_id = operation_id - self.response_model_include = response_model_include - self.response_model_exclude = response_model_exclude - self.response_model_by_alias = response_model_by_alias - self.response_model_exclude_unset = response_model_exclude_unset - self.response_model_exclude_defaults = response_model_exclude_defaults - self.response_model_exclude_none = response_model_exclude_none - self.include_in_schema = include_in_schema - self.response_class = response_class - self.dependency_overrides_provider = dependency_overrides_provider - self.callbacks = callbacks - self.openapi_extra = openapi_extra - self.generate_unique_id_function = generate_unique_id_function - self.strict_content_type = strict_content_type - self.tags = tags or [] - self.responses = responses or {} - self.name = get_name(endpoint) if name is None else name - self.path_regex, self.path_format, self.param_convertors = compile_path(path) - if methods is None: - methods = ["GET"] - self.methods: set[str] = {method.upper() for method in methods} - if isinstance(generate_unique_id_function, DefaultPlaceholder): - current_generate_unique_id: Callable[[APIRoute], str] = ( - generate_unique_id_function.value - ) - else: - current_generate_unique_id = generate_unique_id_function - self.unique_id = self.operation_id or current_generate_unique_id(self) - # normalize enums e.g. http.HTTPStatus - if isinstance(status_code, IntEnum): - status_code = int(status_code) - self.status_code = status_code - if self.response_model: - assert is_body_allowed_for_status_code(status_code), ( - f"Status code {status_code} must not have a response body" - ) - response_name = "Response_" + self.unique_id - self.response_field = create_model_field( - name=response_name, - type_=self.response_model, - mode="serialization", - ) - else: - self.response_field = None # type: ignore[assignment] - if self.stream_item_type: - stream_item_name = "StreamItem_" + self.unique_id - self.stream_item_field: ModelField | None = create_model_field( - name=stream_item_name, - type_=self.stream_item_type, - mode="serialization", - ) - else: - self.stream_item_field = None - self.dependencies = list(dependencies or []) - self.description = description or inspect.cleandoc(self.endpoint.__doc__ or "") - # if a "form feed" character (page break) is found in the description text, - # truncate description text to the content preceding the first "form feed" - self.description = self.description.split("\f")[0].strip() - response_fields = {} - for additional_status_code, response in self.responses.items(): - assert isinstance(response, dict), "An additional response must be a dict" - model = response.get("model") - if model: - assert is_body_allowed_for_status_code(additional_status_code), ( - f"Status code {additional_status_code} must not have a response body" - ) - response_name = f"Response_{additional_status_code}_{self.unique_id}" - response_field = create_model_field( - name=response_name, type_=model, mode="serialization" - ) - response_fields[additional_status_code] = response_field - if response_fields: - self.response_fields: dict[int | str, ModelField] = response_fields - else: - self.response_fields = {} - - assert callable(endpoint), "An endpoint must be a callable" - self.dependant = get_dependant( - path=self.path_format, call=self.endpoint, scope="function" - ) - for depends in self.dependencies[::-1]: - self.dependant.dependencies.insert( - 0, - get_parameterless_sub_dependant(depends=depends, path=self.path_format), - ) - self._flat_dependant = get_flat_dependant(self.dependant) - self._embed_body_fields = _should_embed_body_fields( - self._flat_dependant.body_params - ) - self.body_field = get_body_field( - flat_dependant=self._flat_dependant, - name=self.unique_id, - embed_body_fields=self._embed_body_fields, - ) - # Detect generator endpoints that should stream as JSONL or SSE - is_generator = ( - self.dependant.is_async_gen_callable or self.dependant.is_gen_callable - ) - self.is_sse_stream = is_generator and lenient_issubclass( - response_class, EventSourceResponse - ) - self.is_json_stream = is_generator and isinstance( - response_class, DefaultPlaceholder - ) - self.app = request_response(self.get_route_handler()) - - def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]: - return get_request_handler( - dependant=self.dependant, - body_field=self.body_field, - status_code=self.status_code, - response_class=self.response_class, - response_field=self.response_field, - response_model_include=self.response_model_include, - response_model_exclude=self.response_model_exclude, - response_model_by_alias=self.response_model_by_alias, - response_model_exclude_unset=self.response_model_exclude_unset, - response_model_exclude_defaults=self.response_model_exclude_defaults, - response_model_exclude_none=self.response_model_exclude_none, - dependency_overrides_provider=self.dependency_overrides_provider, - embed_body_fields=self._embed_body_fields, - strict_content_type=self.strict_content_type, - stream_item_field=self.stream_item_field, - is_json_stream=self.is_json_stream, - ) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - match, child_scope = super().matches(scope) - if match != Match.NONE: - child_scope["route"] = self - return match, child_scope - - -class APIRouter(routing.Router): - """ - `APIRouter` class, used to group *path operations*, for example to structure - an app in multiple files. It would then be included in the `FastAPI` app, or - in another `APIRouter` (ultimately included in the app). - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/). - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - - @router.get("/users/", tags=["users"]) - async def read_users(): - return [{"username": "Rick"}, {"username": "Morty"}] - - - app.include_router(router) - ``` - """ - - def __init__( - self, - *, - prefix: Annotated[str, Doc("An optional path prefix for the router.")] = "", - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to all the *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to all the - *path operations* in this router. - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - default_response_class: Annotated[ - type[Response], - Doc( - """ - The default response class to be used. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - """ - ), - ] = Default(JSONResponse), - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - OpenAPI callbacks that should apply to all *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - routes: Annotated[ - list[BaseRoute] | None, - Doc( - """ - **Note**: you probably shouldn't use this parameter, it is inherited - from Starlette and supported for compatibility. - - --- - - A list of routes to serve incoming HTTP and WebSocket requests. - """ - ), - deprecated( - """ - You normally wouldn't use this parameter with FastAPI, it is inherited - from Starlette and supported for compatibility. - - In FastAPI, you normally would use the *path operation methods*, - like `router.get()`, `router.post()`, etc. - """ - ), - ] = None, - redirect_slashes: Annotated[ - bool, - Doc( - """ - Whether to detect and redirect slashes in URLs when the client doesn't - use the same format. - """ - ), - ] = True, - default: Annotated[ - ASGIApp | None, - Doc( - """ - Default function handler for this router. Used to handle - 404 Not Found errors. - """ - ), - ] = None, - dependency_overrides_provider: Annotated[ - Any | None, - Doc( - """ - Only used internally by FastAPI to handle dependency overrides. - - You shouldn't need to use it. It normally points to the `FastAPI` app - object. - """ - ), - ] = None, - route_class: Annotated[ - type[APIRoute], - Doc( - """ - Custom route (*path operation*) class to be used by this router. - - Read more about it in the - [FastAPI docs for Custom Request and APIRoute class](https://fastapi.tiangolo.com/how-to/custom-request-and-route/#custom-apiroute-class-in-a-router). - """ - ), - ] = APIRoute, - on_startup: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of startup event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - on_shutdown: Annotated[ - Sequence[Callable[[], Any]] | None, - Doc( - """ - A list of shutdown event handler functions. - - You should instead use the `lifespan` handlers. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - # the generic to Lifespan[AppType] is the type of the top level application - # which the router cannot know statically, so we use typing.Any - lifespan: Annotated[ - Lifespan[Any] | None, - Doc( - """ - A `Lifespan` context manager handler. This replaces `startup` and - `shutdown` functions with a single context manager. - - Read more in the - [FastAPI docs for `lifespan`](https://fastapi.tiangolo.com/advanced/events/). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all *path operations* in this router as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - To include (or not) all the *path operations* in this router in the - generated OpenAPI. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - strict_content_type: Annotated[ - bool, - Doc( - """ - Enable strict checking for request Content-Type headers. - - When `True` (the default), requests with a body that do not include - a `Content-Type` header will **not** be parsed as JSON. - - This prevents potential cross-site request forgery (CSRF) attacks - that exploit the browser's ability to send requests without a - Content-Type header, bypassing CORS preflight checks. In particular - applicable for apps that need to be run locally (in localhost). - - When `False`, requests without a `Content-Type` header will have - their body parsed as JSON, which maintains compatibility with - certain clients that don't send `Content-Type` headers. - - Read more about it in the - [FastAPI docs for Strict Content-Type](https://fastapi.tiangolo.com/advanced/strict-content-type/). - """ - ), - ] = Default(True), - ) -> None: - # Determine the lifespan context to use - if lifespan is None: - # Use the default lifespan that runs on_startup/on_shutdown handlers - lifespan_context: Lifespan[Any] = _DefaultLifespan(self) - elif inspect.isasyncgenfunction(lifespan): - lifespan_context = asynccontextmanager(lifespan) - elif inspect.isgeneratorfunction(lifespan): - lifespan_context = _wrap_gen_lifespan_context(lifespan) - else: - lifespan_context = lifespan - self.lifespan_context = lifespan_context - - super().__init__( - routes=routes, - redirect_slashes=redirect_slashes, - default=default, - lifespan=lifespan_context, - ) - if prefix: - assert prefix.startswith("/"), "A path prefix must start with '/'" - assert not prefix.endswith("/"), ( - "A path prefix must not end with '/', as the routes will start with '/'" - ) - - # Handle on_startup/on_shutdown locally since Starlette removed support - # Ref: https://github.com/Kludex/starlette/pull/3117 - # TODO: deprecate this once the lifespan (or alternative) interface is improved - self.on_startup: list[Callable[[], Any]] = ( - [] if on_startup is None else list(on_startup) - ) - self.on_shutdown: list[Callable[[], Any]] = ( - [] if on_shutdown is None else list(on_shutdown) - ) - - self.prefix = prefix - self.tags: list[str | Enum] = tags or [] - self.dependencies = list(dependencies or []) - self.deprecated = deprecated - self.include_in_schema = include_in_schema - self.responses = responses or {} - self.callbacks = callbacks or [] - self.dependency_overrides_provider = dependency_overrides_provider - self.route_class = route_class - self.default_response_class = default_response_class - self.generate_unique_id_function = generate_unique_id_function - self.strict_content_type = strict_content_type - - def route( - self, - path: str, - methods: Collection[str] | None = None, - name: str | None = None, - include_in_schema: bool = True, - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_route( - path, - func, - methods=methods, - name=name, - include_in_schema=include_in_schema, - ) - return func - - return decorator - - def add_api_route( - self, - path: str, - endpoint: Callable[..., Any], - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: set[str] | list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] | DefaultPlaceholder = Default(JSONResponse), - name: str | None = None, - route_class_override: type[APIRoute] | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[APIRoute], str] - | DefaultPlaceholder = Default(generate_unique_id), - strict_content_type: bool | DefaultPlaceholder = Default(True), - ) -> None: - route_class = route_class_override or self.route_class - responses = responses or {} - combined_responses = {**self.responses, **responses} - current_response_class = get_value_or_default( - response_class, self.default_response_class - ) - current_tags = self.tags.copy() - if tags: - current_tags.extend(tags) - current_dependencies = self.dependencies.copy() - if dependencies: - current_dependencies.extend(dependencies) - current_callbacks = self.callbacks.copy() - if callbacks: - current_callbacks.extend(callbacks) - current_generate_unique_id = get_value_or_default( - generate_unique_id_function, self.generate_unique_id_function - ) - route = route_class( - self.prefix + path, - endpoint=endpoint, - response_model=response_model, - status_code=status_code, - tags=current_tags, - dependencies=current_dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=combined_responses, - deprecated=deprecated or self.deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema and self.include_in_schema, - response_class=current_response_class, - name=name, - dependency_overrides_provider=self.dependency_overrides_provider, - callbacks=current_callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=current_generate_unique_id, - strict_content_type=get_value_or_default( - strict_content_type, self.strict_content_type - ), - ) - self.routes.append(route) - - def api_route( - self, - path: str, - *, - response_model: Any = Default(None), - status_code: int | None = None, - tags: list[str | Enum] | None = None, - dependencies: Sequence[params.Depends] | None = None, - summary: str | None = None, - description: str | None = None, - response_description: str = "Successful Response", - responses: dict[int | str, dict[str, Any]] | None = None, - deprecated: bool | None = None, - methods: list[str] | None = None, - operation_id: str | None = None, - response_model_include: IncEx | None = None, - response_model_exclude: IncEx | None = None, - response_model_by_alias: bool = True, - response_model_exclude_unset: bool = False, - response_model_exclude_defaults: bool = False, - response_model_exclude_none: bool = False, - include_in_schema: bool = True, - response_class: type[Response] = Default(JSONResponse), - name: str | None = None, - callbacks: list[BaseRoute] | None = None, - openapi_extra: dict[str, Any] | None = None, - generate_unique_id_function: Callable[[APIRoute], str] = Default( - generate_unique_id - ), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_api_route( - path, - func, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=methods, - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - return func - - return decorator - - def add_api_websocket_route( - self, - path: str, - endpoint: Callable[..., Any], - name: str | None = None, - *, - dependencies: Sequence[params.Depends] | None = None, - ) -> None: - current_dependencies = self.dependencies.copy() - if dependencies: - current_dependencies.extend(dependencies) - - route = APIWebSocketRoute( - self.prefix + path, - endpoint=endpoint, - name=name, - dependencies=current_dependencies, - dependency_overrides_provider=self.dependency_overrides_provider, - ) - self.routes.append(route) - - def websocket( - self, - path: Annotated[ - str, - Doc( - """ - WebSocket path. - """ - ), - ], - name: Annotated[ - str | None, - Doc( - """ - A name for the WebSocket. Only used internally. - """ - ), - ] = None, - *, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be used for this - WebSocket. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - """ - ), - ] = None, - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Decorate a WebSocket function. - - Read more about it in the - [FastAPI docs for WebSockets](https://fastapi.tiangolo.com/advanced/websockets/). - - **Example** - - ## Example - - ```python - from fastapi import APIRouter, FastAPI, WebSocket - - app = FastAPI() - router = APIRouter() - - @router.websocket("/ws") - async def websocket_endpoint(websocket: WebSocket): - await websocket.accept() - while True: - data = await websocket.receive_text() - await websocket.send_text(f"Message text was: {data}") - - app.include_router(router) - ``` - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_api_websocket_route( - path, func, name=name, dependencies=dependencies - ) - return func - - return decorator - - def websocket_route( - self, path: str, name: str | None = None - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_websocket_route(path, func, name=name) - return func - - return decorator - - def include_router( - self, - router: Annotated["APIRouter", Doc("The `APIRouter` to include.")], - *, - prefix: Annotated[str, Doc("An optional path prefix for the router.")] = "", - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to all the *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to all the - *path operations* in this router. - - Read more about it in the - [FastAPI docs for Bigger Applications - Multiple Files](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - default_response_class: Annotated[ - type[Response], - Doc( - """ - The default response class to be used. - - Read more in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#default-response-class). - """ - ), - ] = Default(JSONResponse), - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses to be shown in OpenAPI. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Additional Responses in OpenAPI](https://fastapi.tiangolo.com/advanced/additional-responses/). - - And in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/#include-an-apirouter-with-a-custom-prefix-tags-responses-and-dependencies). - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - OpenAPI callbacks that should apply to all *path operations* in this - router. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark all *path operations* in this router as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include (or not) all the *path operations* in this router in the - generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = True, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> None: - """ - Include another `APIRouter` in the same current `APIRouter`. - - Read more about it in the - [FastAPI docs for Bigger Applications](https://fastapi.tiangolo.com/tutorial/bigger-applications/). - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - internal_router = APIRouter() - users_router = APIRouter() - - @users_router.get("/users/") - def read_users(): - return [{"name": "Rick"}, {"name": "Morty"}] - - internal_router.include_router(users_router) - app.include_router(internal_router) - ``` - """ - assert self is not router, ( - "Cannot include the same APIRouter instance into itself. " - "Did you mean to include a different router?" - ) - if prefix: - assert prefix.startswith("/"), "A path prefix must start with '/'" - assert not prefix.endswith("/"), ( - "A path prefix must not end with '/', as the routes will start with '/'" - ) - else: - for r in router.routes: - path = getattr(r, "path") # noqa: B009 - name = getattr(r, "name", "unknown") - if path is not None and not path: - raise FastAPIError( - f"Prefix and path cannot be both empty (path operation: {name})" - ) - if responses is None: - responses = {} - for route in router.routes: - if isinstance(route, APIRoute): - combined_responses = {**responses, **route.responses} - use_response_class = get_value_or_default( - route.response_class, - router.default_response_class, - default_response_class, - self.default_response_class, - ) - current_tags = [] - if tags: - current_tags.extend(tags) - if route.tags: - current_tags.extend(route.tags) - current_dependencies: list[params.Depends] = [] - if dependencies: - current_dependencies.extend(dependencies) - if route.dependencies: - current_dependencies.extend(route.dependencies) - current_callbacks = [] - if callbacks: - current_callbacks.extend(callbacks) - if route.callbacks: - current_callbacks.extend(route.callbacks) - current_generate_unique_id = get_value_or_default( - route.generate_unique_id_function, - router.generate_unique_id_function, - generate_unique_id_function, - self.generate_unique_id_function, - ) - self.add_api_route( - prefix + route.path, - route.endpoint, - response_model=route.response_model, - status_code=route.status_code, - tags=current_tags, - dependencies=current_dependencies, - summary=route.summary, - description=route.description, - response_description=route.response_description, - responses=combined_responses, - deprecated=route.deprecated or deprecated or self.deprecated, - methods=route.methods, - operation_id=route.operation_id, - response_model_include=route.response_model_include, - response_model_exclude=route.response_model_exclude, - response_model_by_alias=route.response_model_by_alias, - response_model_exclude_unset=route.response_model_exclude_unset, - response_model_exclude_defaults=route.response_model_exclude_defaults, - response_model_exclude_none=route.response_model_exclude_none, - include_in_schema=route.include_in_schema - and self.include_in_schema - and include_in_schema, - response_class=use_response_class, - name=route.name, - route_class_override=type(route), - callbacks=current_callbacks, - openapi_extra=route.openapi_extra, - generate_unique_id_function=current_generate_unique_id, - strict_content_type=get_value_or_default( - route.strict_content_type, - router.strict_content_type, - self.strict_content_type, - ), - ) - elif isinstance(route, routing.Route): - methods = list(route.methods or []) - self.add_route( - prefix + route.path, - route.endpoint, - methods=methods, - include_in_schema=route.include_in_schema, - name=route.name, - ) - elif isinstance(route, APIWebSocketRoute): - current_dependencies = [] - if dependencies: - current_dependencies.extend(dependencies) - if route.dependencies: - current_dependencies.extend(route.dependencies) - self.add_api_websocket_route( - prefix + route.path, - route.endpoint, - dependencies=current_dependencies, - name=route.name, - ) - elif isinstance(route, routing.WebSocketRoute): - self.add_websocket_route( - prefix + route.path, route.endpoint, name=route.name - ) - for handler in router.on_startup: - self.add_event_handler("startup", handler) - for handler in router.on_shutdown: - self.add_event_handler("shutdown", handler) - self.lifespan_context = _merge_lifespan_context( - self.lifespan_context, - router.lifespan_context, - ) - - def get( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP GET operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - @router.get("/items/") - def read_items(): - return [{"name": "Empanada"}, {"name": "Arepa"}] - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["GET"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def put( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PUT operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.put("/items/{item_id}") - def replace_item(item_id: str, item: Item): - return {"message": "Item replaced", "id": item_id} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["PUT"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def post( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP POST operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.post("/items/") - def create_item(item: Item): - return {"message": "Item created"} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["POST"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def delete( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP DELETE operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - @router.delete("/items/{item_id}") - def delete_item(item_id: str): - return {"message": "Item deleted"} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["DELETE"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def options( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP OPTIONS operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - - app = FastAPI() - router = APIRouter() - - @router.options("/items/") - def get_item_options(): - return {"additions": ["Aji", "Guacamole"]} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["OPTIONS"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def head( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP HEAD operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.head("/items/", status_code=204) - def get_items_headers(response: Response): - response.headers["X-Cat-Dog"] = "Alone in the world" - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["HEAD"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def patch( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP PATCH operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.patch("/items/") - def update_item(item: Item): - return {"message": "Item updated in place"} - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["PATCH"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - def trace( - self, - path: Annotated[ - str, - Doc( - """ - The URL path to be used for this *path operation*. - - For example, in `http://example.com/items`, the path is `/items`. - """ - ), - ], - *, - response_model: Annotated[ - Any, - Doc( - """ - The type to use for the response. - - It could be any valid Pydantic *field* type. So, it doesn't have to - be a Pydantic model, it could be other things, like a `list`, `dict`, - etc. - - It will be used for: - - * Documentation: the generated OpenAPI (and the UI at `/docs`) will - show it as the response (JSON Schema). - * Serialization: you could return an arbitrary object and the - `response_model` would be used to serialize that object into the - corresponding JSON. - * Filtering: the JSON sent to the client will only contain the data - (fields) defined in the `response_model`. If you returned an object - that contains an attribute `password` but the `response_model` does - not include that field, the JSON sent to the client would not have - that `password`. - * Validation: whatever you return will be serialized with the - `response_model`, converting any data as necessary to generate the - corresponding JSON. But if the data in the object returned is not - valid, that would mean a violation of the contract with the client, - so it's an error from the API developer. So, FastAPI will raise an - error and return a 500 error code (Internal Server Error). - - Read more about it in the - [FastAPI docs for Response Model](https://fastapi.tiangolo.com/tutorial/response-model/). - """ - ), - ] = Default(None), - status_code: Annotated[ - int | None, - Doc( - """ - The default status code to be used for the response. - - You could override the status code by returning a response directly. - - Read more about it in the - [FastAPI docs for Response Status Code](https://fastapi.tiangolo.com/tutorial/response-status-code/). - """ - ), - ] = None, - tags: Annotated[ - list[str | Enum] | None, - Doc( - """ - A list of tags to be applied to the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/#tags). - """ - ), - ] = None, - dependencies: Annotated[ - Sequence[params.Depends] | None, - Doc( - """ - A list of dependencies (using `Depends()`) to be applied to the - *path operation*. - - Read more about it in the - [FastAPI docs for Dependencies in path operation decorators](https://fastapi.tiangolo.com/tutorial/dependencies/dependencies-in-path-operation-decorators/). - """ - ), - ] = None, - summary: Annotated[ - str | None, - Doc( - """ - A summary for the *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - A description for the *path operation*. - - If not provided, it will be extracted automatically from the docstring - of the *path operation function*. - - It can contain Markdown. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Path Operation Configuration](https://fastapi.tiangolo.com/tutorial/path-operation-configuration/). - """ - ), - ] = None, - response_description: Annotated[ - str, - Doc( - """ - The description for the default response. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = "Successful Response", - responses: Annotated[ - dict[int | str, dict[str, Any]] | None, - Doc( - """ - Additional responses that could be returned by this *path operation*. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - deprecated: Annotated[ - bool | None, - Doc( - """ - Mark this *path operation* as deprecated. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - operation_id: Annotated[ - str | None, - Doc( - """ - Custom operation ID to be used by this *path operation*. - - By default, it is generated automatically. - - If you provide a custom operation ID, you need to make sure it is - unique for the whole API. - - You can customize the - operation ID generation with the parameter - `generate_unique_id_function` in the `FastAPI` class. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = None, - response_model_include: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to include only certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_exclude: Annotated[ - IncEx | None, - Doc( - """ - Configuration passed to Pydantic to exclude certain fields in the - response data. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = None, - response_model_by_alias: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response model - should be serialized by alias when an alias is used. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_include-and-response_model_exclude). - """ - ), - ] = True, - response_model_exclude_unset: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that were not set and - have their default values. This is different from - `response_model_exclude_defaults` in that if the fields are set, - they will be included in the response, even if the value is the same - as the default. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_defaults: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data - should have all the fields, including the ones that have the same value - as the default. This is different from `response_model_exclude_unset` - in that if the fields are set but contain the same default values, - they will be excluded from the response. - - When `True`, default values are omitted from the response. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#use-the-response_model_exclude_unset-parameter). - """ - ), - ] = False, - response_model_exclude_none: Annotated[ - bool, - Doc( - """ - Configuration passed to Pydantic to define if the response data should - exclude fields set to `None`. - - This is much simpler (less smart) than `response_model_exclude_unset` - and `response_model_exclude_defaults`. You probably want to use one of - those two instead of this one, as those allow returning `None` values - when it makes sense. - - Read more about it in the - [FastAPI docs for Response Model - Return Type](https://fastapi.tiangolo.com/tutorial/response-model/#response_model_exclude_none). - """ - ), - ] = False, - include_in_schema: Annotated[ - bool, - Doc( - """ - Include this *path operation* in the generated OpenAPI schema. - - This affects the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for Query Parameters and String Validations](https://fastapi.tiangolo.com/tutorial/query-params-str-validations/#exclude-parameters-from-openapi). - """ - ), - ] = True, - response_class: Annotated[ - type[Response], - Doc( - """ - Response class to be used for this *path operation*. - - This will not be used if you return a response directly. - - Read more about it in the - [FastAPI docs for Custom Response - HTML, Stream, File, others](https://fastapi.tiangolo.com/advanced/custom-response/#redirectresponse). - """ - ), - ] = Default(JSONResponse), - name: Annotated[ - str | None, - Doc( - """ - Name for this *path operation*. Only used internally. - """ - ), - ] = None, - callbacks: Annotated[ - list[BaseRoute] | None, - Doc( - """ - List of *path operations* that will be used as OpenAPI callbacks. - - This is only for OpenAPI documentation, the callbacks won't be used - directly. - - It will be added to the generated OpenAPI (e.g. visible at `/docs`). - - Read more about it in the - [FastAPI docs for OpenAPI Callbacks](https://fastapi.tiangolo.com/advanced/openapi-callbacks/). - """ - ), - ] = None, - openapi_extra: Annotated[ - dict[str, Any] | None, - Doc( - """ - Extra metadata to be included in the OpenAPI schema for this *path - operation*. - - Read more about it in the - [FastAPI docs for Path Operation Advanced Configuration](https://fastapi.tiangolo.com/advanced/path-operation-advanced-configuration/#custom-openapi-path-operation-schema). - """ - ), - ] = None, - generate_unique_id_function: Annotated[ - Callable[[APIRoute], str], - Doc( - """ - Customize the function used to generate unique IDs for the *path - operations* shown in the generated OpenAPI. - - This is particularly useful when automatically generating clients or - SDKs for your API. - - Read more about it in the - [FastAPI docs about how to Generate Clients](https://fastapi.tiangolo.com/advanced/generate-clients/#custom-generate-unique-id-function). - """ - ), - ] = Default(generate_unique_id), - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add a *path operation* using an HTTP TRACE operation. - - ## Example - - ```python - from fastapi import APIRouter, FastAPI - from pydantic import BaseModel - - class Item(BaseModel): - name: str - description: str | None = None - - app = FastAPI() - router = APIRouter() - - @router.trace("/items/{item_id}") - def trace_item(item_id: str): - return None - - app.include_router(router) - ``` - """ - return self.api_route( - path=path, - response_model=response_model, - status_code=status_code, - tags=tags, - dependencies=dependencies, - summary=summary, - description=description, - response_description=response_description, - responses=responses, - deprecated=deprecated, - methods=["TRACE"], - operation_id=operation_id, - response_model_include=response_model_include, - response_model_exclude=response_model_exclude, - response_model_by_alias=response_model_by_alias, - response_model_exclude_unset=response_model_exclude_unset, - response_model_exclude_defaults=response_model_exclude_defaults, - response_model_exclude_none=response_model_exclude_none, - include_in_schema=include_in_schema, - response_class=response_class, - name=name, - callbacks=callbacks, - openapi_extra=openapi_extra, - generate_unique_id_function=generate_unique_id_function, - ) - - # TODO: remove this once the lifespan (or alternative) interface is improved - async def _startup(self) -> None: - """ - Run any `.on_startup` event handlers. - - This method is kept for backward compatibility after Starlette removed - support for on_startup/on_shutdown handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - for handler in self.on_startup: - if is_async_callable(handler): - await handler() - else: - handler() - - # TODO: remove this once the lifespan (or alternative) interface is improved - async def _shutdown(self) -> None: - """ - Run any `.on_shutdown` event handlers. - - This method is kept for backward compatibility after Starlette removed - support for on_startup/on_shutdown handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - for handler in self.on_shutdown: - if is_async_callable(handler): - await handler() - else: - handler() - - # TODO: remove this once the lifespan (or alternative) interface is improved - def add_event_handler( - self, - event_type: str, - func: Callable[[], Any], - ) -> None: - """ - Add an event handler function for startup or shutdown. - - This method is kept for backward compatibility after Starlette removed - support for on_startup/on_shutdown handlers. - - Ref: https://github.com/Kludex/starlette/pull/3117 - """ - assert event_type in ("startup", "shutdown") - if event_type == "startup": - self.on_startup.append(func) - else: - self.on_shutdown.append(func) - - @deprecated( - """ - on_event is deprecated, use lifespan event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/). - """ - ) - def on_event( - self, - event_type: Annotated[ - str, - Doc( - """ - The type of event. `startup` or `shutdown`. - """ - ), - ], - ) -> Callable[[DecoratedCallable], DecoratedCallable]: - """ - Add an event handler for the router. - - `on_event` is deprecated, use `lifespan` event handlers instead. - - Read more about it in the - [FastAPI docs for Lifespan Events](https://fastapi.tiangolo.com/advanced/events/#alternative-events-deprecated). - """ - - def decorator(func: DecoratedCallable) -> DecoratedCallable: - self.add_event_handler(event_type, func) - return func - - return decorator diff --git a/functions/python/data-processor/fastapi/security/__init__.py b/functions/python/data-processor/fastapi/security/__init__.py deleted file mode 100644 index 3aa6bf21e..000000000 --- a/functions/python/data-processor/fastapi/security/__init__.py +++ /dev/null @@ -1,15 +0,0 @@ -from .api_key import APIKeyCookie as APIKeyCookie -from .api_key import APIKeyHeader as APIKeyHeader -from .api_key import APIKeyQuery as APIKeyQuery -from .http import HTTPAuthorizationCredentials as HTTPAuthorizationCredentials -from .http import HTTPBasic as HTTPBasic -from .http import HTTPBasicCredentials as HTTPBasicCredentials -from .http import HTTPBearer as HTTPBearer -from .http import HTTPDigest as HTTPDigest -from .oauth2 import OAuth2 as OAuth2 -from .oauth2 import OAuth2AuthorizationCodeBearer as OAuth2AuthorizationCodeBearer -from .oauth2 import OAuth2PasswordBearer as OAuth2PasswordBearer -from .oauth2 import OAuth2PasswordRequestForm as OAuth2PasswordRequestForm -from .oauth2 import OAuth2PasswordRequestFormStrict as OAuth2PasswordRequestFormStrict -from .oauth2 import SecurityScopes as SecurityScopes -from .open_id_connect_url import OpenIdConnect as OpenIdConnect diff --git a/functions/python/data-processor/fastapi/security/api_key.py b/functions/python/data-processor/fastapi/security/api_key.py deleted file mode 100644 index 83a4585a0..000000000 --- a/functions/python/data-processor/fastapi/security/api_key.py +++ /dev/null @@ -1,320 +0,0 @@ -from typing import Annotated - -from annotated_doc import Doc -from fastapi.openapi.models import APIKey, APIKeyIn -from fastapi.security.base import SecurityBase -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class APIKeyBase(SecurityBase): - model: APIKey - - def __init__( - self, - location: APIKeyIn, - name: str, - description: str | None, - scheme_name: str | None, - auto_error: bool, - ): - self.auto_error = auto_error - - self.model: APIKey = APIKey( - **{"in": location}, # ty: ignore[invalid-argument-type] - name=name, - description=description, - ) - self.scheme_name = scheme_name or self.__class__.__name__ - - def make_not_authenticated_error(self) -> HTTPException: - """ - The WWW-Authenticate header is not standardized for API Key authentication but - the HTTP specification requires that an error of 401 "Unauthorized" must - include a WWW-Authenticate header. - - Ref: https://datatracker.ietf.org/doc/html/rfc9110#name-401-unauthorized - - For this, this method sends a custom challenge `APIKey`. - """ - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers={"WWW-Authenticate": "APIKey"}, - ) - - def check_api_key(self, api_key: str | None) -> str | None: - if not api_key: - if self.auto_error: - raise self.make_not_authenticated_error() - return None - return api_key - - -class APIKeyQuery(APIKeyBase): - """ - API key authentication using a query parameter. - - This defines the name of the query parameter that should be provided in the request - with the API key and integrates that into the OpenAPI documentation. It extracts - the key value sent in the query parameter automatically and provides it as the - dependency result. But it doesn't define how to send that API key to the client. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be a string containing the key value. - - ## Example - - ```python - from fastapi import Depends, FastAPI - from fastapi.security import APIKeyQuery - - app = FastAPI() - - query_scheme = APIKeyQuery(name="api_key") - - - @app.get("/items/") - async def read_items(api_key: str = Depends(query_scheme)): - return {"api_key": api_key} - ``` - """ - - def __init__( - self, - *, - name: Annotated[ - str, - Doc("Query parameter name."), - ], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the query parameter is not provided, `APIKeyQuery` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the query parameter is not - available, instead of erroring out, the dependency result will be - `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in a query - parameter or in an HTTP Bearer token). - """ - ), - ] = True, - ): - super().__init__( - location=APIKeyIn.query, - name=name, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - api_key = request.query_params.get(self.model.name) - return self.check_api_key(api_key) - - -class APIKeyHeader(APIKeyBase): - """ - API key authentication using a header. - - This defines the name of the header that should be provided in the request with - the API key and integrates that into the OpenAPI documentation. It extracts - the key value sent in the header automatically and provides it as the dependency - result. But it doesn't define how to send that key to the client. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be a string containing the key value. - - ## Example - - ```python - from fastapi import Depends, FastAPI - from fastapi.security import APIKeyHeader - - app = FastAPI() - - header_scheme = APIKeyHeader(name="x-key") - - - @app.get("/items/") - async def read_items(key: str = Depends(header_scheme)): - return {"key": key} - ``` - """ - - def __init__( - self, - *, - name: Annotated[str, Doc("Header name.")], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the header is not provided, `APIKeyHeader` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the header is not available, - instead of erroring out, the dependency result will be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in a header or - in an HTTP Bearer token). - """ - ), - ] = True, - ): - super().__init__( - location=APIKeyIn.header, - name=name, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - api_key = request.headers.get(self.model.name) - return self.check_api_key(api_key) - - -class APIKeyCookie(APIKeyBase): - """ - API key authentication using a cookie. - - This defines the name of the cookie that should be provided in the request with - the API key and integrates that into the OpenAPI documentation. It extracts - the key value sent in the cookie automatically and provides it as the dependency - result. But it doesn't define how to set that cookie. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be a string containing the key value. - - ## Example - - ```python - from fastapi import Depends, FastAPI - from fastapi.security import APIKeyCookie - - app = FastAPI() - - cookie_scheme = APIKeyCookie(name="session") - - - @app.get("/items/") - async def read_items(session: str = Depends(cookie_scheme)): - return {"session": session} - ``` - """ - - def __init__( - self, - *, - name: Annotated[str, Doc("Cookie name.")], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the cookie is not provided, `APIKeyCookie` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the cookie is not available, - instead of erroring out, the dependency result will be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in a cookie or - in an HTTP Bearer token). - """ - ), - ] = True, - ): - super().__init__( - location=APIKeyIn.cookie, - name=name, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - api_key = request.cookies.get(self.model.name) - return self.check_api_key(api_key) diff --git a/functions/python/data-processor/fastapi/security/base.py b/functions/python/data-processor/fastapi/security/base.py deleted file mode 100644 index c43555deb..000000000 --- a/functions/python/data-processor/fastapi/security/base.py +++ /dev/null @@ -1,6 +0,0 @@ -from fastapi.openapi.models import SecurityBase as SecurityBaseModel - - -class SecurityBase: - model: SecurityBaseModel - scheme_name: str diff --git a/functions/python/data-processor/fastapi/security/http.py b/functions/python/data-processor/fastapi/security/http.py deleted file mode 100644 index a32948ef0..000000000 --- a/functions/python/data-processor/fastapi/security/http.py +++ /dev/null @@ -1,417 +0,0 @@ -import binascii -from base64 import b64decode -from typing import Annotated - -from annotated_doc import Doc -from fastapi.exceptions import HTTPException -from fastapi.openapi.models import HTTPBase as HTTPBaseModel -from fastapi.openapi.models import HTTPBearer as HTTPBearerModel -from fastapi.security.base import SecurityBase -from fastapi.security.utils import get_authorization_scheme_param -from pydantic import BaseModel -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class HTTPBasicCredentials(BaseModel): - """ - The HTTP Basic credentials given as the result of using `HTTPBasic` in a - dependency. - - Read more about it in the - [FastAPI docs for HTTP Basic Auth](https://fastapi.tiangolo.com/advanced/security/http-basic-auth/). - """ - - username: Annotated[str, Doc("The HTTP Basic username.")] - password: Annotated[str, Doc("The HTTP Basic password.")] - - -class HTTPAuthorizationCredentials(BaseModel): - """ - The HTTP authorization credentials in the result of using `HTTPBearer` or - `HTTPDigest` in a dependency. - - The HTTP authorization header value is split by the first space. - - The first part is the `scheme`, the second part is the `credentials`. - - For example, in an HTTP Bearer token scheme, the client will send a header - like: - - ``` - Authorization: Bearer deadbeef12346 - ``` - - In this case: - - * `scheme` will have the value `"Bearer"` - * `credentials` will have the value `"deadbeef12346"` - """ - - scheme: Annotated[ - str, - Doc( - """ - The HTTP authorization scheme extracted from the header value. - """ - ), - ] - credentials: Annotated[ - str, - Doc( - """ - The HTTP authorization credentials extracted from the header value. - """ - ), - ] - - -class HTTPBase(SecurityBase): - model: HTTPBaseModel - - def __init__( - self, - *, - scheme: str, - scheme_name: str | None = None, - description: str | None = None, - auto_error: bool = True, - ): - self.model = HTTPBaseModel(scheme=scheme, description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - def make_authenticate_headers(self) -> dict[str, str]: - return {"WWW-Authenticate": f"{self.model.scheme.title()}"} - - def make_not_authenticated_error(self) -> HTTPException: - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers=self.make_authenticate_headers(), - ) - - async def __call__(self, request: Request) -> HTTPAuthorizationCredentials | None: - authorization = request.headers.get("Authorization") - scheme, credentials = get_authorization_scheme_param(authorization) - if not (authorization and scheme and credentials): - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return HTTPAuthorizationCredentials(scheme=scheme, credentials=credentials) - - -class HTTPBasic(HTTPBase): - """ - HTTP Basic authentication. - - Ref: https://datatracker.ietf.org/doc/html/rfc7617 - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be an `HTTPBasicCredentials` object containing the - `username` and the `password`. - - Read more about it in the - [FastAPI docs for HTTP Basic Auth](https://fastapi.tiangolo.com/advanced/security/http-basic-auth/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import HTTPBasic, HTTPBasicCredentials - - app = FastAPI() - - security = HTTPBasic() - - - @app.get("/users/me") - def read_current_user(credentials: Annotated[HTTPBasicCredentials, Depends(security)]): - return {"username": credentials.username, "password": credentials.password} - ``` - """ - - def __init__( - self, - *, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - realm: Annotated[ - str | None, - Doc( - """ - HTTP Basic authentication realm. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the HTTP Basic authentication is not provided (a - header), `HTTPBasic` will automatically cancel the request and send the - client an error. - - If `auto_error` is set to `False`, when the HTTP Basic authentication - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in HTTP Basic - authentication or in an HTTP Bearer token). - """ - ), - ] = True, - ): - self.model = HTTPBaseModel(scheme="basic", description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.realm = realm - self.auto_error = auto_error - - def make_authenticate_headers(self) -> dict[str, str]: - if self.realm: - return {"WWW-Authenticate": f'Basic realm="{self.realm}"'} - return {"WWW-Authenticate": "Basic"} - - async def __call__( # type: ignore - self, request: Request - ) -> HTTPBasicCredentials | None: - authorization = request.headers.get("Authorization") - scheme, param = get_authorization_scheme_param(authorization) - if not authorization or scheme.lower() != "basic": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - try: - data = b64decode(param).decode("ascii") - except (ValueError, UnicodeDecodeError, binascii.Error) as e: - raise self.make_not_authenticated_error() from e - username, separator, password = data.partition(":") - if not separator: - raise self.make_not_authenticated_error() - return HTTPBasicCredentials(username=username, password=password) - - -class HTTPBearer(HTTPBase): - """ - HTTP Bearer token authentication. - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be an `HTTPAuthorizationCredentials` object containing - the `scheme` and the `credentials`. - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer - - app = FastAPI() - - security = HTTPBearer() - - - @app.get("/users/me") - def read_current_user( - credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)] - ): - return {"scheme": credentials.scheme, "credentials": credentials.credentials} - ``` - """ - - def __init__( - self, - *, - bearerFormat: Annotated[str | None, Doc("Bearer token format.")] = None, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the HTTP Bearer token is not provided (in an - `Authorization` header), `HTTPBearer` will automatically cancel the - request and send the client an error. - - If `auto_error` is set to `False`, when the HTTP Bearer token - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in an HTTP - Bearer token or in a cookie). - """ - ), - ] = True, - ): - self.model = HTTPBearerModel(bearerFormat=bearerFormat, description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - async def __call__(self, request: Request) -> HTTPAuthorizationCredentials | None: - authorization = request.headers.get("Authorization") - scheme, credentials = get_authorization_scheme_param(authorization) - if not (authorization and scheme and credentials): - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - if scheme.lower() != "bearer": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return HTTPAuthorizationCredentials(scheme=scheme, credentials=credentials) - - -class HTTPDigest(HTTPBase): - """ - HTTP Digest authentication. - - **Warning**: this is only a stub to connect the components with OpenAPI in FastAPI, - but it doesn't implement the full Digest scheme, you would need to subclass it - and implement it in your code. - - Ref: https://datatracker.ietf.org/doc/html/rfc7616 - - ## Usage - - Create an instance object and use that object as the dependency in `Depends()`. - - The dependency result will be an `HTTPAuthorizationCredentials` object containing - the `scheme` and the `credentials`. - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import HTTPAuthorizationCredentials, HTTPDigest - - app = FastAPI() - - security = HTTPDigest() - - - @app.get("/users/me") - def read_current_user( - credentials: Annotated[HTTPAuthorizationCredentials, Depends(security)] - ): - return {"scheme": credentials.scheme, "credentials": credentials.credentials} - ``` - """ - - def __init__( - self, - *, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if the HTTP Digest is not provided, `HTTPDigest` will - automatically cancel the request and send the client an error. - - If `auto_error` is set to `False`, when the HTTP Digest is not - available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, in HTTP - Digest or in a cookie). - """ - ), - ] = True, - ): - self.model = HTTPBaseModel(scheme="digest", description=description) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - async def __call__(self, request: Request) -> HTTPAuthorizationCredentials | None: - authorization = request.headers.get("Authorization") - scheme, credentials = get_authorization_scheme_param(authorization) - if not (authorization and scheme and credentials): - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - if scheme.lower() != "digest": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return HTTPAuthorizationCredentials(scheme=scheme, credentials=credentials) diff --git a/functions/python/data-processor/fastapi/security/oauth2.py b/functions/python/data-processor/fastapi/security/oauth2.py deleted file mode 100644 index 3fd9e41eb..000000000 --- a/functions/python/data-processor/fastapi/security/oauth2.py +++ /dev/null @@ -1,693 +0,0 @@ -from typing import Annotated, Any, cast - -from annotated_doc import Doc -from fastapi.exceptions import HTTPException -from fastapi.openapi.models import OAuth2 as OAuth2Model -from fastapi.openapi.models import OAuthFlows as OAuthFlowsModel -from fastapi.param_functions import Form -from fastapi.security.base import SecurityBase -from fastapi.security.utils import get_authorization_scheme_param -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class OAuth2PasswordRequestForm: - """ - This is a dependency class to collect the `username` and `password` as form data - for an OAuth2 password flow. - - The OAuth2 specification dictates that for a password flow the data should be - collected using form data (instead of JSON) and that it should have the specific - fields `username` and `password`. - - All the initialization parameters are extracted from the request. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import OAuth2PasswordRequestForm - - app = FastAPI() - - - @app.post("/login") - def login(form_data: Annotated[OAuth2PasswordRequestForm, Depends()]): - data = {} - data["scopes"] = [] - for scope in form_data.scopes: - data["scopes"].append(scope) - if form_data.client_id: - data["client_id"] = form_data.client_id - if form_data.client_secret: - data["client_secret"] = form_data.client_secret - return data - ``` - - Note that for OAuth2 the scope `items:read` is a single scope in an opaque string. - You could have custom internal logic to separate it by colon characters (`:`) or - similar, and get the two parts `items` and `read`. Many applications do that to - group and organize permissions, you could do it as well in your application, just - know that it is application specific, it's not part of the specification. - """ - - def __init__( - self, - *, - grant_type: Annotated[ - str | None, - Form(pattern="^password$"), - Doc( - """ - The OAuth2 spec says it is required and MUST be the fixed string - "password". Nevertheless, this dependency class is permissive and - allows not passing it. If you want to enforce it, use instead the - `OAuth2PasswordRequestFormStrict` dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = None, - username: Annotated[ - str, - Form(), - Doc( - """ - `username` string. The OAuth2 spec requires the exact field name - `username`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - password: Annotated[ - str, - Form(json_schema_extra={"format": "password"}), - Doc( - """ - `password` string. The OAuth2 spec requires the exact field name - `password`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - scope: Annotated[ - str, - Form(), - Doc( - """ - A single string with actually several scopes separated by spaces. Each - scope is also a string. - - For example, a single string with: - - ```python - "items:read items:write users:read profile openid" - ```` - - would represent the scopes: - - * `items:read` - * `items:write` - * `users:read` - * `profile` - * `openid` - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = "", - client_id: Annotated[ - str | None, - Form(), - Doc( - """ - If there's a `client_id`, it can be sent as part of the form fields. - But the OAuth2 specification recommends sending the `client_id` and - `client_secret` (if any) using HTTP Basic auth. - """ - ), - ] = None, - client_secret: Annotated[ - str | None, - Form(json_schema_extra={"format": "password"}), - Doc( - """ - If there's a `client_secret` (and a `client_id`), they can be sent - as part of the form fields. But the OAuth2 specification recommends - sending the `client_id` and `client_secret` (if any) using HTTP Basic - auth. - """ - ), - ] = None, - ): - self.grant_type = grant_type - self.username = username - self.password = password - self.scopes = scope.split() - self.client_id = client_id - self.client_secret = client_secret - - -class OAuth2PasswordRequestFormStrict(OAuth2PasswordRequestForm): - """ - This is a dependency class to collect the `username` and `password` as form data - for an OAuth2 password flow. - - The OAuth2 specification dictates that for a password flow the data should be - collected using form data (instead of JSON) and that it should have the specific - fields `username` and `password`. - - All the initialization parameters are extracted from the request. - - The only difference between `OAuth2PasswordRequestFormStrict` and - `OAuth2PasswordRequestForm` is that `OAuth2PasswordRequestFormStrict` requires the - client to send the form field `grant_type` with the value `"password"`, which - is required in the OAuth2 specification (it seems that for no particular reason), - while for `OAuth2PasswordRequestForm` `grant_type` is optional. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - - ## Example - - ```python - from typing import Annotated - - from fastapi import Depends, FastAPI - from fastapi.security import OAuth2PasswordRequestForm - - app = FastAPI() - - - @app.post("/login") - def login(form_data: Annotated[OAuth2PasswordRequestFormStrict, Depends()]): - data = {} - data["scopes"] = [] - for scope in form_data.scopes: - data["scopes"].append(scope) - if form_data.client_id: - data["client_id"] = form_data.client_id - if form_data.client_secret: - data["client_secret"] = form_data.client_secret - return data - ``` - - Note that for OAuth2 the scope `items:read` is a single scope in an opaque string. - You could have custom internal logic to separate it by colon characters (`:`) or - similar, and get the two parts `items` and `read`. Many applications do that to - group and organize permissions, you could do it as well in your application, just - know that it is application specific, it's not part of the specification. - - - grant_type: the OAuth2 spec says it is required and MUST be the fixed string "password". - This dependency is strict about it. If you want to be permissive, use instead the - OAuth2PasswordRequestForm dependency class. - username: username string. The OAuth2 spec requires the exact field name "username". - password: password string. The OAuth2 spec requires the exact field name "password". - scope: Optional string. Several scopes (each one a string) separated by spaces. E.g. - "items:read items:write users:read profile openid" - client_id: optional string. OAuth2 recommends sending the client_id and client_secret (if any) - using HTTP Basic auth, as: client_id:client_secret - client_secret: optional string. OAuth2 recommends sending the client_id and client_secret (if any) - using HTTP Basic auth, as: client_id:client_secret - """ - - def __init__( - self, - grant_type: Annotated[ - str, - Form(pattern="^password$"), - Doc( - """ - The OAuth2 spec says it is required and MUST be the fixed string - "password". This dependency is strict about it. If you want to be - permissive, use instead the `OAuth2PasswordRequestForm` dependency - class. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - username: Annotated[ - str, - Form(), - Doc( - """ - `username` string. The OAuth2 spec requires the exact field name - `username`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - password: Annotated[ - str, - Form(), - Doc( - """ - `password` string. The OAuth2 spec requires the exact field name - `password`. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - scope: Annotated[ - str, - Form(), - Doc( - """ - A single string with actually several scopes separated by spaces. Each - scope is also a string. - - For example, a single string with: - - ```python - "items:read items:write users:read profile openid" - ```` - - would represent the scopes: - - * `items:read` - * `items:write` - * `users:read` - * `profile` - * `openid` - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = "", - client_id: Annotated[ - str | None, - Form(), - Doc( - """ - If there's a `client_id`, it can be sent as part of the form fields. - But the OAuth2 specification recommends sending the `client_id` and - `client_secret` (if any) using HTTP Basic auth. - """ - ), - ] = None, - client_secret: Annotated[ - str | None, - Form(), - Doc( - """ - If there's a `client_secret` (and a `client_id`), they can be sent - as part of the form fields. But the OAuth2 specification recommends - sending the `client_id` and `client_secret` (if any) using HTTP Basic - auth. - """ - ), - ] = None, - ): - super().__init__( - grant_type=grant_type, - username=username, - password=password, - scope=scope, - client_id=client_id, - client_secret=client_secret, - ) - - -class OAuth2(SecurityBase): - """ - This is the base class for OAuth2 authentication, an instance of it would be used - as a dependency. All other OAuth2 classes inherit from it and customize it for - each OAuth2 flow. - - You normally would not create a new class inheriting from it but use one of the - existing subclasses, and maybe compose them if you want to support multiple flows. - - Read more about it in the - [FastAPI docs for Security](https://fastapi.tiangolo.com/tutorial/security/). - """ - - def __init__( - self, - *, - flows: Annotated[ - OAuthFlowsModel | dict[str, dict[str, Any]], - Doc( - """ - The dictionary of OAuth2 flows. - """ - ), - ] = OAuthFlowsModel(), - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OAuth2 authentication, it will automatically cancel the request and - send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OAuth2 - or in a cookie). - """ - ), - ] = True, - ): - self.model = OAuth2Model( - flows=cast(OAuthFlowsModel, flows), description=description - ) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - def make_not_authenticated_error(self) -> HTTPException: - """ - The OAuth 2 specification doesn't define the challenge that should be used, - because a `Bearer` token is not really the only option to authenticate. - - But declaring any other authentication challenge would be application-specific - as it's not defined in the specification. - - For practical reasons, this method uses the `Bearer` challenge by default, as - it's probably the most common one. - - If you are implementing an OAuth2 authentication scheme other than the provided - ones in FastAPI (based on bearer tokens), you might want to override this. - - Ref: https://datatracker.ietf.org/doc/html/rfc6749 - """ - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers={"WWW-Authenticate": "Bearer"}, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - if not authorization: - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return authorization - - -class OAuth2PasswordBearer(OAuth2): - """ - OAuth2 flow for authentication using a bearer token obtained with a password. - An instance of it would be used as a dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - - def __init__( - self, - tokenUrl: Annotated[ - str, - Doc( - """ - The URL to obtain the OAuth2 token. This would be the *path operation* - that has `OAuth2PasswordRequestForm` as a dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - scopes: Annotated[ - dict[str, str] | None, - Doc( - """ - The OAuth2 scopes that would be required by the *path operations* that - use this dependency. - - Read more about it in the - [FastAPI docs for Simple OAuth2 with Password and Bearer](https://fastapi.tiangolo.com/tutorial/security/simple-oauth2/). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OAuth2 authentication, it will automatically cancel the request and - send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OAuth2 - or in a cookie). - """ - ), - ] = True, - refreshUrl: Annotated[ - str | None, - Doc( - """ - The URL to refresh the token and obtain a new one. - """ - ), - ] = None, - ): - if not scopes: - scopes = {} - flows = OAuthFlowsModel( - password=cast( - Any, - { - "tokenUrl": tokenUrl, - "refreshUrl": refreshUrl, - "scopes": scopes, - }, - ) - ) - super().__init__( - flows=flows, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - scheme, param = get_authorization_scheme_param(authorization) - if not authorization or scheme.lower() != "bearer": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return param - - -class OAuth2AuthorizationCodeBearer(OAuth2): - """ - OAuth2 flow for authentication using a bearer token obtained with an OAuth2 code - flow. An instance of it would be used as a dependency. - """ - - def __init__( - self, - authorizationUrl: str, - tokenUrl: Annotated[ - str, - Doc( - """ - The URL to obtain the OAuth2 token. - """ - ), - ], - refreshUrl: Annotated[ - str | None, - Doc( - """ - The URL to refresh the token and obtain a new one. - """ - ), - ] = None, - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - scopes: Annotated[ - dict[str, str] | None, - Doc( - """ - The OAuth2 scopes that would be required by the *path operations* that - use this dependency. - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OAuth2 authentication, it will automatically cancel the request and - send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OAuth2 - or in a cookie). - """ - ), - ] = True, - ): - if not scopes: - scopes = {} - flows = OAuthFlowsModel( - authorizationCode=cast( - Any, - { - "authorizationUrl": authorizationUrl, - "tokenUrl": tokenUrl, - "refreshUrl": refreshUrl, - "scopes": scopes, - }, - ) - ) - super().__init__( - flows=flows, - scheme_name=scheme_name, - description=description, - auto_error=auto_error, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - scheme, param = get_authorization_scheme_param(authorization) - if not authorization or scheme.lower() != "bearer": - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None # pragma: nocover - return param - - -class SecurityScopes: - """ - This is a special class that you can define in a parameter in a dependency to - obtain the OAuth2 scopes required by all the dependencies in the same chain. - - This way, multiple dependencies can have different scopes, even when used in the - same *path operation*. And with this, you can access all the scopes required in - all those dependencies in a single place. - - Read more about it in the - [FastAPI docs for OAuth2 scopes](https://fastapi.tiangolo.com/advanced/security/oauth2-scopes/). - """ - - def __init__( - self, - scopes: Annotated[ - list[str] | None, - Doc( - """ - This will be filled by FastAPI. - """ - ), - ] = None, - ): - self.scopes: Annotated[ - list[str], - Doc( - """ - The list of all the scopes required by dependencies. - """ - ), - ] = scopes or [] - self.scope_str: Annotated[ - str, - Doc( - """ - All the scopes required by all the dependencies in a single string - separated by spaces, as defined in the OAuth2 specification. - """ - ), - ] = " ".join(self.scopes) diff --git a/functions/python/data-processor/fastapi/security/open_id_connect_url.py b/functions/python/data-processor/fastapi/security/open_id_connect_url.py deleted file mode 100644 index 125a81943..000000000 --- a/functions/python/data-processor/fastapi/security/open_id_connect_url.py +++ /dev/null @@ -1,94 +0,0 @@ -from typing import Annotated - -from annotated_doc import Doc -from fastapi.openapi.models import OpenIdConnect as OpenIdConnectModel -from fastapi.security.base import SecurityBase -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.status import HTTP_401_UNAUTHORIZED - - -class OpenIdConnect(SecurityBase): - """ - OpenID Connect authentication class. An instance of it would be used as a - dependency. - - **Warning**: this is only a stub to connect the components with OpenAPI in FastAPI, - but it doesn't implement the full OpenIdConnect scheme, for example, it doesn't use - the OpenIDConnect URL. You would need to subclass it and implement it in your - code. - """ - - def __init__( - self, - *, - openIdConnectUrl: Annotated[ - str, - Doc( - """ - The OpenID Connect URL. - """ - ), - ], - scheme_name: Annotated[ - str | None, - Doc( - """ - Security scheme name. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - description: Annotated[ - str | None, - Doc( - """ - Security scheme description. - - It will be included in the generated OpenAPI (e.g. visible at `/docs`). - """ - ), - ] = None, - auto_error: Annotated[ - bool, - Doc( - """ - By default, if no HTTP Authorization header is provided, required for - OpenID Connect authentication, it will automatically cancel the request - and send the client an error. - - If `auto_error` is set to `False`, when the HTTP Authorization header - is not available, instead of erroring out, the dependency result will - be `None`. - - This is useful when you want to have optional authentication. - - It is also useful when you want to have authentication that can be - provided in one of multiple optional ways (for example, with OpenID - Connect or in a cookie). - """ - ), - ] = True, - ): - self.model = OpenIdConnectModel( - openIdConnectUrl=openIdConnectUrl, description=description - ) - self.scheme_name = scheme_name or self.__class__.__name__ - self.auto_error = auto_error - - def make_not_authenticated_error(self) -> HTTPException: - return HTTPException( - status_code=HTTP_401_UNAUTHORIZED, - detail="Not authenticated", - headers={"WWW-Authenticate": "Bearer"}, - ) - - async def __call__(self, request: Request) -> str | None: - authorization = request.headers.get("Authorization") - if not authorization: - if self.auto_error: - raise self.make_not_authenticated_error() - else: - return None - return authorization diff --git a/functions/python/data-processor/fastapi/security/utils.py b/functions/python/data-processor/fastapi/security/utils.py deleted file mode 100644 index 8ee66fd38..000000000 --- a/functions/python/data-processor/fastapi/security/utils.py +++ /dev/null @@ -1,7 +0,0 @@ -def get_authorization_scheme_param( - authorization_header_value: str | None, -) -> tuple[str, str]: - if not authorization_header_value: - return "", "" - scheme, _, param = authorization_header_value.partition(" ") - return scheme, param.strip() diff --git a/functions/python/data-processor/fastapi/sse.py b/functions/python/data-processor/fastapi/sse.py deleted file mode 100644 index 1e2bd8617..000000000 --- a/functions/python/data-processor/fastapi/sse.py +++ /dev/null @@ -1,235 +0,0 @@ -from typing import Annotated, Any - -from annotated_doc import Doc -from pydantic import AfterValidator, BaseModel, Field, model_validator -from starlette.responses import StreamingResponse - -# Canonical SSE event schema matching the OpenAPI 3.2 spec -# (Section 4.14.4 "Special Considerations for Server-Sent Events") -_SSE_EVENT_SCHEMA: dict[str, Any] = { - "type": "object", - "properties": { - "data": {"type": "string"}, - "event": {"type": "string"}, - "id": {"type": "string"}, - "retry": {"type": "integer", "minimum": 0}, - }, -} - - -class EventSourceResponse(StreamingResponse): - """Streaming response with `text/event-stream` media type. - - Use as `response_class=EventSourceResponse` on a *path operation* that uses `yield` - to enable Server Sent Events (SSE) responses. - - Works with **any HTTP method** (`GET`, `POST`, etc.), which makes it compatible - with protocols like MCP that stream SSE over `POST`. - - The actual encoding logic lives in the FastAPI routing layer. This class - serves mainly as a marker and sets the correct `Content-Type`. - """ - - media_type = "text/event-stream" - - -def _check_single_line(v: str | None, field_name: str) -> str | None: - if v is not None and ("\r" in v or "\n" in v): - raise ValueError(f"SSE '{field_name}' must be a single line") - return v - - -def _check_event_single_line(v: str | None) -> str | None: - return _check_single_line(v, "event") - - -def _check_id_valid(v: str | None) -> str | None: - if v is not None and "\0" in v: - raise ValueError("SSE 'id' must not contain null characters") - return _check_single_line(v, "id") - - -class ServerSentEvent(BaseModel): - """Represents a single Server-Sent Event. - - When `yield`ed from a *path operation function* that uses - `response_class=EventSourceResponse`, each `ServerSentEvent` is encoded - into the [SSE wire format](https://html.spec.whatwg.org/multipage/server-sent-events.html#parsing-an-event-stream) - (`text/event-stream`). - - If you yield a plain object (dict, Pydantic model, etc.) instead, it is - automatically JSON-encoded and sent as the `data:` field. - - All `data` values **including plain strings** are JSON-serialized. - - For example, `data="hello"` produces `data: "hello"` on the wire (with - quotes). - """ - - data: Annotated[ - Any, - Doc( - """ - The event payload. - - Can be any JSON-serializable value: a Pydantic model, dict, list, - string, number, etc. It is **always** serialized to JSON: strings - are quoted (`"hello"` becomes `data: "hello"` on the wire). - - Mutually exclusive with `raw_data`. - """ - ), - ] = None - raw_data: Annotated[ - str | None, - Doc( - """ - Raw string to send as the `data:` field **without** JSON encoding. - - Use this when you need to send pre-formatted text, HTML fragments, - CSV lines, or any non-JSON payload. The string is placed directly - into the `data:` field as-is. - - Mutually exclusive with `data`. - """ - ), - ] = None - event: Annotated[ - str | None, - AfterValidator(_check_event_single_line), - Doc( - """ - Optional event type name. - - Maps to `addEventListener(event, ...)` on the browser. When omitted, - the browser dispatches on the generic `message` event. Must be a - single line. - """ - ), - ] = None - id: Annotated[ - str | None, - AfterValidator(_check_id_valid), - Doc( - """ - Optional event ID. - - The browser sends this value back as the `Last-Event-ID` header on - automatic reconnection. **Must be a single line** and must not contain - null (`\\0`) characters. - """ - ), - ] = None - retry: Annotated[ - int | None, - Field(ge=0), - Doc( - """ - Optional reconnection time in **milliseconds**. - - Tells the browser how long to wait before reconnecting after the - connection is lost. Must be a non-negative integer. - """ - ), - ] = None - comment: Annotated[ - str | None, - Doc( - """ - Optional comment line(s). - - Comment lines start with `:` in the SSE wire format and are ignored by - `EventSource` clients. Useful for keep-alive pings to prevent - proxy/load-balancer timeouts. - """ - ), - ] = None - - @model_validator(mode="after") - def _check_data_exclusive(self) -> "ServerSentEvent": - if self.data is not None and self.raw_data is not None: - raise ValueError( - "Cannot set both 'data' and 'raw_data' on the same " - "ServerSentEvent. Use 'data' for JSON-serialized payloads " - "or 'raw_data' for pre-formatted strings." - ) - return self - - -def format_sse_event( - *, - data_str: Annotated[ - str | None, - Doc( - """ - Pre-serialized data string to use as the `data:` field. - """ - ), - ] = None, - event: Annotated[ - str | None, - Doc( - """ - Optional event type name (`event:` field). - """ - ), - ] = None, - id: Annotated[ - str | None, - Doc( - """ - Optional event ID (`id:` field). - """ - ), - ] = None, - retry: Annotated[ - int | None, - Doc( - """ - Optional reconnection time in milliseconds (`retry:` field). - """ - ), - ] = None, - comment: Annotated[ - str | None, - Doc( - """ - Optional comment line(s) (`:` prefix). - """ - ), - ] = None, -) -> bytes: - """Build SSE wire-format bytes from **pre-serialized** data. - - The result always ends with `\n\n` (the event terminator). - """ - lines: list[str] = [] - - if comment is not None: - for line in comment.splitlines(): - lines.append(f": {line}") - - if event is not None: - lines.append(f"event: {event}") - - if data_str is not None: - for line in data_str.splitlines(): - lines.append(f"data: {line}") - - if id is not None: - lines.append(f"id: {id}") - - if retry is not None: - lines.append(f"retry: {retry}") - - lines.append("") - lines.append("") - return "\n".join(lines).encode("utf-8") - - -# Keep-alive comment, per the SSE spec recommendation -KEEPALIVE_COMMENT = b": ping\n\n" - -# Seconds between keep-alive pings when a generator is idle. -# Private but importable so tests can monkeypatch it. -_PING_INTERVAL: float = 15.0 diff --git a/functions/python/data-processor/fastapi/staticfiles.py b/functions/python/data-processor/fastapi/staticfiles.py deleted file mode 100644 index 299015d4f..000000000 --- a/functions/python/data-processor/fastapi/staticfiles.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.staticfiles import StaticFiles as StaticFiles # noqa diff --git a/functions/python/data-processor/fastapi/templating.py b/functions/python/data-processor/fastapi/templating.py deleted file mode 100644 index 0cb868486..000000000 --- a/functions/python/data-processor/fastapi/templating.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.templating import Jinja2Templates as Jinja2Templates # noqa diff --git a/functions/python/data-processor/fastapi/testclient.py b/functions/python/data-processor/fastapi/testclient.py deleted file mode 100644 index 4012406aa..000000000 --- a/functions/python/data-processor/fastapi/testclient.py +++ /dev/null @@ -1 +0,0 @@ -from starlette.testclient import TestClient as TestClient # noqa diff --git a/functions/python/data-processor/fastapi/types.py b/functions/python/data-processor/fastapi/types.py deleted file mode 100644 index 1fb86e13b..000000000 --- a/functions/python/data-processor/fastapi/types.py +++ /dev/null @@ -1,12 +0,0 @@ -import types -from collections.abc import Callable -from enum import Enum -from typing import Any, TypeVar, Union - -from pydantic import BaseModel -from pydantic.main import IncEx as IncEx - -DecoratedCallable = TypeVar("DecoratedCallable", bound=Callable[..., Any]) -UnionType = getattr(types, "UnionType", Union) -ModelNameMap = dict[type[BaseModel] | type[Enum], str] -DependencyCacheKey = tuple[Callable[..., Any] | None, tuple[str, ...], str] diff --git a/functions/python/data-processor/fastapi/utils.py b/functions/python/data-processor/fastapi/utils.py deleted file mode 100644 index 12eaa2bf0..000000000 --- a/functions/python/data-processor/fastapi/utils.py +++ /dev/null @@ -1,136 +0,0 @@ -import re -import warnings -from typing import ( - TYPE_CHECKING, - Any, - Literal, -) - -import fastapi -from fastapi._compat import ( - ModelField, - PydanticSchemaGenerationError, - Undefined, - annotation_is_pydantic_v1, -) -from fastapi.datastructures import DefaultPlaceholder, DefaultType -from fastapi.exceptions import FastAPIDeprecationWarning, PydanticV1NotSupportedError -from pydantic.fields import FieldInfo - -from ._compat import v2 - -if TYPE_CHECKING: # pragma: nocover - from .routing import APIRoute - - -def is_body_allowed_for_status_code(status_code: int | str | None) -> bool: - if status_code is None: - return True - # Ref: https://github.com/OAI/OpenAPI-Specification/blob/main/versions/3.1.0.md#patterned-fields-1 - if status_code in { - "default", - "1XX", - "2XX", - "3XX", - "4XX", - "5XX", - }: - return True - current_status_code = int(status_code) - return not (current_status_code < 200 or current_status_code in {204, 205, 304}) - - -def get_path_param_names(path: str) -> set[str]: - return set(re.findall("{(.*?)}", path)) - - -_invalid_args_message = ( - "Invalid args for response field! Hint: " - "check that {type_} is a valid Pydantic field type. " - "If you are using a return type annotation that is not a valid Pydantic " - "field (e.g. Union[Response, dict, None]) you can disable generating the " - "response model from the type annotation with the path operation decorator " - "parameter response_model=None. Read more: " - "https://fastapi.tiangolo.com/tutorial/response-model/" -) - - -def create_model_field( - name: str, - type_: Any, - default: Any | None = Undefined, - field_info: FieldInfo | None = None, - alias: str | None = None, - mode: Literal["validation", "serialization"] = "validation", -) -> ModelField: - if annotation_is_pydantic_v1(type_): - raise PydanticV1NotSupportedError( - "pydantic.v1 models are no longer supported by FastAPI." - f" Please update the response model {type_!r}." - ) - field_info = field_info or FieldInfo(annotation=type_, default=default, alias=alias) - try: - return v2.ModelField(mode=mode, name=name, field_info=field_info) - except PydanticSchemaGenerationError: - raise fastapi.exceptions.FastAPIError( - _invalid_args_message.format(type_=type_) - ) from None - - -def generate_operation_id_for_path( - *, name: str, path: str, method: str -) -> str: # pragma: nocover - warnings.warn( - message="fastapi.utils.generate_operation_id_for_path() was deprecated, " - "it is not used internally, and will be removed soon", - category=FastAPIDeprecationWarning, - stacklevel=2, - ) - operation_id = f"{name}{path}" - operation_id = re.sub(r"\W", "_", operation_id) - operation_id = f"{operation_id}_{method.lower()}" - return operation_id - - -def generate_unique_id(route: "APIRoute") -> str: - operation_id = f"{route.name}{route.path_format}" - operation_id = re.sub(r"\W", "_", operation_id) - assert route.methods - operation_id = f"{operation_id}_{list(route.methods)[0].lower()}" - return operation_id - - -def deep_dict_update(main_dict: dict[Any, Any], update_dict: dict[Any, Any]) -> None: - for key, value in update_dict.items(): - if ( - key in main_dict - and isinstance(main_dict[key], dict) - and isinstance(value, dict) - ): - deep_dict_update(main_dict[key], value) - elif ( - key in main_dict - and isinstance(main_dict[key], list) - and isinstance(update_dict[key], list) - ): - main_dict[key] = main_dict[key] + update_dict[key] - else: - main_dict[key] = value - - -def get_value_or_default( - first_item: DefaultPlaceholder | DefaultType, - *extra_items: DefaultPlaceholder | DefaultType, -) -> DefaultPlaceholder | DefaultType: - """ - Pass items or `DefaultPlaceholder`s by descending priority. - - The first one to _not_ be a `DefaultPlaceholder` will be returned. - - Otherwise, the first item (a `DefaultPlaceholder`) will be returned. - """ - items = (first_item,) + extra_items - for item in items: - if not isinstance(item, DefaultPlaceholder): - return item - return first_item diff --git a/functions/python/data-processor/fastapi/websockets.py b/functions/python/data-processor/fastapi/websockets.py deleted file mode 100644 index 55a4ac4a1..000000000 --- a/functions/python/data-processor/fastapi/websockets.py +++ /dev/null @@ -1,3 +0,0 @@ -from starlette.websockets import WebSocket as WebSocket # noqa -from starlette.websockets import WebSocketDisconnect as WebSocketDisconnect # noqa -from starlette.websockets import WebSocketState as WebSocketState # noqa diff --git a/functions/python/data-processor/h11-0.16.0.dist-info/INSTALLER b/functions/python/data-processor/h11-0.16.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/h11-0.16.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/h11-0.16.0.dist-info/METADATA b/functions/python/data-processor/h11-0.16.0.dist-info/METADATA deleted file mode 100644 index 8a2f63906..000000000 --- a/functions/python/data-processor/h11-0.16.0.dist-info/METADATA +++ /dev/null @@ -1,202 +0,0 @@ -Metadata-Version: 2.4 -Name: h11 -Version: 0.16.0 -Summary: A pure-Python, bring-your-own-I/O implementation of HTTP/1.1 -Home-page: https://github.com/python-hyper/h11 -Author: Nathaniel J. Smith -Author-email: njs@pobox.com -License: MIT -Classifier: Development Status :: 3 - Alpha -Classifier: Intended Audience :: Developers -Classifier: License :: OSI Approved :: MIT License -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.8 -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Topic :: Internet :: WWW/HTTP -Classifier: Topic :: System :: Networking -Requires-Python: >=3.8 -License-File: LICENSE.txt -Dynamic: author -Dynamic: author-email -Dynamic: classifier -Dynamic: description -Dynamic: home-page -Dynamic: license -Dynamic: license-file -Dynamic: requires-python -Dynamic: summary - -h11 -=== - -.. image:: https://travis-ci.org/python-hyper/h11.svg?branch=master - :target: https://travis-ci.org/python-hyper/h11 - :alt: Automated test status - -.. image:: https://codecov.io/gh/python-hyper/h11/branch/master/graph/badge.svg - :target: https://codecov.io/gh/python-hyper/h11 - :alt: Test coverage - -.. image:: https://readthedocs.org/projects/h11/badge/?version=latest - :target: http://h11.readthedocs.io/en/latest/?badge=latest - :alt: Documentation Status - -This is a little HTTP/1.1 library written from scratch in Python, -heavily inspired by `hyper-h2 `_. - -It's a "bring-your-own-I/O" library; h11 contains no IO code -whatsoever. This means you can hook h11 up to your favorite network -API, and that could be anything you want: synchronous, threaded, -asynchronous, or your own implementation of `RFC 6214 -`_ -- h11 won't judge you. -(Compare this to the current state of the art, where every time a `new -network API `_ comes along then someone -gets to start over reimplementing the entire HTTP protocol from -scratch.) Cory Benfield made an `excellent blog post describing the -benefits of this approach -`_, or if you like video -then here's his `PyCon 2016 talk on the same theme -`_. - -This also means that h11 is not immediately useful out of the box: -it's a toolkit for building programs that speak HTTP, not something -that could directly replace ``requests`` or ``twisted.web`` or -whatever. But h11 makes it much easier to implement something like -``requests`` or ``twisted.web``. - -At a high level, working with h11 goes like this: - -1) First, create an ``h11.Connection`` object to track the state of a - single HTTP/1.1 connection. - -2) When you read data off the network, pass it to - ``conn.receive_data(...)``; you'll get back a list of objects - representing high-level HTTP "events". - -3) When you want to send a high-level HTTP event, create the - corresponding "event" object and pass it to ``conn.send(...)``; - this will give you back some bytes that you can then push out - through the network. - -For example, a client might instantiate and then send a -``h11.Request`` object, then zero or more ``h11.Data`` objects for the -request body (e.g., if this is a POST), and then a -``h11.EndOfMessage`` to indicate the end of the message. Then the -server would then send back a ``h11.Response``, some ``h11.Data``, and -its own ``h11.EndOfMessage``. If either side violates the protocol, -you'll get a ``h11.ProtocolError`` exception. - -h11 is suitable for implementing both servers and clients, and has a -pleasantly symmetric API: the events you send as a client are exactly -the ones that you receive as a server and vice-versa. - -`Here's an example of a tiny HTTP client -`_ - -It also has `a fine manual `_. - -FAQ ---- - -*Whyyyyy?* - -I wanted to play with HTTP in `Curio -`__ and `Trio -`__, which at the time didn't have any -HTTP libraries. So I thought, no big deal, Python has, like, a dozen -different implementations of HTTP, surely I can find one that's -reusable. I didn't find one, but I did find Cory's call-to-arms -blog-post. So I figured, well, fine, if I have to implement HTTP from -scratch, at least I can make sure no-one *else* has to ever again. - -*Should I use it?* - -Maybe. You should be aware that it's a very young project. But, it's -feature complete and has an exhaustive test-suite and complete docs, -so the next step is for people to try using it and see how it goes -:-). If you do then please let us know -- if nothing else we'll want -to talk to you before making any incompatible changes! - -*What are the features/limitations?* - -Roughly speaking, it's trying to be a robust, complete, and non-hacky -implementation of the first "chapter" of the HTTP/1.1 spec: `RFC 7230: -HTTP/1.1 Message Syntax and Routing -`_. That is, it mostly focuses on -implementing HTTP at the level of taking bytes on and off the wire, -and the headers related to that, and tries to be anal about spec -conformance. It doesn't know about higher-level concerns like URL -routing, conditional GETs, cross-origin cookie policies, or content -negotiation. But it does know how to take care of framing, -cross-version differences in keep-alive handling, and the "obsolete -line folding" rule, so you can focus your energies on the hard / -interesting parts for your application, and it tries to support the -full specification in the sense that any useful HTTP/1.1 conformant -application should be able to use h11. - -It's pure Python, and has no dependencies outside of the standard -library. - -It has a test suite with 100.0% coverage for both statements and -branches. - -Currently it supports Python 3 (testing on 3.8-3.12) and PyPy 3. -The last Python 2-compatible version was h11 0.11.x. -(Originally it had a Cython wrapper for `http-parser -`_ and a beautiful nested state -machine implemented with ``yield from`` to postprocess the output. But -I had to take these out -- the new *parser* needs fewer lines-of-code -than the old *parser wrapper*, is written in pure Python, uses no -exotic language syntax, and has more features. It's sad, really; that -old state machine was really slick. I just need a few sentences here -to mourn that.) - -I don't know how fast it is. I haven't benchmarked or profiled it yet, -so it's probably got a few pointless hot spots, and I've been trying -to err on the side of simplicity and robustness instead of -micro-optimization. But at the architectural level I tried hard to -avoid fundamentally bad decisions, e.g., I believe that all the -parsing algorithms remain linear-time even in the face of pathological -input like slowloris, and there are no byte-by-byte loops. (I also -believe that it maintains bounded memory usage in the face of -arbitrary/pathological input.) - -The whole library is ~800 lines-of-code. You can read and understand -the whole thing in less than an hour. Most of the energy invested in -this so far has been spent on trying to keep things simple by -minimizing special-cases and ad hoc state manipulation; even though it -is now quite small and simple, I'm still annoyed that I haven't -figured out how to make it even smaller and simpler. (Unfortunately, -HTTP does not lend itself to simplicity.) - -The API is ~feature complete and I don't expect the general outlines -to change much, but you can't judge an API's ergonomics until you -actually document and use it, so I'd expect some changes in the -details. - -*How do I try it?* - -.. code-block:: sh - - $ pip install h11 - $ git clone git@github.com:python-hyper/h11 - $ cd h11/examples - $ python basic-client.py - -and go from there. - -*License?* - -MIT - -*Code of conduct?* - -Contributors are requested to follow our `code of conduct -`_ in -all project spaces. diff --git a/functions/python/data-processor/h11-0.16.0.dist-info/RECORD b/functions/python/data-processor/h11-0.16.0.dist-info/RECORD deleted file mode 100644 index ada1389b7..000000000 --- a/functions/python/data-processor/h11-0.16.0.dist-info/RECORD +++ /dev/null @@ -1,29 +0,0 @@ -h11-0.16.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -h11-0.16.0.dist-info/METADATA,sha256=KPMmCYrAn8unm48YD5YIfIQf4kViFct7hyqcfVzRnWQ,8348 -h11-0.16.0.dist-info/RECORD,, -h11-0.16.0.dist-info/WHEEL,sha256=CmyFI0kx5cdEMTLiONQRbGQwjIoR1aIYB7eCAQ4KPJ0,91 -h11-0.16.0.dist-info/licenses/LICENSE.txt,sha256=N9tbuFkm2yikJ6JYZ_ELEjIAOuob5pzLhRE4rbjm82E,1124 -h11-0.16.0.dist-info/top_level.txt,sha256=F7dC4jl3zeh8TGHEPaWJrMbeuoWbS379Gwdi-Yvdcis,4 -h11/__init__.py,sha256=iO1KzkSO42yZ6ffg-VMgbx_ZVTWGUY00nRYEWn-s3kY,1507 -h11/__pycache__/__init__.cpython-314.pyc,, -h11/__pycache__/_abnf.cpython-314.pyc,, -h11/__pycache__/_connection.cpython-314.pyc,, -h11/__pycache__/_events.cpython-314.pyc,, -h11/__pycache__/_headers.cpython-314.pyc,, -h11/__pycache__/_readers.cpython-314.pyc,, -h11/__pycache__/_receivebuffer.cpython-314.pyc,, -h11/__pycache__/_state.cpython-314.pyc,, -h11/__pycache__/_util.cpython-314.pyc,, -h11/__pycache__/_version.cpython-314.pyc,, -h11/__pycache__/_writers.cpython-314.pyc,, -h11/_abnf.py,sha256=ybixr0xsupnkA6GFAyMubuXF6Tc1lb_hF890NgCsfNc,4815 -h11/_connection.py,sha256=k9YRVf6koZqbttBW36xSWaJpWdZwa-xQVU9AHEo9DuI,26863 -h11/_events.py,sha256=I97aXoal1Wu7dkL548BANBUCkOIbe-x5CioYA9IBY14,11792 -h11/_headers.py,sha256=P7D-lBNxHwdLZPLimmYwrPG-9ZkjElvvJZJdZAgSP-4,10412 -h11/_readers.py,sha256=a4RypORUCC3d0q_kxPuBIM7jTD8iLt5X91TH0FsduN4,8590 -h11/_receivebuffer.py,sha256=xrspsdsNgWFxRfQcTXxR8RrdjRXXTK0Io5cQYWpJ1Ws,5252 -h11/_state.py,sha256=_5LG_BGR8FCcFQeBPH-TMHgm_-B-EUcWCnQof_9XjFE,13231 -h11/_util.py,sha256=LWkkjXyJaFlAy6Lt39w73UStklFT5ovcvo0TkY7RYuk,4888 -h11/_version.py,sha256=GVSsbPSPDcOuF6ptfIiXnVJoaEm3ygXbMnqlr_Giahw,686 -h11/_writers.py,sha256=oFKm6PtjeHfbj4RLX7VB7KDc1gIY53gXG3_HR9ltmTA,5081 -h11/py.typed,sha256=sow9soTwP9T_gEAQSVh7Gb8855h04Nwmhs2We-JRgZM,7 diff --git a/functions/python/data-processor/h11-0.16.0.dist-info/WHEEL b/functions/python/data-processor/h11-0.16.0.dist-info/WHEEL deleted file mode 100644 index 1eb3c49d9..000000000 --- a/functions/python/data-processor/h11-0.16.0.dist-info/WHEEL +++ /dev/null @@ -1,5 +0,0 @@ -Wheel-Version: 1.0 -Generator: setuptools (78.1.0) -Root-Is-Purelib: true -Tag: py3-none-any - diff --git a/functions/python/data-processor/h11-0.16.0.dist-info/licenses/LICENSE.txt b/functions/python/data-processor/h11-0.16.0.dist-info/licenses/LICENSE.txt deleted file mode 100644 index 8f080eae8..000000000 --- a/functions/python/data-processor/h11-0.16.0.dist-info/licenses/LICENSE.txt +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Nathaniel J. Smith and other contributors - -Permission is hereby granted, free of charge, to any person obtaining -a copy of this software and associated documentation files (the -"Software"), to deal in the Software without restriction, including -without limitation the rights to use, copy, modify, merge, publish, -distribute, sublicense, and/or sell copies of the Software, and to -permit persons to whom the Software is furnished to do so, subject to -the following conditions: - -The above copyright notice and this permission notice shall be -included in all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND -NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE -LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION -OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION -WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/functions/python/data-processor/h11-0.16.0.dist-info/top_level.txt b/functions/python/data-processor/h11-0.16.0.dist-info/top_level.txt deleted file mode 100644 index 0d24def71..000000000 --- a/functions/python/data-processor/h11-0.16.0.dist-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -h11 diff --git a/functions/python/data-processor/h11/__init__.py b/functions/python/data-processor/h11/__init__.py deleted file mode 100644 index 989e92c34..000000000 --- a/functions/python/data-processor/h11/__init__.py +++ /dev/null @@ -1,62 +0,0 @@ -# A highish-level implementation of the HTTP/1.1 wire protocol (RFC 7230), -# containing no networking code at all, loosely modelled on hyper-h2's generic -# implementation of HTTP/2 (and in particular the h2.connection.H2Connection -# class). There's still a bunch of subtle details you need to get right if you -# want to make this actually useful, because it doesn't implement all the -# semantics to check that what you're asking to write to the wire is sensible, -# but at least it gets you out of dealing with the wire itself. - -from h11._connection import Connection, NEED_DATA, PAUSED -from h11._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) -from h11._state import ( - CLIENT, - CLOSED, - DONE, - ERROR, - IDLE, - MIGHT_SWITCH_PROTOCOL, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, - SWITCHED_PROTOCOL, -) -from h11._util import LocalProtocolError, ProtocolError, RemoteProtocolError -from h11._version import __version__ - -PRODUCT_ID = "python-h11/" + __version__ - - -__all__ = ( - "Connection", - "NEED_DATA", - "PAUSED", - "ConnectionClosed", - "Data", - "EndOfMessage", - "Event", - "InformationalResponse", - "Request", - "Response", - "CLIENT", - "CLOSED", - "DONE", - "ERROR", - "IDLE", - "MUST_CLOSE", - "SEND_BODY", - "SEND_RESPONSE", - "SERVER", - "SWITCHED_PROTOCOL", - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", -) diff --git a/functions/python/data-processor/h11/_abnf.py b/functions/python/data-processor/h11/_abnf.py deleted file mode 100644 index 933587fba..000000000 --- a/functions/python/data-processor/h11/_abnf.py +++ /dev/null @@ -1,132 +0,0 @@ -# We use native strings for all the re patterns, to take advantage of string -# formatting, and then convert to bytestrings when compiling the final re -# objects. - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#whitespace -# OWS = *( SP / HTAB ) -# ; optional whitespace -OWS = r"[ \t]*" - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.token.separators -# token = 1*tchar -# -# tchar = "!" / "#" / "$" / "%" / "&" / "'" / "*" -# / "+" / "-" / "." / "^" / "_" / "`" / "|" / "~" -# / DIGIT / ALPHA -# ; any VCHAR, except delimiters -token = r"[-!#$%&'*+.^_`|~0-9a-zA-Z]+" - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#header.fields -# field-name = token -field_name = token - -# The standard says: -# -# field-value = *( field-content / obs-fold ) -# field-content = field-vchar [ 1*( SP / HTAB ) field-vchar ] -# field-vchar = VCHAR / obs-text -# obs-fold = CRLF 1*( SP / HTAB ) -# ; obsolete line folding -# ; see Section 3.2.4 -# -# https://tools.ietf.org/html/rfc5234#appendix-B.1 -# -# VCHAR = %x21-7E -# ; visible (printing) characters -# -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#rule.quoted-string -# obs-text = %x80-FF -# -# However, the standard definition of field-content is WRONG! It disallows -# fields containing a single visible character surrounded by whitespace, -# e.g. "foo a bar". -# -# See: https://www.rfc-editor.org/errata_search.php?rfc=7230&eid=4189 -# -# So our definition of field_content attempts to fix it up... -# -# Also, we allow lots of control characters, because apparently people assume -# that they're legal in practice (e.g., google analytics makes cookies with -# \x01 in them!): -# https://github.com/python-hyper/h11/issues/57 -# We still don't allow NUL or whitespace, because those are often treated as -# meta-characters and letting them through can lead to nasty issues like SSRF. -vchar = r"[\x21-\x7e]" -vchar_or_obs_text = r"[^\x00\s]" -field_vchar = vchar_or_obs_text -field_content = r"{field_vchar}+(?:[ \t]+{field_vchar}+)*".format(**globals()) - -# We handle obs-fold at a different level, and our fixed-up field_content -# already grows to swallow the whole value, so ? instead of * -field_value = r"({field_content})?".format(**globals()) - -# header-field = field-name ":" OWS field-value OWS -header_field = ( - r"(?P{field_name})" - r":" - r"{OWS}" - r"(?P{field_value})" - r"{OWS}".format(**globals()) -) - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#request.line -# -# request-line = method SP request-target SP HTTP-version CRLF -# method = token -# HTTP-version = HTTP-name "/" DIGIT "." DIGIT -# HTTP-name = %x48.54.54.50 ; "HTTP", case-sensitive -# -# request-target is complicated (see RFC 7230 sec 5.3) -- could be path, full -# URL, host+port (for connect), or even "*", but in any case we are guaranteed -# that it contists of the visible printing characters. -method = token -request_target = r"{vchar}+".format(**globals()) -http_version = r"HTTP/(?P[0-9]\.[0-9])" -request_line = ( - r"(?P{method})" - r" " - r"(?P{request_target})" - r" " - r"{http_version}".format(**globals()) -) - -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#status.line -# -# status-line = HTTP-version SP status-code SP reason-phrase CRLF -# status-code = 3DIGIT -# reason-phrase = *( HTAB / SP / VCHAR / obs-text ) -status_code = r"[0-9]{3}" -reason_phrase = r"([ \t]|{vchar_or_obs_text})*".format(**globals()) -status_line = ( - r"{http_version}" - r" " - r"(?P{status_code})" - # However, there are apparently a few too many servers out there that just - # leave out the reason phrase: - # https://github.com/scrapy/scrapy/issues/345#issuecomment-281756036 - # https://github.com/seanmonstar/httparse/issues/29 - # so make it optional. ?: is a non-capturing group. - r"(?: (?P{reason_phrase}))?".format(**globals()) -) - -HEXDIG = r"[0-9A-Fa-f]" -# Actually -# -# chunk-size = 1*HEXDIG -# -# but we impose an upper-limit to avoid ridiculosity. len(str(2**64)) == 20 -chunk_size = r"({HEXDIG}){{1,20}}".format(**globals()) -# Actually -# -# chunk-ext = *( ";" chunk-ext-name [ "=" chunk-ext-val ] ) -# -# but we aren't parsing the things so we don't really care. -chunk_ext = r";.*" -chunk_header = ( - r"(?P{chunk_size})" - r"(?P{chunk_ext})?" - r"{OWS}\r\n".format( - **globals() - ) # Even though the specification does not allow for extra whitespaces, - # we are lenient with trailing whitespaces because some servers on the wild use it. -) diff --git a/functions/python/data-processor/h11/_connection.py b/functions/python/data-processor/h11/_connection.py deleted file mode 100644 index e37d82a82..000000000 --- a/functions/python/data-processor/h11/_connection.py +++ /dev/null @@ -1,659 +0,0 @@ -# This contains the main Connection class. Everything in h11 revolves around -# this. -from typing import ( - Any, - Callable, - cast, - Dict, - List, - Optional, - overload, - Tuple, - Type, - Union, -) - -from ._events import ( - ConnectionClosed, - Data, - EndOfMessage, - Event, - InformationalResponse, - Request, - Response, -) -from ._headers import get_comma_header, has_expect_100_continue, set_comma_header -from ._readers import READERS, ReadersType -from ._receivebuffer import ReceiveBuffer -from ._state import ( - _SWITCH_CONNECT, - _SWITCH_UPGRADE, - CLIENT, - ConnectionState, - DONE, - ERROR, - MIGHT_SWITCH_PROTOCOL, - SEND_BODY, - SERVER, - SWITCHED_PROTOCOL, -) -from ._util import ( # Import the internal things we need - LocalProtocolError, - RemoteProtocolError, - Sentinel, -) -from ._writers import WRITERS, WritersType - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = ["Connection", "NEED_DATA", "PAUSED"] - - -class NEED_DATA(Sentinel, metaclass=Sentinel): - pass - - -class PAUSED(Sentinel, metaclass=Sentinel): - pass - - -# If we ever have this much buffered without it making a complete parseable -# event, we error out. The only time we really buffer is when reading the -# request/response line + headers together, so this is effectively the limit on -# the size of that. -# -# Some precedents for defaults: -# - node.js: 80 * 1024 -# - tomcat: 8 * 1024 -# - IIS: 16 * 1024 -# - Apache: <8 KiB per line> -DEFAULT_MAX_INCOMPLETE_EVENT_SIZE = 16 * 1024 - - -# RFC 7230's rules for connection lifecycles: -# - If either side says they want to close the connection, then the connection -# must close. -# - HTTP/1.1 defaults to keep-alive unless someone says Connection: close -# - HTTP/1.0 defaults to close unless both sides say Connection: keep-alive -# (and even this is a mess -- e.g. if you're implementing a proxy then -# sending Connection: keep-alive is forbidden). -# -# We simplify life by simply not supporting keep-alive with HTTP/1.0 peers. So -# our rule is: -# - If someone says Connection: close, we will close -# - If someone uses HTTP/1.0, we will close. -def _keep_alive(event: Union[Request, Response]) -> bool: - connection = get_comma_header(event.headers, b"connection") - if b"close" in connection: - return False - if getattr(event, "http_version", b"1.1") < b"1.1": - return False - return True - - -def _body_framing( - request_method: bytes, event: Union[Request, Response] -) -> Tuple[str, Union[Tuple[()], Tuple[int]]]: - # Called when we enter SEND_BODY to figure out framing information for - # this body. - # - # These are the only two events that can trigger a SEND_BODY state: - assert type(event) in (Request, Response) - # Returns one of: - # - # ("content-length", count) - # ("chunked", ()) - # ("http/1.0", ()) - # - # which are (lookup key, *args) for constructing body reader/writer - # objects. - # - # Reference: https://tools.ietf.org/html/rfc7230#section-3.3.3 - # - # Step 1: some responses always have an empty body, regardless of what the - # headers say. - if type(event) is Response: - if ( - event.status_code in (204, 304) - or request_method == b"HEAD" - or (request_method == b"CONNECT" and 200 <= event.status_code < 300) - ): - return ("content-length", (0,)) - # Section 3.3.3 also lists another case -- responses with status_code - # < 200. For us these are InformationalResponses, not Responses, so - # they can't get into this function in the first place. - assert event.status_code >= 200 - - # Step 2: check for Transfer-Encoding (T-E beats C-L): - transfer_encodings = get_comma_header(event.headers, b"transfer-encoding") - if transfer_encodings: - assert transfer_encodings == [b"chunked"] - return ("chunked", ()) - - # Step 3: check for Content-Length - content_lengths = get_comma_header(event.headers, b"content-length") - if content_lengths: - return ("content-length", (int(content_lengths[0]),)) - - # Step 4: no applicable headers; fallback/default depends on type - if type(event) is Request: - return ("content-length", (0,)) - else: - return ("http/1.0", ()) - - -################################################################ -# -# The main Connection class -# -################################################################ - - -class Connection: - """An object encapsulating the state of an HTTP connection. - - Args: - our_role: If you're implementing a client, pass :data:`h11.CLIENT`. If - you're implementing a server, pass :data:`h11.SERVER`. - - max_incomplete_event_size (int): - The maximum number of bytes we're willing to buffer of an - incomplete event. In practice this mostly sets a limit on the - maximum size of the request/response line + headers. If this is - exceeded, then :meth:`next_event` will raise - :exc:`RemoteProtocolError`. - - """ - - def __init__( - self, - our_role: Type[Sentinel], - max_incomplete_event_size: int = DEFAULT_MAX_INCOMPLETE_EVENT_SIZE, - ) -> None: - self._max_incomplete_event_size = max_incomplete_event_size - # State and role tracking - if our_role not in (CLIENT, SERVER): - raise ValueError(f"expected CLIENT or SERVER, not {our_role!r}") - self.our_role = our_role - self.their_role: Type[Sentinel] - if our_role is CLIENT: - self.their_role = SERVER - else: - self.their_role = CLIENT - self._cstate = ConnectionState() - - # Callables for converting data->events or vice-versa given the - # current state - self._writer = self._get_io_object(self.our_role, None, WRITERS) - self._reader = self._get_io_object(self.their_role, None, READERS) - - # Holds any unprocessed received data - self._receive_buffer = ReceiveBuffer() - # If this is true, then it indicates that the incoming connection was - # closed *after* the end of whatever's in self._receive_buffer: - self._receive_buffer_closed = False - - # Extra bits of state that don't fit into the state machine. - # - # These two are only used to interpret framing headers for figuring - # out how to read/write response bodies. their_http_version is also - # made available as a convenient public API. - self.their_http_version: Optional[bytes] = None - self._request_method: Optional[bytes] = None - # This is pure flow-control and doesn't at all affect the set of legal - # transitions, so no need to bother ConnectionState with it: - self.client_is_waiting_for_100_continue = False - - @property - def states(self) -> Dict[Type[Sentinel], Type[Sentinel]]: - """A dictionary like:: - - {CLIENT: , SERVER: } - - See :ref:`state-machine` for details. - - """ - return dict(self._cstate.states) - - @property - def our_state(self) -> Type[Sentinel]: - """The current state of whichever role we are playing. See - :ref:`state-machine` for details. - """ - return self._cstate.states[self.our_role] - - @property - def their_state(self) -> Type[Sentinel]: - """The current state of whichever role we are NOT playing. See - :ref:`state-machine` for details. - """ - return self._cstate.states[self.their_role] - - @property - def they_are_waiting_for_100_continue(self) -> bool: - return self.their_role is CLIENT and self.client_is_waiting_for_100_continue - - def start_next_cycle(self) -> None: - """Attempt to reset our connection state for a new request/response - cycle. - - If both client and server are in :data:`DONE` state, then resets them - both to :data:`IDLE` state in preparation for a new request/response - cycle on this same connection. Otherwise, raises a - :exc:`LocalProtocolError`. - - See :ref:`keepalive-and-pipelining`. - - """ - old_states = dict(self._cstate.states) - self._cstate.start_next_cycle() - self._request_method = None - # self.their_http_version gets left alone, since it presumably lasts - # beyond a single request/response cycle - assert not self.client_is_waiting_for_100_continue - self._respond_to_state_changes(old_states) - - def _process_error(self, role: Type[Sentinel]) -> None: - old_states = dict(self._cstate.states) - self._cstate.process_error(role) - self._respond_to_state_changes(old_states) - - def _server_switch_event(self, event: Event) -> Optional[Type[Sentinel]]: - if type(event) is InformationalResponse and event.status_code == 101: - return _SWITCH_UPGRADE - if type(event) is Response: - if ( - _SWITCH_CONNECT in self._cstate.pending_switch_proposals - and 200 <= event.status_code < 300 - ): - return _SWITCH_CONNECT - return None - - # All events go through here - def _process_event(self, role: Type[Sentinel], event: Event) -> None: - # First, pass the event through the state machine to make sure it - # succeeds. - old_states = dict(self._cstate.states) - if role is CLIENT and type(event) is Request: - if event.method == b"CONNECT": - self._cstate.process_client_switch_proposal(_SWITCH_CONNECT) - if get_comma_header(event.headers, b"upgrade"): - self._cstate.process_client_switch_proposal(_SWITCH_UPGRADE) - server_switch_event = None - if role is SERVER: - server_switch_event = self._server_switch_event(event) - self._cstate.process_event(role, type(event), server_switch_event) - - # Then perform the updates triggered by it. - - if type(event) is Request: - self._request_method = event.method - - if role is self.their_role and type(event) in ( - Request, - Response, - InformationalResponse, - ): - event = cast(Union[Request, Response, InformationalResponse], event) - self.their_http_version = event.http_version - - # Keep alive handling - # - # RFC 7230 doesn't really say what one should do if Connection: close - # shows up on a 1xx InformationalResponse. I think the idea is that - # this is not supposed to happen. In any case, if it does happen, we - # ignore it. - if type(event) in (Request, Response) and not _keep_alive( - cast(Union[Request, Response], event) - ): - self._cstate.process_keep_alive_disabled() - - # 100-continue - if type(event) is Request and has_expect_100_continue(event): - self.client_is_waiting_for_100_continue = True - if type(event) in (InformationalResponse, Response): - self.client_is_waiting_for_100_continue = False - if role is CLIENT and type(event) in (Data, EndOfMessage): - self.client_is_waiting_for_100_continue = False - - self._respond_to_state_changes(old_states, event) - - def _get_io_object( - self, - role: Type[Sentinel], - event: Optional[Event], - io_dict: Union[ReadersType, WritersType], - ) -> Optional[Callable[..., Any]]: - # event may be None; it's only used when entering SEND_BODY - state = self._cstate.states[role] - if state is SEND_BODY: - # Special case: the io_dict has a dict of reader/writer factories - # that depend on the request/response framing. - framing_type, args = _body_framing( - cast(bytes, self._request_method), cast(Union[Request, Response], event) - ) - return io_dict[SEND_BODY][framing_type](*args) # type: ignore[index] - else: - # General case: the io_dict just has the appropriate reader/writer - # for this state - return io_dict.get((role, state)) # type: ignore[return-value] - - # This must be called after any action that might have caused - # self._cstate.states to change. - def _respond_to_state_changes( - self, - old_states: Dict[Type[Sentinel], Type[Sentinel]], - event: Optional[Event] = None, - ) -> None: - # Update reader/writer - if self.our_state != old_states[self.our_role]: - self._writer = self._get_io_object(self.our_role, event, WRITERS) - if self.their_state != old_states[self.their_role]: - self._reader = self._get_io_object(self.their_role, event, READERS) - - @property - def trailing_data(self) -> Tuple[bytes, bool]: - """Data that has been received, but not yet processed, represented as - a tuple with two elements, where the first is a byte-string containing - the unprocessed data itself, and the second is a bool that is True if - the receive connection was closed. - - See :ref:`switching-protocols` for discussion of why you'd want this. - """ - return (bytes(self._receive_buffer), self._receive_buffer_closed) - - def receive_data(self, data: bytes) -> None: - """Add data to our internal receive buffer. - - This does not actually do any processing on the data, just stores - it. To trigger processing, you have to call :meth:`next_event`. - - Args: - data (:term:`bytes-like object`): - The new data that was just received. - - Special case: If *data* is an empty byte-string like ``b""``, - then this indicates that the remote side has closed the - connection (end of file). Normally this is convenient, because - standard Python APIs like :meth:`file.read` or - :meth:`socket.recv` use ``b""`` to indicate end-of-file, while - other failures to read are indicated using other mechanisms - like raising :exc:`TimeoutError`. When using such an API you - can just blindly pass through whatever you get from ``read`` - to :meth:`receive_data`, and everything will work. - - But, if you have an API where reading an empty string is a - valid non-EOF condition, then you need to be aware of this and - make sure to check for such strings and avoid passing them to - :meth:`receive_data`. - - Returns: - Nothing, but after calling this you should call :meth:`next_event` - to parse the newly received data. - - Raises: - RuntimeError: - Raised if you pass an empty *data*, indicating EOF, and then - pass a non-empty *data*, indicating more data that somehow - arrived after the EOF. - - (Calling ``receive_data(b"")`` multiple times is fine, - and equivalent to calling it once.) - - """ - if data: - if self._receive_buffer_closed: - raise RuntimeError("received close, then received more data?") - self._receive_buffer += data - else: - self._receive_buffer_closed = True - - def _extract_next_receive_event( - self, - ) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: - state = self.their_state - # We don't pause immediately when they enter DONE, because even in - # DONE state we can still process a ConnectionClosed() event. But - # if we have data in our buffer, then we definitely aren't getting - # a ConnectionClosed() immediately and we need to pause. - if state is DONE and self._receive_buffer: - return PAUSED - if state is MIGHT_SWITCH_PROTOCOL or state is SWITCHED_PROTOCOL: - return PAUSED - assert self._reader is not None - event = self._reader(self._receive_buffer) - if event is None: - if not self._receive_buffer and self._receive_buffer_closed: - # In some unusual cases (basically just HTTP/1.0 bodies), EOF - # triggers an actual protocol event; in that case, we want to - # return that event, and then the state will change and we'll - # get called again to generate the actual ConnectionClosed(). - if hasattr(self._reader, "read_eof"): - event = self._reader.read_eof() - else: - event = ConnectionClosed() - if event is None: - event = NEED_DATA - return event # type: ignore[no-any-return] - - def next_event(self) -> Union[Event, Type[NEED_DATA], Type[PAUSED]]: - """Parse the next event out of our receive buffer, update our internal - state, and return it. - - This is a mutating operation -- think of it like calling :func:`next` - on an iterator. - - Returns: - : One of three things: - - 1) An event object -- see :ref:`events`. - - 2) The special constant :data:`NEED_DATA`, which indicates that - you need to read more data from your socket and pass it to - :meth:`receive_data` before this method will be able to return - any more events. - - 3) The special constant :data:`PAUSED`, which indicates that we - are not in a state where we can process incoming data (usually - because the peer has finished their part of the current - request/response cycle, and you have not yet called - :meth:`start_next_cycle`). See :ref:`flow-control` for details. - - Raises: - RemoteProtocolError: - The peer has misbehaved. You should close the connection - (possibly after sending some kind of 4xx response). - - Once this method returns :class:`ConnectionClosed` once, then all - subsequent calls will also return :class:`ConnectionClosed`. - - If this method raises any exception besides :exc:`RemoteProtocolError` - then that's a bug -- if it happens please file a bug report! - - If this method raises any exception then it also sets - :attr:`Connection.their_state` to :data:`ERROR` -- see - :ref:`error-handling` for discussion. - - """ - - if self.their_state is ERROR: - raise RemoteProtocolError("Can't receive data when peer state is ERROR") - try: - event = self._extract_next_receive_event() - if event not in [NEED_DATA, PAUSED]: - self._process_event(self.their_role, cast(Event, event)) - if event is NEED_DATA: - if len(self._receive_buffer) > self._max_incomplete_event_size: - # 431 is "Request header fields too large" which is pretty - # much the only situation where we can get here - raise RemoteProtocolError( - "Receive buffer too long", error_status_hint=431 - ) - if self._receive_buffer_closed: - # We're still trying to complete some event, but that's - # never going to happen because no more data is coming - raise RemoteProtocolError("peer unexpectedly closed connection") - return event - except BaseException as exc: - self._process_error(self.their_role) - if isinstance(exc, LocalProtocolError): - exc._reraise_as_remote_protocol_error() - else: - raise - - @overload - def send(self, event: ConnectionClosed) -> None: - ... - - @overload - def send( - self, event: Union[Request, InformationalResponse, Response, Data, EndOfMessage] - ) -> bytes: - ... - - @overload - def send(self, event: Event) -> Optional[bytes]: - ... - - def send(self, event: Event) -> Optional[bytes]: - """Convert a high-level event into bytes that can be sent to the peer, - while updating our internal state machine. - - Args: - event: The :ref:`event ` to send. - - Returns: - If ``type(event) is ConnectionClosed``, then returns - ``None``. Otherwise, returns a :term:`bytes-like object`. - - Raises: - LocalProtocolError: - Sending this event at this time would violate our - understanding of the HTTP/1.1 protocol. - - If this method raises any exception then it also sets - :attr:`Connection.our_state` to :data:`ERROR` -- see - :ref:`error-handling` for discussion. - - """ - data_list = self.send_with_data_passthrough(event) - if data_list is None: - return None - else: - return b"".join(data_list) - - def send_with_data_passthrough(self, event: Event) -> Optional[List[bytes]]: - """Identical to :meth:`send`, except that in situations where - :meth:`send` returns a single :term:`bytes-like object`, this instead - returns a list of them -- and when sending a :class:`Data` event, this - list is guaranteed to contain the exact object you passed in as - :attr:`Data.data`. See :ref:`sendfile` for discussion. - - """ - if self.our_state is ERROR: - raise LocalProtocolError("Can't send data when our state is ERROR") - try: - if type(event) is Response: - event = self._clean_up_response_headers_for_sending(event) - # We want to call _process_event before calling the writer, - # because if someone tries to do something invalid then this will - # give a sensible error message, while our writers all just assume - # they will only receive valid events. But, _process_event might - # change self._writer. So we have to do a little dance: - writer = self._writer - self._process_event(self.our_role, event) - if type(event) is ConnectionClosed: - return None - else: - # In any situation where writer is None, process_event should - # have raised ProtocolError - assert writer is not None - data_list: List[bytes] = [] - writer(event, data_list.append) - return data_list - except: - self._process_error(self.our_role) - raise - - def send_failed(self) -> None: - """Notify the state machine that we failed to send the data it gave - us. - - This causes :attr:`Connection.our_state` to immediately become - :data:`ERROR` -- see :ref:`error-handling` for discussion. - - """ - self._process_error(self.our_role) - - # When sending a Response, we take responsibility for a few things: - # - # - Sometimes you MUST set Connection: close. We take care of those - # times. (You can also set it yourself if you want, and if you do then - # we'll respect that and close the connection at the right time. But you - # don't have to worry about that unless you want to.) - # - # - The user has to set Content-Length if they want it. Otherwise, for - # responses that have bodies (e.g. not HEAD), then we will automatically - # select the right mechanism for streaming a body of unknown length, - # which depends on depending on the peer's HTTP version. - # - # This function's *only* responsibility is making sure headers are set up - # right -- everything downstream just looks at the headers. There are no - # side channels. - def _clean_up_response_headers_for_sending(self, response: Response) -> Response: - assert type(response) is Response - - headers = response.headers - need_close = False - - # HEAD requests need some special handling: they always act like they - # have Content-Length: 0, and that's how _body_framing treats - # them. But their headers are supposed to match what we would send if - # the request was a GET. (Technically there is one deviation allowed: - # we're allowed to leave out the framing headers -- see - # https://tools.ietf.org/html/rfc7231#section-4.3.2 . But it's just as - # easy to get them right.) - method_for_choosing_headers = cast(bytes, self._request_method) - if method_for_choosing_headers == b"HEAD": - method_for_choosing_headers = b"GET" - framing_type, _ = _body_framing(method_for_choosing_headers, response) - if framing_type in ("chunked", "http/1.0"): - # This response has a body of unknown length. - # If our peer is HTTP/1.1, we use Transfer-Encoding: chunked - # If our peer is HTTP/1.0, we use no framing headers, and close the - # connection afterwards. - # - # Make sure to clear Content-Length (in principle user could have - # set both and then we ignored Content-Length b/c - # Transfer-Encoding overwrote it -- this would be naughty of them, - # but the HTTP spec says that if our peer does this then we have - # to fix it instead of erroring out, so we'll accord the user the - # same respect). - headers = set_comma_header(headers, b"content-length", []) - if self.their_http_version is None or self.their_http_version < b"1.1": - # Either we never got a valid request and are sending back an - # error (their_http_version is None), so we assume the worst; - # or else we did get a valid HTTP/1.0 request, so we know that - # they don't understand chunked encoding. - headers = set_comma_header(headers, b"transfer-encoding", []) - # This is actually redundant ATM, since currently we - # unconditionally disable keep-alive when talking to HTTP/1.0 - # peers. But let's be defensive just in case we add - # Connection: keep-alive support later: - if self._request_method != b"HEAD": - need_close = True - else: - headers = set_comma_header(headers, b"transfer-encoding", [b"chunked"]) - - if not self._cstate.keep_alive or need_close: - # Make sure Connection: close is set - connection = set(get_comma_header(headers, b"connection")) - connection.discard(b"keep-alive") - connection.add(b"close") - headers = set_comma_header(headers, b"connection", sorted(connection)) - - return Response( - headers=headers, - status_code=response.status_code, - http_version=response.http_version, - reason=response.reason, - ) diff --git a/functions/python/data-processor/h11/_events.py b/functions/python/data-processor/h11/_events.py deleted file mode 100644 index ca1c3adbd..000000000 --- a/functions/python/data-processor/h11/_events.py +++ /dev/null @@ -1,369 +0,0 @@ -# High level events that make up HTTP/1.1 conversations. Loosely inspired by -# the corresponding events in hyper-h2: -# -# http://python-hyper.org/h2/en/stable/api.html#events -# -# Don't subclass these. Stuff will break. - -import re -from abc import ABC -from dataclasses import dataclass -from typing import List, Tuple, Union - -from ._abnf import method, request_target -from ._headers import Headers, normalize_and_validate -from ._util import bytesify, LocalProtocolError, validate - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = [ - "Event", - "Request", - "InformationalResponse", - "Response", - "Data", - "EndOfMessage", - "ConnectionClosed", -] - -method_re = re.compile(method.encode("ascii")) -request_target_re = re.compile(request_target.encode("ascii")) - - -class Event(ABC): - """ - Base class for h11 events. - """ - - __slots__ = () - - -@dataclass(init=False, frozen=True) -class Request(Event): - """The beginning of an HTTP request. - - Fields: - - .. attribute:: method - - An HTTP method, e.g. ``b"GET"`` or ``b"POST"``. Always a byte - string. :term:`Bytes-like objects ` and native - strings containing only ascii characters will be automatically - converted to byte strings. - - .. attribute:: target - - The target of an HTTP request, e.g. ``b"/index.html"``, or one of the - more exotic formats described in `RFC 7320, section 5.3 - `_. Always a byte - string. :term:`Bytes-like objects ` and native - strings containing only ascii characters will be automatically - converted to byte strings. - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules ` for details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - ` for details. - - """ - - __slots__ = ("method", "headers", "target", "http_version") - - method: bytes - headers: Headers - target: bytes - http_version: bytes - - def __init__( - self, - *, - method: Union[bytes, str], - headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], - target: Union[bytes, str], - http_version: Union[bytes, str] = b"1.1", - _parsed: bool = False, - ) -> None: - super().__init__() - if isinstance(headers, Headers): - object.__setattr__(self, "headers", headers) - else: - object.__setattr__( - self, "headers", normalize_and_validate(headers, _parsed=_parsed) - ) - if not _parsed: - object.__setattr__(self, "method", bytesify(method)) - object.__setattr__(self, "target", bytesify(target)) - object.__setattr__(self, "http_version", bytesify(http_version)) - else: - object.__setattr__(self, "method", method) - object.__setattr__(self, "target", target) - object.__setattr__(self, "http_version", http_version) - - # "A server MUST respond with a 400 (Bad Request) status code to any - # HTTP/1.1 request message that lacks a Host header field and to any - # request message that contains more than one Host header field or a - # Host header field with an invalid field-value." - # -- https://tools.ietf.org/html/rfc7230#section-5.4 - host_count = 0 - for name, value in self.headers: - if name == b"host": - host_count += 1 - if self.http_version == b"1.1" and host_count == 0: - raise LocalProtocolError("Missing mandatory Host: header") - if host_count > 1: - raise LocalProtocolError("Found multiple Host: headers") - - validate(method_re, self.method, "Illegal method characters") - validate(request_target_re, self.target, "Illegal target characters") - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class _ResponseBase(Event): - __slots__ = ("headers", "http_version", "reason", "status_code") - - headers: Headers - http_version: bytes - reason: bytes - status_code: int - - def __init__( - self, - *, - headers: Union[Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]]], - status_code: int, - http_version: Union[bytes, str] = b"1.1", - reason: Union[bytes, str] = b"", - _parsed: bool = False, - ) -> None: - super().__init__() - if isinstance(headers, Headers): - object.__setattr__(self, "headers", headers) - else: - object.__setattr__( - self, "headers", normalize_and_validate(headers, _parsed=_parsed) - ) - if not _parsed: - object.__setattr__(self, "reason", bytesify(reason)) - object.__setattr__(self, "http_version", bytesify(http_version)) - if not isinstance(status_code, int): - raise LocalProtocolError("status code must be integer") - # Because IntEnum objects are instances of int, but aren't - # duck-compatible (sigh), see gh-72. - object.__setattr__(self, "status_code", int(status_code)) - else: - object.__setattr__(self, "reason", reason) - object.__setattr__(self, "http_version", http_version) - object.__setattr__(self, "status_code", status_code) - - self.__post_init__() - - def __post_init__(self) -> None: - pass - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class InformationalResponse(_ResponseBase): - """An HTTP informational response. - - Fields: - - .. attribute:: status_code - - The status code of this response, as an integer. For an - :class:`InformationalResponse`, this is always in the range [100, - 200). - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules ` for - details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - ` for details. - - .. attribute:: reason - - The reason phrase of this response, as a byte string. For example: - ``b"OK"``, or ``b"Not Found"``. - - """ - - def __post_init__(self) -> None: - if not (100 <= self.status_code < 200): - raise LocalProtocolError( - "InformationalResponse status_code should be in range " - "[100, 200), not {}".format(self.status_code) - ) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class Response(_ResponseBase): - """The beginning of an HTTP response. - - Fields: - - .. attribute:: status_code - - The status code of this response, as an integer. For an - :class:`Response`, this is always in the range [200, - 1000). - - .. attribute:: headers - - Request headers, represented as a list of (name, value) pairs. See - :ref:`the header normalization rules ` for details. - - .. attribute:: http_version - - The HTTP protocol version, represented as a byte string like - ``b"1.1"``. See :ref:`the HTTP version normalization rules - ` for details. - - .. attribute:: reason - - The reason phrase of this response, as a byte string. For example: - ``b"OK"``, or ``b"Not Found"``. - - """ - - def __post_init__(self) -> None: - if not (200 <= self.status_code < 1000): - raise LocalProtocolError( - "Response status_code should be in range [200, 1000), not {}".format( - self.status_code - ) - ) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(init=False, frozen=True) -class Data(Event): - """Part of an HTTP message body. - - Fields: - - .. attribute:: data - - A :term:`bytes-like object` containing part of a message body. Or, if - using the ``combine=False`` argument to :meth:`Connection.send`, then - any object that your socket writing code knows what to do with, and for - which calling :func:`len` returns the number of bytes that will be - written -- see :ref:`sendfile` for details. - - .. attribute:: chunk_start - - A marker that indicates whether this data object is from the start of a - chunked transfer encoding chunk. This field is ignored when when a Data - event is provided to :meth:`Connection.send`: it is only valid on - events emitted from :meth:`Connection.next_event`. You probably - shouldn't use this attribute at all; see - :ref:`chunk-delimiters-are-bad` for details. - - .. attribute:: chunk_end - - A marker that indicates whether this data object is the last for a - given chunked transfer encoding chunk. This field is ignored when when - a Data event is provided to :meth:`Connection.send`: it is only valid - on events emitted from :meth:`Connection.next_event`. You probably - shouldn't use this attribute at all; see - :ref:`chunk-delimiters-are-bad` for details. - - """ - - __slots__ = ("data", "chunk_start", "chunk_end") - - data: bytes - chunk_start: bool - chunk_end: bool - - def __init__( - self, data: bytes, chunk_start: bool = False, chunk_end: bool = False - ) -> None: - object.__setattr__(self, "data", data) - object.__setattr__(self, "chunk_start", chunk_start) - object.__setattr__(self, "chunk_end", chunk_end) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -# XX FIXME: "A recipient MUST ignore (or consider as an error) any fields that -# are forbidden to be sent in a trailer, since processing them as if they were -# present in the header section might bypass external security filters." -# https://svn.tools.ietf.org/svn/wg/httpbis/specs/rfc7230.html#chunked.trailer.part -# Unfortunately, the list of forbidden fields is long and vague :-/ -@dataclass(init=False, frozen=True) -class EndOfMessage(Event): - """The end of an HTTP message. - - Fields: - - .. attribute:: headers - - Default value: ``[]`` - - Any trailing headers attached to this message, represented as a list of - (name, value) pairs. See :ref:`the header normalization rules - ` for details. - - Must be empty unless ``Transfer-Encoding: chunked`` is in use. - - """ - - __slots__ = ("headers",) - - headers: Headers - - def __init__( - self, - *, - headers: Union[ - Headers, List[Tuple[bytes, bytes]], List[Tuple[str, str]], None - ] = None, - _parsed: bool = False, - ) -> None: - super().__init__() - if headers is None: - headers = Headers([]) - elif not isinstance(headers, Headers): - headers = normalize_and_validate(headers, _parsed=_parsed) - - object.__setattr__(self, "headers", headers) - - # This is an unhashable type. - __hash__ = None # type: ignore - - -@dataclass(frozen=True) -class ConnectionClosed(Event): - """This event indicates that the sender has closed their outgoing - connection. - - Note that this does not necessarily mean that they can't *receive* further - data, because TCP connections are composed to two one-way channels which - can be closed independently. See :ref:`closing` for details. - - No fields. - """ - - pass diff --git a/functions/python/data-processor/h11/_headers.py b/functions/python/data-processor/h11/_headers.py deleted file mode 100644 index 31da3e2b2..000000000 --- a/functions/python/data-processor/h11/_headers.py +++ /dev/null @@ -1,282 +0,0 @@ -import re -from typing import AnyStr, cast, List, overload, Sequence, Tuple, TYPE_CHECKING, Union - -from ._abnf import field_name, field_value -from ._util import bytesify, LocalProtocolError, validate - -if TYPE_CHECKING: - from ._events import Request - -try: - from typing import Literal -except ImportError: - from typing_extensions import Literal # type: ignore - -CONTENT_LENGTH_MAX_DIGITS = 20 # allow up to 1 billion TB - 1 - - -# Facts -# ----- -# -# Headers are: -# keys: case-insensitive ascii -# values: mixture of ascii and raw bytes -# -# "Historically, HTTP has allowed field content with text in the ISO-8859-1 -# charset [ISO-8859-1], supporting other charsets only through use of -# [RFC2047] encoding. In practice, most HTTP header field values use only a -# subset of the US-ASCII charset [USASCII]. Newly defined header fields SHOULD -# limit their field values to US-ASCII octets. A recipient SHOULD treat other -# octets in field content (obs-text) as opaque data." -# And it deprecates all non-ascii values -# -# Leading/trailing whitespace in header names is forbidden -# -# Values get leading/trailing whitespace stripped -# -# Content-Disposition actually needs to contain unicode semantically; to -# accomplish this it has a terrifically weird way of encoding the filename -# itself as ascii (and even this still has lots of cross-browser -# incompatibilities) -# -# Order is important: -# "a proxy MUST NOT change the order of these field values when forwarding a -# message" -# (and there are several headers where the order indicates a preference) -# -# Multiple occurences of the same header: -# "A sender MUST NOT generate multiple header fields with the same field name -# in a message unless either the entire field value for that header field is -# defined as a comma-separated list [or the header is Set-Cookie which gets a -# special exception]" - RFC 7230. (cookies are in RFC 6265) -# -# So every header aside from Set-Cookie can be merged by b", ".join if it -# occurs repeatedly. But, of course, they can't necessarily be split by -# .split(b","), because quoting. -# -# Given all this mess (case insensitive, duplicates allowed, order is -# important, ...), there doesn't appear to be any standard way to handle -# headers in Python -- they're almost like dicts, but... actually just -# aren't. For now we punt and just use a super simple representation: headers -# are a list of pairs -# -# [(name1, value1), (name2, value2), ...] -# -# where all entries are bytestrings, names are lowercase and have no -# leading/trailing whitespace, and values are bytestrings with no -# leading/trailing whitespace. Searching and updating are done via naive O(n) -# methods. -# -# Maybe a dict-of-lists would be better? - -_content_length_re = re.compile(rb"[0-9]+") -_field_name_re = re.compile(field_name.encode("ascii")) -_field_value_re = re.compile(field_value.encode("ascii")) - - -class Headers(Sequence[Tuple[bytes, bytes]]): - """ - A list-like interface that allows iterating over headers as byte-pairs - of (lowercased-name, value). - - Internally we actually store the representation as three-tuples, - including both the raw original casing, in order to preserve casing - over-the-wire, and the lowercased name, for case-insensitive comparisions. - - r = Request( - method="GET", - target="/", - headers=[("Host", "example.org"), ("Connection", "keep-alive")], - http_version="1.1", - ) - assert r.headers == [ - (b"host", b"example.org"), - (b"connection", b"keep-alive") - ] - assert r.headers.raw_items() == [ - (b"Host", b"example.org"), - (b"Connection", b"keep-alive") - ] - """ - - __slots__ = "_full_items" - - def __init__(self, full_items: List[Tuple[bytes, bytes, bytes]]) -> None: - self._full_items = full_items - - def __bool__(self) -> bool: - return bool(self._full_items) - - def __eq__(self, other: object) -> bool: - return list(self) == list(other) # type: ignore - - def __len__(self) -> int: - return len(self._full_items) - - def __repr__(self) -> str: - return "" % repr(list(self)) - - def __getitem__(self, idx: int) -> Tuple[bytes, bytes]: # type: ignore[override] - _, name, value = self._full_items[idx] - return (name, value) - - def raw_items(self) -> List[Tuple[bytes, bytes]]: - return [(raw_name, value) for raw_name, _, value in self._full_items] - - -HeaderTypes = Union[ - List[Tuple[bytes, bytes]], - List[Tuple[bytes, str]], - List[Tuple[str, bytes]], - List[Tuple[str, str]], -] - - -@overload -def normalize_and_validate(headers: Headers, _parsed: Literal[True]) -> Headers: - ... - - -@overload -def normalize_and_validate(headers: HeaderTypes, _parsed: Literal[False]) -> Headers: - ... - - -@overload -def normalize_and_validate( - headers: Union[Headers, HeaderTypes], _parsed: bool = False -) -> Headers: - ... - - -def normalize_and_validate( - headers: Union[Headers, HeaderTypes], _parsed: bool = False -) -> Headers: - new_headers = [] - seen_content_length = None - saw_transfer_encoding = False - for name, value in headers: - # For headers coming out of the parser, we can safely skip some steps, - # because it always returns bytes and has already run these regexes - # over the data: - if not _parsed: - name = bytesify(name) - value = bytesify(value) - validate(_field_name_re, name, "Illegal header name {!r}", name) - validate(_field_value_re, value, "Illegal header value {!r}", value) - assert isinstance(name, bytes) - assert isinstance(value, bytes) - - raw_name = name - name = name.lower() - if name == b"content-length": - lengths = {length.strip() for length in value.split(b",")} - if len(lengths) != 1: - raise LocalProtocolError("conflicting Content-Length headers") - value = lengths.pop() - validate(_content_length_re, value, "bad Content-Length") - if len(value) > CONTENT_LENGTH_MAX_DIGITS: - raise LocalProtocolError("bad Content-Length") - if seen_content_length is None: - seen_content_length = value - new_headers.append((raw_name, name, value)) - elif seen_content_length != value: - raise LocalProtocolError("conflicting Content-Length headers") - elif name == b"transfer-encoding": - # "A server that receives a request message with a transfer coding - # it does not understand SHOULD respond with 501 (Not - # Implemented)." - # https://tools.ietf.org/html/rfc7230#section-3.3.1 - if saw_transfer_encoding: - raise LocalProtocolError( - "multiple Transfer-Encoding headers", error_status_hint=501 - ) - # "All transfer-coding names are case-insensitive" - # -- https://tools.ietf.org/html/rfc7230#section-4 - value = value.lower() - if value != b"chunked": - raise LocalProtocolError( - "Only Transfer-Encoding: chunked is supported", - error_status_hint=501, - ) - saw_transfer_encoding = True - new_headers.append((raw_name, name, value)) - else: - new_headers.append((raw_name, name, value)) - return Headers(new_headers) - - -def get_comma_header(headers: Headers, name: bytes) -> List[bytes]: - # Should only be used for headers whose value is a list of - # comma-separated, case-insensitive values. - # - # The header name `name` is expected to be lower-case bytes. - # - # Connection: meets these criteria (including cast insensitivity). - # - # Content-Length: technically is just a single value (1*DIGIT), but the - # standard makes reference to implementations that do multiple values, and - # using this doesn't hurt. Ditto, case insensitivity doesn't things either - # way. - # - # Transfer-Encoding: is more complex (allows for quoted strings), so - # splitting on , is actually wrong. For example, this is legal: - # - # Transfer-Encoding: foo; options="1,2", chunked - # - # and should be parsed as - # - # foo; options="1,2" - # chunked - # - # but this naive function will parse it as - # - # foo; options="1 - # 2" - # chunked - # - # However, this is okay because the only thing we are going to do with - # any Transfer-Encoding is reject ones that aren't just "chunked", so - # both of these will be treated the same anyway. - # - # Expect: the only legal value is the literal string - # "100-continue". Splitting on commas is harmless. Case insensitive. - # - out: List[bytes] = [] - for _, found_name, found_raw_value in headers._full_items: - if found_name == name: - found_raw_value = found_raw_value.lower() - for found_split_value in found_raw_value.split(b","): - found_split_value = found_split_value.strip() - if found_split_value: - out.append(found_split_value) - return out - - -def set_comma_header(headers: Headers, name: bytes, new_values: List[bytes]) -> Headers: - # The header name `name` is expected to be lower-case bytes. - # - # Note that when we store the header we use title casing for the header - # names, in order to match the conventional HTTP header style. - # - # Simply calling `.title()` is a blunt approach, but it's correct - # here given the cases where we're using `set_comma_header`... - # - # Connection, Content-Length, Transfer-Encoding. - new_headers: List[Tuple[bytes, bytes]] = [] - for found_raw_name, found_name, found_raw_value in headers._full_items: - if found_name != name: - new_headers.append((found_raw_name, found_raw_value)) - for new_value in new_values: - new_headers.append((name.title(), new_value)) - return normalize_and_validate(new_headers) - - -def has_expect_100_continue(request: "Request") -> bool: - # https://tools.ietf.org/html/rfc7231#section-5.1.1 - # "A server that receives a 100-continue expectation in an HTTP/1.0 request - # MUST ignore that expectation." - if request.http_version < b"1.1": - return False - expect = get_comma_header(request.headers, b"expect") - return b"100-continue" in expect diff --git a/functions/python/data-processor/h11/_readers.py b/functions/python/data-processor/h11/_readers.py deleted file mode 100644 index 576804cc2..000000000 --- a/functions/python/data-processor/h11/_readers.py +++ /dev/null @@ -1,250 +0,0 @@ -# Code to read HTTP data -# -# Strategy: each reader is a callable which takes a ReceiveBuffer object, and -# either: -# 1) consumes some of it and returns an Event -# 2) raises a LocalProtocolError (for consistency -- e.g. we call validate() -# and it might raise a LocalProtocolError, so simpler just to always use -# this) -# 3) returns None, meaning "I need more data" -# -# If they have a .read_eof attribute, then this will be called if an EOF is -# received -- but this is optional. Either way, the actual ConnectionClosed -# event will be generated afterwards. -# -# READERS is a dict describing how to pick a reader. It maps states to either: -# - a reader -# - or, for body readers, a dict of per-framing reader factories - -import re -from typing import Any, Callable, Dict, Iterable, NoReturn, Optional, Tuple, Type, Union - -from ._abnf import chunk_header, header_field, request_line, status_line -from ._events import Data, EndOfMessage, InformationalResponse, Request, Response -from ._receivebuffer import ReceiveBuffer -from ._state import ( - CLIENT, - CLOSED, - DONE, - IDLE, - MUST_CLOSE, - SEND_BODY, - SEND_RESPONSE, - SERVER, -) -from ._util import LocalProtocolError, RemoteProtocolError, Sentinel, validate - -__all__ = ["READERS"] - -header_field_re = re.compile(header_field.encode("ascii")) -obs_fold_re = re.compile(rb"[ \t]+") - - -def _obsolete_line_fold(lines: Iterable[bytes]) -> Iterable[bytes]: - it = iter(lines) - last: Optional[bytes] = None - for line in it: - match = obs_fold_re.match(line) - if match: - if last is None: - raise LocalProtocolError("continuation line at start of headers") - if not isinstance(last, bytearray): - # Cast to a mutable type, avoiding copy on append to ensure O(n) time - last = bytearray(last) - last += b" " - last += line[match.end() :] - else: - if last is not None: - yield last - last = line - if last is not None: - yield last - - -def _decode_header_lines( - lines: Iterable[bytes], -) -> Iterable[Tuple[bytes, bytes]]: - for line in _obsolete_line_fold(lines): - matches = validate(header_field_re, line, "illegal header line: {!r}", line) - yield (matches["field_name"], matches["field_value"]) - - -request_line_re = re.compile(request_line.encode("ascii")) - - -def maybe_read_from_IDLE_client(buf: ReceiveBuffer) -> Optional[Request]: - lines = buf.maybe_extract_lines() - if lines is None: - if buf.is_next_line_obviously_invalid_request_line(): - raise LocalProtocolError("illegal request line") - return None - if not lines: - raise LocalProtocolError("no request line received") - matches = validate( - request_line_re, lines[0], "illegal request line: {!r}", lines[0] - ) - return Request( - headers=list(_decode_header_lines(lines[1:])), _parsed=True, **matches - ) - - -status_line_re = re.compile(status_line.encode("ascii")) - - -def maybe_read_from_SEND_RESPONSE_server( - buf: ReceiveBuffer, -) -> Union[InformationalResponse, Response, None]: - lines = buf.maybe_extract_lines() - if lines is None: - if buf.is_next_line_obviously_invalid_request_line(): - raise LocalProtocolError("illegal request line") - return None - if not lines: - raise LocalProtocolError("no response line received") - matches = validate(status_line_re, lines[0], "illegal status line: {!r}", lines[0]) - http_version = ( - b"1.1" if matches["http_version"] is None else matches["http_version"] - ) - reason = b"" if matches["reason"] is None else matches["reason"] - status_code = int(matches["status_code"]) - class_: Union[Type[InformationalResponse], Type[Response]] = ( - InformationalResponse if status_code < 200 else Response - ) - return class_( - headers=list(_decode_header_lines(lines[1:])), - _parsed=True, - status_code=status_code, - reason=reason, - http_version=http_version, - ) - - -class ContentLengthReader: - def __init__(self, length: int) -> None: - self._length = length - self._remaining = length - - def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: - if self._remaining == 0: - return EndOfMessage() - data = buf.maybe_extract_at_most(self._remaining) - if data is None: - return None - self._remaining -= len(data) - return Data(data=data) - - def read_eof(self) -> NoReturn: - raise RemoteProtocolError( - "peer closed connection without sending complete message body " - "(received {} bytes, expected {})".format( - self._length - self._remaining, self._length - ) - ) - - -chunk_header_re = re.compile(chunk_header.encode("ascii")) - - -class ChunkedReader: - def __init__(self) -> None: - self._bytes_in_chunk = 0 - # After reading a chunk, we have to throw away the trailing \r\n. - # This tracks the bytes that we need to match and throw away. - self._bytes_to_discard = b"" - self._reading_trailer = False - - def __call__(self, buf: ReceiveBuffer) -> Union[Data, EndOfMessage, None]: - if self._reading_trailer: - lines = buf.maybe_extract_lines() - if lines is None: - return None - return EndOfMessage(headers=list(_decode_header_lines(lines))) - if self._bytes_to_discard: - data = buf.maybe_extract_at_most(len(self._bytes_to_discard)) - if data is None: - return None - if data != self._bytes_to_discard[: len(data)]: - raise LocalProtocolError( - f"malformed chunk footer: {data!r} (expected {self._bytes_to_discard!r})" - ) - self._bytes_to_discard = self._bytes_to_discard[len(data) :] - if self._bytes_to_discard: - return None - # else, fall through and read some more - assert self._bytes_to_discard == b"" - if self._bytes_in_chunk == 0: - # We need to refill our chunk count - chunk_header = buf.maybe_extract_next_line() - if chunk_header is None: - return None - matches = validate( - chunk_header_re, - chunk_header, - "illegal chunk header: {!r}", - chunk_header, - ) - # XX FIXME: we discard chunk extensions. Does anyone care? - self._bytes_in_chunk = int(matches["chunk_size"], base=16) - if self._bytes_in_chunk == 0: - self._reading_trailer = True - return self(buf) - chunk_start = True - else: - chunk_start = False - assert self._bytes_in_chunk > 0 - data = buf.maybe_extract_at_most(self._bytes_in_chunk) - if data is None: - return None - self._bytes_in_chunk -= len(data) - if self._bytes_in_chunk == 0: - self._bytes_to_discard = b"\r\n" - chunk_end = True - else: - chunk_end = False - return Data(data=data, chunk_start=chunk_start, chunk_end=chunk_end) - - def read_eof(self) -> NoReturn: - raise RemoteProtocolError( - "peer closed connection without sending complete message body " - "(incomplete chunked read)" - ) - - -class Http10Reader: - def __call__(self, buf: ReceiveBuffer) -> Optional[Data]: - data = buf.maybe_extract_at_most(999999999) - if data is None: - return None - return Data(data=data) - - def read_eof(self) -> EndOfMessage: - return EndOfMessage() - - -def expect_nothing(buf: ReceiveBuffer) -> None: - if buf: - raise LocalProtocolError("Got data when expecting EOF") - return None - - -ReadersType = Dict[ - Union[Type[Sentinel], Tuple[Type[Sentinel], Type[Sentinel]]], - Union[Callable[..., Any], Dict[str, Callable[..., Any]]], -] - -READERS: ReadersType = { - (CLIENT, IDLE): maybe_read_from_IDLE_client, - (SERVER, IDLE): maybe_read_from_SEND_RESPONSE_server, - (SERVER, SEND_RESPONSE): maybe_read_from_SEND_RESPONSE_server, - (CLIENT, DONE): expect_nothing, - (CLIENT, MUST_CLOSE): expect_nothing, - (CLIENT, CLOSED): expect_nothing, - (SERVER, DONE): expect_nothing, - (SERVER, MUST_CLOSE): expect_nothing, - (SERVER, CLOSED): expect_nothing, - SEND_BODY: { - "chunked": ChunkedReader, - "content-length": ContentLengthReader, - "http/1.0": Http10Reader, - }, -} diff --git a/functions/python/data-processor/h11/_receivebuffer.py b/functions/python/data-processor/h11/_receivebuffer.py deleted file mode 100644 index e5c4e08a5..000000000 --- a/functions/python/data-processor/h11/_receivebuffer.py +++ /dev/null @@ -1,153 +0,0 @@ -import re -import sys -from typing import List, Optional, Union - -__all__ = ["ReceiveBuffer"] - - -# Operations we want to support: -# - find next \r\n or \r\n\r\n (\n or \n\n are also acceptable), -# or wait until there is one -# - read at-most-N bytes -# Goals: -# - on average, do this fast -# - worst case, do this in O(n) where n is the number of bytes processed -# Plan: -# - store bytearray, offset, how far we've searched for a separator token -# - use the how-far-we've-searched data to avoid rescanning -# - while doing a stream of uninterrupted processing, advance offset instead -# of constantly copying -# WARNING: -# - I haven't benchmarked or profiled any of this yet. -# -# Note that starting in Python 3.4, deleting the initial n bytes from a -# bytearray is amortized O(n), thanks to some excellent work by Antoine -# Martin: -# -# https://bugs.python.org/issue19087 -# -# This means that if we only supported 3.4+, we could get rid of the code here -# involving self._start and self.compress, because it's doing exactly the same -# thing that bytearray now does internally. -# -# BUT unfortunately, we still support 2.7, and reading short segments out of a -# long buffer MUST be O(bytes read) to avoid DoS issues, so we can't actually -# delete this code. Yet: -# -# https://pythonclock.org/ -# -# (Two things to double-check first though: make sure PyPy also has the -# optimization, and benchmark to make sure it's a win, since we do have a -# slightly clever thing where we delay calling compress() until we've -# processed a whole event, which could in theory be slightly more efficient -# than the internal bytearray support.) -blank_line_regex = re.compile(b"\n\r?\n", re.MULTILINE) - - -class ReceiveBuffer: - def __init__(self) -> None: - self._data = bytearray() - self._next_line_search = 0 - self._multiple_lines_search = 0 - - def __iadd__(self, byteslike: Union[bytes, bytearray]) -> "ReceiveBuffer": - self._data += byteslike - return self - - def __bool__(self) -> bool: - return bool(len(self)) - - def __len__(self) -> int: - return len(self._data) - - # for @property unprocessed_data - def __bytes__(self) -> bytes: - return bytes(self._data) - - def _extract(self, count: int) -> bytearray: - # extracting an initial slice of the data buffer and return it - out = self._data[:count] - del self._data[:count] - - self._next_line_search = 0 - self._multiple_lines_search = 0 - - return out - - def maybe_extract_at_most(self, count: int) -> Optional[bytearray]: - """ - Extract a fixed number of bytes from the buffer. - """ - out = self._data[:count] - if not out: - return None - - return self._extract(count) - - def maybe_extract_next_line(self) -> Optional[bytearray]: - """ - Extract the first line, if it is completed in the buffer. - """ - # Only search in buffer space that we've not already looked at. - search_start_index = max(0, self._next_line_search - 1) - partial_idx = self._data.find(b"\r\n", search_start_index) - - if partial_idx == -1: - self._next_line_search = len(self._data) - return None - - # + 2 is to compensate len(b"\r\n") - idx = partial_idx + 2 - - return self._extract(idx) - - def maybe_extract_lines(self) -> Optional[List[bytearray]]: - """ - Extract everything up to the first blank line, and return a list of lines. - """ - # Handle the case where we have an immediate empty line. - if self._data[:1] == b"\n": - self._extract(1) - return [] - - if self._data[:2] == b"\r\n": - self._extract(2) - return [] - - # Only search in buffer space that we've not already looked at. - match = blank_line_regex.search(self._data, self._multiple_lines_search) - if match is None: - self._multiple_lines_search = max(0, len(self._data) - 2) - return None - - # Truncate the buffer and return it. - idx = match.span(0)[-1] - out = self._extract(idx) - lines = out.split(b"\n") - - for line in lines: - if line.endswith(b"\r"): - del line[-1] - - assert lines[-2] == lines[-1] == b"" - - del lines[-2:] - - return lines - - # In theory we should wait until `\r\n` before starting to validate - # incoming data. However it's interesting to detect (very) invalid data - # early given they might not even contain `\r\n` at all (hence only - # timeout will get rid of them). - # This is not a 100% effective detection but more of a cheap sanity check - # allowing for early abort in some useful cases. - # This is especially interesting when peer is messing up with HTTPS and - # sent us a TLS stream where we were expecting plain HTTP given all - # versions of TLS so far start handshake with a 0x16 message type code. - def is_next_line_obviously_invalid_request_line(self) -> bool: - try: - # HTTP header line must not contain non-printable characters - # and should not start with a space - return self._data[0] < 0x21 - except IndexError: - return False diff --git a/functions/python/data-processor/h11/_state.py b/functions/python/data-processor/h11/_state.py deleted file mode 100644 index 3ad444b04..000000000 --- a/functions/python/data-processor/h11/_state.py +++ /dev/null @@ -1,365 +0,0 @@ -################################################################ -# The core state machine -################################################################ -# -# Rule 1: everything that affects the state machine and state transitions must -# live here in this file. As much as possible goes into the table-based -# representation, but for the bits that don't quite fit, the actual code and -# state must nonetheless live here. -# -# Rule 2: this file does not know about what role we're playing; it only knows -# about HTTP request/response cycles in the abstract. This ensures that we -# don't cheat and apply different rules to local and remote parties. -# -# -# Theory of operation -# =================== -# -# Possibly the simplest way to think about this is that we actually have 5 -# different state machines here. Yes, 5. These are: -# -# 1) The client state, with its complicated automaton (see the docs) -# 2) The server state, with its complicated automaton (see the docs) -# 3) The keep-alive state, with possible states {True, False} -# 4) The SWITCH_CONNECT state, with possible states {False, True} -# 5) The SWITCH_UPGRADE state, with possible states {False, True} -# -# For (3)-(5), the first state listed is the initial state. -# -# (1)-(3) are stored explicitly in member variables. The last -# two are stored implicitly in the pending_switch_proposals set as: -# (state of 4) == (_SWITCH_CONNECT in pending_switch_proposals) -# (state of 5) == (_SWITCH_UPGRADE in pending_switch_proposals) -# -# And each of these machines has two different kinds of transitions: -# -# a) Event-triggered -# b) State-triggered -# -# Event triggered is the obvious thing that you'd think it is: some event -# happens, and if it's the right event at the right time then a transition -# happens. But there are somewhat complicated rules for which machines can -# "see" which events. (As a rule of thumb, if a machine "sees" an event, this -# means two things: the event can affect the machine, and if the machine is -# not in a state where it expects that event then it's an error.) These rules -# are: -# -# 1) The client machine sees all h11.events objects emitted by the client. -# -# 2) The server machine sees all h11.events objects emitted by the server. -# -# It also sees the client's Request event. -# -# And sometimes, server events are annotated with a _SWITCH_* event. For -# example, we can have a (Response, _SWITCH_CONNECT) event, which is -# different from a regular Response event. -# -# 3) The keep-alive machine sees the process_keep_alive_disabled() event -# (which is derived from Request/Response events), and this event -# transitions it from True -> False, or from False -> False. There's no way -# to transition back. -# -# 4&5) The _SWITCH_* machines transition from False->True when we get a -# Request that proposes the relevant type of switch (via -# process_client_switch_proposals), and they go from True->False when we -# get a Response that has no _SWITCH_* annotation. -# -# So that's event-triggered transitions. -# -# State-triggered transitions are less standard. What they do here is couple -# the machines together. The way this works is, when certain *joint* -# configurations of states are achieved, then we automatically transition to a -# new *joint* state. So, for example, if we're ever in a joint state with -# -# client: DONE -# keep-alive: False -# -# then the client state immediately transitions to: -# -# client: MUST_CLOSE -# -# This is fundamentally different from an event-based transition, because it -# doesn't matter how we arrived at the {client: DONE, keep-alive: False} state -# -- maybe the client transitioned SEND_BODY -> DONE, or keep-alive -# transitioned True -> False. Either way, once this precondition is satisfied, -# this transition is immediately triggered. -# -# What if two conflicting state-based transitions get enabled at the same -# time? In practice there's only one case where this arises (client DONE -> -# MIGHT_SWITCH_PROTOCOL versus DONE -> MUST_CLOSE), and we resolve it by -# explicitly prioritizing the DONE -> MIGHT_SWITCH_PROTOCOL transition. -# -# Implementation -# -------------- -# -# The event-triggered transitions for the server and client machines are all -# stored explicitly in a table. Ditto for the state-triggered transitions that -# involve just the server and client state. -# -# The transitions for the other machines, and the state-triggered transitions -# that involve the other machines, are written out as explicit Python code. -# -# It'd be nice if there were some cleaner way to do all this. This isn't -# *too* terrible, but I feel like it could probably be better. -# -# WARNING -# ------- -# -# The script that generates the state machine diagrams for the docs knows how -# to read out the EVENT_TRIGGERED_TRANSITIONS and STATE_TRIGGERED_TRANSITIONS -# tables. But it can't automatically read the transitions that are written -# directly in Python code. So if you touch those, you need to also update the -# script to keep it in sync! -from typing import cast, Dict, Optional, Set, Tuple, Type, Union - -from ._events import * -from ._util import LocalProtocolError, Sentinel - -# Everything in __all__ gets re-exported as part of the h11 public API. -__all__ = [ - "CLIENT", - "SERVER", - "IDLE", - "SEND_RESPONSE", - "SEND_BODY", - "DONE", - "MUST_CLOSE", - "CLOSED", - "MIGHT_SWITCH_PROTOCOL", - "SWITCHED_PROTOCOL", - "ERROR", -] - - -class CLIENT(Sentinel, metaclass=Sentinel): - pass - - -class SERVER(Sentinel, metaclass=Sentinel): - pass - - -# States -class IDLE(Sentinel, metaclass=Sentinel): - pass - - -class SEND_RESPONSE(Sentinel, metaclass=Sentinel): - pass - - -class SEND_BODY(Sentinel, metaclass=Sentinel): - pass - - -class DONE(Sentinel, metaclass=Sentinel): - pass - - -class MUST_CLOSE(Sentinel, metaclass=Sentinel): - pass - - -class CLOSED(Sentinel, metaclass=Sentinel): - pass - - -class ERROR(Sentinel, metaclass=Sentinel): - pass - - -# Switch types -class MIGHT_SWITCH_PROTOCOL(Sentinel, metaclass=Sentinel): - pass - - -class SWITCHED_PROTOCOL(Sentinel, metaclass=Sentinel): - pass - - -class _SWITCH_UPGRADE(Sentinel, metaclass=Sentinel): - pass - - -class _SWITCH_CONNECT(Sentinel, metaclass=Sentinel): - pass - - -EventTransitionType = Dict[ - Type[Sentinel], - Dict[ - Type[Sentinel], - Dict[Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], Type[Sentinel]], - ], -] - -EVENT_TRIGGERED_TRANSITIONS: EventTransitionType = { - CLIENT: { - IDLE: {Request: SEND_BODY, ConnectionClosed: CLOSED}, - SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, - DONE: {ConnectionClosed: CLOSED}, - MUST_CLOSE: {ConnectionClosed: CLOSED}, - CLOSED: {ConnectionClosed: CLOSED}, - MIGHT_SWITCH_PROTOCOL: {}, - SWITCHED_PROTOCOL: {}, - ERROR: {}, - }, - SERVER: { - IDLE: { - ConnectionClosed: CLOSED, - Response: SEND_BODY, - # Special case: server sees client Request events, in this form - (Request, CLIENT): SEND_RESPONSE, - }, - SEND_RESPONSE: { - InformationalResponse: SEND_RESPONSE, - Response: SEND_BODY, - (InformationalResponse, _SWITCH_UPGRADE): SWITCHED_PROTOCOL, - (Response, _SWITCH_CONNECT): SWITCHED_PROTOCOL, - }, - SEND_BODY: {Data: SEND_BODY, EndOfMessage: DONE}, - DONE: {ConnectionClosed: CLOSED}, - MUST_CLOSE: {ConnectionClosed: CLOSED}, - CLOSED: {ConnectionClosed: CLOSED}, - SWITCHED_PROTOCOL: {}, - ERROR: {}, - }, -} - -StateTransitionType = Dict[ - Tuple[Type[Sentinel], Type[Sentinel]], Dict[Type[Sentinel], Type[Sentinel]] -] - -# NB: there are also some special-case state-triggered transitions hard-coded -# into _fire_state_triggered_transitions below. -STATE_TRIGGERED_TRANSITIONS: StateTransitionType = { - # (Client state, Server state) -> new states - # Protocol negotiation - (MIGHT_SWITCH_PROTOCOL, SWITCHED_PROTOCOL): {CLIENT: SWITCHED_PROTOCOL}, - # Socket shutdown - (CLOSED, DONE): {SERVER: MUST_CLOSE}, - (CLOSED, IDLE): {SERVER: MUST_CLOSE}, - (ERROR, DONE): {SERVER: MUST_CLOSE}, - (DONE, CLOSED): {CLIENT: MUST_CLOSE}, - (IDLE, CLOSED): {CLIENT: MUST_CLOSE}, - (DONE, ERROR): {CLIENT: MUST_CLOSE}, -} - - -class ConnectionState: - def __init__(self) -> None: - # Extra bits of state that don't quite fit into the state model. - - # If this is False then it enables the automatic DONE -> MUST_CLOSE - # transition. Don't set this directly; call .keep_alive_disabled() - self.keep_alive = True - - # This is a subset of {UPGRADE, CONNECT}, containing the proposals - # made by the client for switching protocols. - self.pending_switch_proposals: Set[Type[Sentinel]] = set() - - self.states: Dict[Type[Sentinel], Type[Sentinel]] = {CLIENT: IDLE, SERVER: IDLE} - - def process_error(self, role: Type[Sentinel]) -> None: - self.states[role] = ERROR - self._fire_state_triggered_transitions() - - def process_keep_alive_disabled(self) -> None: - self.keep_alive = False - self._fire_state_triggered_transitions() - - def process_client_switch_proposal(self, switch_event: Type[Sentinel]) -> None: - self.pending_switch_proposals.add(switch_event) - self._fire_state_triggered_transitions() - - def process_event( - self, - role: Type[Sentinel], - event_type: Type[Event], - server_switch_event: Optional[Type[Sentinel]] = None, - ) -> None: - _event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]] = event_type - if server_switch_event is not None: - assert role is SERVER - if server_switch_event not in self.pending_switch_proposals: - raise LocalProtocolError( - "Received server _SWITCH_UPGRADE event without a pending proposal" - ) - _event_type = (event_type, server_switch_event) - if server_switch_event is None and _event_type is Response: - self.pending_switch_proposals = set() - self._fire_event_triggered_transitions(role, _event_type) - # Special case: the server state does get to see Request - # events. - if _event_type is Request: - assert role is CLIENT - self._fire_event_triggered_transitions(SERVER, (Request, CLIENT)) - self._fire_state_triggered_transitions() - - def _fire_event_triggered_transitions( - self, - role: Type[Sentinel], - event_type: Union[Type[Event], Tuple[Type[Event], Type[Sentinel]]], - ) -> None: - state = self.states[role] - try: - new_state = EVENT_TRIGGERED_TRANSITIONS[role][state][event_type] - except KeyError: - event_type = cast(Type[Event], event_type) - raise LocalProtocolError( - "can't handle event type {} when role={} and state={}".format( - event_type.__name__, role, self.states[role] - ) - ) from None - self.states[role] = new_state - - def _fire_state_triggered_transitions(self) -> None: - # We apply these rules repeatedly until converging on a fixed point - while True: - start_states = dict(self.states) - - # It could happen that both these special-case transitions are - # enabled at the same time: - # - # DONE -> MIGHT_SWITCH_PROTOCOL - # DONE -> MUST_CLOSE - # - # For example, this will always be true of a HTTP/1.0 client - # requesting CONNECT. If this happens, the protocol switch takes - # priority. From there the client will either go to - # SWITCHED_PROTOCOL, in which case it's none of our business when - # they close the connection, or else the server will deny the - # request, in which case the client will go back to DONE and then - # from there to MUST_CLOSE. - if self.pending_switch_proposals: - if self.states[CLIENT] is DONE: - self.states[CLIENT] = MIGHT_SWITCH_PROTOCOL - - if not self.pending_switch_proposals: - if self.states[CLIENT] is MIGHT_SWITCH_PROTOCOL: - self.states[CLIENT] = DONE - - if not self.keep_alive: - for role in (CLIENT, SERVER): - if self.states[role] is DONE: - self.states[role] = MUST_CLOSE - - # Tabular state-triggered transitions - joint_state = (self.states[CLIENT], self.states[SERVER]) - changes = STATE_TRIGGERED_TRANSITIONS.get(joint_state, {}) - self.states.update(changes) - - if self.states == start_states: - # Fixed point reached - return - - def start_next_cycle(self) -> None: - if self.states != {CLIENT: DONE, SERVER: DONE}: - raise LocalProtocolError( - f"not in a reusable state. self.states={self.states}" - ) - # Can't reach DONE/DONE with any of these active, but still, let's be - # sure. - assert self.keep_alive - assert not self.pending_switch_proposals - self.states = {CLIENT: IDLE, SERVER: IDLE} diff --git a/functions/python/data-processor/h11/_util.py b/functions/python/data-processor/h11/_util.py deleted file mode 100644 index 671844529..000000000 --- a/functions/python/data-processor/h11/_util.py +++ /dev/null @@ -1,135 +0,0 @@ -from typing import Any, Dict, NoReturn, Pattern, Tuple, Type, TypeVar, Union - -__all__ = [ - "ProtocolError", - "LocalProtocolError", - "RemoteProtocolError", - "validate", - "bytesify", -] - - -class ProtocolError(Exception): - """Exception indicating a violation of the HTTP/1.1 protocol. - - This as an abstract base class, with two concrete base classes: - :exc:`LocalProtocolError`, which indicates that you tried to do something - that HTTP/1.1 says is illegal, and :exc:`RemoteProtocolError`, which - indicates that the remote peer tried to do something that HTTP/1.1 says is - illegal. See :ref:`error-handling` for details. - - In addition to the normal :exc:`Exception` features, it has one attribute: - - .. attribute:: error_status_hint - - This gives a suggestion as to what status code a server might use if - this error occurred as part of a request. - - For a :exc:`RemoteProtocolError`, this is useful as a suggestion for - how you might want to respond to a misbehaving peer, if you're - implementing a server. - - For a :exc:`LocalProtocolError`, this can be taken as a suggestion for - how your peer might have responded to *you* if h11 had allowed you to - continue. - - The default is 400 Bad Request, a generic catch-all for protocol - violations. - - """ - - def __init__(self, msg: str, error_status_hint: int = 400) -> None: - if type(self) is ProtocolError: - raise TypeError("tried to directly instantiate ProtocolError") - Exception.__init__(self, msg) - self.error_status_hint = error_status_hint - - -# Strategy: there are a number of public APIs where a LocalProtocolError can -# be raised (send(), all the different event constructors, ...), and only one -# public API where RemoteProtocolError can be raised -# (receive_data()). Therefore we always raise LocalProtocolError internally, -# and then receive_data will translate this into a RemoteProtocolError. -# -# Internally: -# LocalProtocolError is the generic "ProtocolError". -# Externally: -# LocalProtocolError is for local errors and RemoteProtocolError is for -# remote errors. -class LocalProtocolError(ProtocolError): - def _reraise_as_remote_protocol_error(self) -> NoReturn: - # After catching a LocalProtocolError, use this method to re-raise it - # as a RemoteProtocolError. This method must be called from inside an - # except: block. - # - # An easy way to get an equivalent RemoteProtocolError is just to - # modify 'self' in place. - self.__class__ = RemoteProtocolError # type: ignore - # But the re-raising is somewhat non-trivial -- you might think that - # now that we've modified the in-flight exception object, that just - # doing 'raise' to re-raise it would be enough. But it turns out that - # this doesn't work, because Python tracks the exception type - # (exc_info[0]) separately from the exception object (exc_info[1]), - # and we only modified the latter. So we really do need to re-raise - # the new type explicitly. - # On py3, the traceback is part of the exception object, so our - # in-place modification preserved it and we can just re-raise: - raise self - - -class RemoteProtocolError(ProtocolError): - pass - - -def validate( - regex: Pattern[bytes], data: bytes, msg: str = "malformed data", *format_args: Any -) -> Dict[str, bytes]: - match = regex.fullmatch(data) - if not match: - if format_args: - msg = msg.format(*format_args) - raise LocalProtocolError(msg) - return match.groupdict() - - -# Sentinel values -# -# - Inherit identity-based comparison and hashing from object -# - Have a nice repr -# - Have a *bonus property*: type(sentinel) is sentinel -# -# The bonus property is useful if you want to take the return value from -# next_event() and do some sort of dispatch based on type(event). - -_T_Sentinel = TypeVar("_T_Sentinel", bound="Sentinel") - - -class Sentinel(type): - def __new__( - cls: Type[_T_Sentinel], - name: str, - bases: Tuple[type, ...], - namespace: Dict[str, Any], - **kwds: Any - ) -> _T_Sentinel: - assert bases == (Sentinel,) - v = super().__new__(cls, name, bases, namespace, **kwds) - v.__class__ = v # type: ignore - return v - - def __repr__(self) -> str: - return self.__name__ - - -# Used for methods, request targets, HTTP versions, header names, and header -# values. Accepts ascii-strings, or bytes/bytearray/memoryview/..., and always -# returns bytes. -def bytesify(s: Union[bytes, bytearray, memoryview, int, str]) -> bytes: - # Fast-path: - if type(s) is bytes: - return s - if isinstance(s, str): - s = s.encode("ascii") - if isinstance(s, int): - raise TypeError("expected bytes-like object, not int") - return bytes(s) diff --git a/functions/python/data-processor/h11/_version.py b/functions/python/data-processor/h11/_version.py deleted file mode 100644 index 76e7327b8..000000000 --- a/functions/python/data-processor/h11/_version.py +++ /dev/null @@ -1,16 +0,0 @@ -# This file must be kept very simple, because it is consumed from several -# places -- it is imported by h11/__init__.py, execfile'd by setup.py, etc. - -# We use a simple scheme: -# 1.0.0 -> 1.0.0+dev -> 1.1.0 -> 1.1.0+dev -# where the +dev versions are never released into the wild, they're just what -# we stick into the VCS in between releases. -# -# This is compatible with PEP 440: -# http://legacy.python.org/dev/peps/pep-0440/ -# via the use of the "local suffix" "+dev", which is disallowed on index -# servers and causes 1.0.0+dev to sort after plain 1.0.0, which is what we -# want. (Contrast with the special suffix 1.0.0.dev, which sorts *before* -# 1.0.0.) - -__version__ = "0.16.0" diff --git a/functions/python/data-processor/h11/_writers.py b/functions/python/data-processor/h11/_writers.py deleted file mode 100644 index 939cdb912..000000000 --- a/functions/python/data-processor/h11/_writers.py +++ /dev/null @@ -1,145 +0,0 @@ -# Code to read HTTP data -# -# Strategy: each writer takes an event + a write-some-bytes function, which is -# calls. -# -# WRITERS is a dict describing how to pick a reader. It maps states to either: -# - a writer -# - or, for body writers, a dict of framin-dependent writer factories - -from typing import Any, Callable, Dict, List, Tuple, Type, Union - -from ._events import Data, EndOfMessage, Event, InformationalResponse, Request, Response -from ._headers import Headers -from ._state import CLIENT, IDLE, SEND_BODY, SEND_RESPONSE, SERVER -from ._util import LocalProtocolError, Sentinel - -__all__ = ["WRITERS"] - -Writer = Callable[[bytes], Any] - - -def write_headers(headers: Headers, write: Writer) -> None: - # "Since the Host field-value is critical information for handling a - # request, a user agent SHOULD generate Host as the first header field - # following the request-line." - RFC 7230 - raw_items = headers._full_items - for raw_name, name, value in raw_items: - if name == b"host": - write(b"%s: %s\r\n" % (raw_name, value)) - for raw_name, name, value in raw_items: - if name != b"host": - write(b"%s: %s\r\n" % (raw_name, value)) - write(b"\r\n") - - -def write_request(request: Request, write: Writer) -> None: - if request.http_version != b"1.1": - raise LocalProtocolError("I only send HTTP/1.1") - write(b"%s %s HTTP/1.1\r\n" % (request.method, request.target)) - write_headers(request.headers, write) - - -# Shared between InformationalResponse and Response -def write_any_response( - response: Union[InformationalResponse, Response], write: Writer -) -> None: - if response.http_version != b"1.1": - raise LocalProtocolError("I only send HTTP/1.1") - status_bytes = str(response.status_code).encode("ascii") - # We don't bother sending ascii status messages like "OK"; they're - # optional and ignored by the protocol. (But the space after the numeric - # status code is mandatory.) - # - # XX FIXME: could at least make an effort to pull out the status message - # from stdlib's http.HTTPStatus table. Or maybe just steal their enums - # (either by import or copy/paste). We already accept them as status codes - # since they're of type IntEnum < int. - write(b"HTTP/1.1 %s %s\r\n" % (status_bytes, response.reason)) - write_headers(response.headers, write) - - -class BodyWriter: - def __call__(self, event: Event, write: Writer) -> None: - if type(event) is Data: - self.send_data(event.data, write) - elif type(event) is EndOfMessage: - self.send_eom(event.headers, write) - else: # pragma: no cover - assert False - - def send_data(self, data: bytes, write: Writer) -> None: - pass - - def send_eom(self, headers: Headers, write: Writer) -> None: - pass - - -# -# These are all careful not to do anything to 'data' except call len(data) and -# write(data). This allows us to transparently pass-through funny objects, -# like placeholder objects referring to files on disk that will be sent via -# sendfile(2). -# -class ContentLengthWriter(BodyWriter): - def __init__(self, length: int) -> None: - self._length = length - - def send_data(self, data: bytes, write: Writer) -> None: - self._length -= len(data) - if self._length < 0: - raise LocalProtocolError("Too much data for declared Content-Length") - write(data) - - def send_eom(self, headers: Headers, write: Writer) -> None: - if self._length != 0: - raise LocalProtocolError("Too little data for declared Content-Length") - if headers: - raise LocalProtocolError("Content-Length and trailers don't mix") - - -class ChunkedWriter(BodyWriter): - def send_data(self, data: bytes, write: Writer) -> None: - # if we encoded 0-length data in the naive way, it would look like an - # end-of-message. - if not data: - return - write(b"%x\r\n" % len(data)) - write(data) - write(b"\r\n") - - def send_eom(self, headers: Headers, write: Writer) -> None: - write(b"0\r\n") - write_headers(headers, write) - - -class Http10Writer(BodyWriter): - def send_data(self, data: bytes, write: Writer) -> None: - write(data) - - def send_eom(self, headers: Headers, write: Writer) -> None: - if headers: - raise LocalProtocolError("can't send trailers to HTTP/1.0 client") - # no need to close the socket ourselves, that will be taken care of by - # Connection: close machinery - - -WritersType = Dict[ - Union[Tuple[Type[Sentinel], Type[Sentinel]], Type[Sentinel]], - Union[ - Dict[str, Type[BodyWriter]], - Callable[[Union[InformationalResponse, Response], Writer], None], - Callable[[Request, Writer], None], - ], -] - -WRITERS: WritersType = { - (CLIENT, IDLE): write_request, - (SERVER, IDLE): write_any_response, - (SERVER, SEND_RESPONSE): write_any_response, - SEND_BODY: { - "chunked": ChunkedWriter, - "content-length": ContentLengthWriter, - "http/1.0": Http10Writer, - }, -} diff --git a/functions/python/data-processor/h11/py.typed b/functions/python/data-processor/h11/py.typed deleted file mode 100644 index f5642f79f..000000000 --- a/functions/python/data-processor/h11/py.typed +++ /dev/null @@ -1 +0,0 @@ -Marker diff --git a/functions/python/data-processor/idna-3.18.dist-info/INSTALLER b/functions/python/data-processor/idna-3.18.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/idna-3.18.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/idna-3.18.dist-info/METADATA b/functions/python/data-processor/idna-3.18.dist-info/METADATA deleted file mode 100644 index 6c4bf89b2..000000000 --- a/functions/python/data-processor/idna-3.18.dist-info/METADATA +++ /dev/null @@ -1,155 +0,0 @@ -Metadata-Version: 2.4 -Name: idna -Version: 3.18 -Summary: Internationalized Domain Names in Applications (IDNA) -Author-email: Kim Davies -Requires-Python: >=3.9 -Description-Content-Type: text/markdown -License-Expression: BSD-3-Clause -Classifier: Development Status :: 5 - Production/Stable -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: System Administrators -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Internet :: Name Service (DNS) -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Topic :: Utilities -License-File: LICENSE.md -Requires-Dist: ruff >= 0.6.2 ; extra == "all" -Requires-Dist: mypy >= 1.11.2 ; extra == "all" -Requires-Dist: pytest >= 8.3.2 ; extra == "all" -Project-URL: Changelog, https://github.com/kjd/idna/blob/master/HISTORY.md -Project-URL: Issue tracker, https://github.com/kjd/idna/issues -Project-URL: Source, https://github.com/kjd/idna -Provides-Extra: all - -# Internationalized Domain Names in Applications (IDNA) - -Support for [Internationalized Domain Names in Applications -(IDNA)](https://tools.ietf.org/html/rfc5891) and [Unicode IDNA -Compatibility Processing](https://unicode.org/reports/tr46/). It -supersedes the standard library's `encodings.idna`, which only -implements the 2003 specification, offering broader script coverage and -limiting domains with known security vulnerabilities. - -## Usage - -Package may be installed from [PyPI](https://pypi.org/project/idna/) via -the typical methods (e.g. `python3 -m pip install idna`) - -For typical usage, the `encode` and `decode` functions will take a -domain name argument and perform a conversion to ASCII-compatible encoding -(known as A-labels), or to Unicode strings (known as U-labels) -respectively. - -```pycon ->>> import idna ->>> idna.encode('ドメイン.テスト') -b'xn--eckwd4c7c.xn--zckzah' ->>> print(idna.decode('xn--eckwd4c7c.xn--zckzah')) -ドメイン.テスト -``` - -Conversions can be applied at a per-label basis using the `ulabel` or -`alabel` functions for specialized use cases. - - -### Compatibility Mapping (UTS #46) - -This library provides support for [Unicode IDNA Compatibility -Processing](https://unicode.org/reports/tr46/) which normalizes input from -different potential ways a user may input a domain prior to performing the IDNA -conversion operations. This functionality, known as a -[mapping](https://tools.ietf.org/html/rfc5895), is considered by the -specification to be a local user-interface issue distinct from IDNA -conversion functionality. - -For example, "Königsgäßchen" is not a permissible label as capital letters -are not allowed. UTS 46 will convert this into lower case prior to applying -the IDNA conversion. - -```pycon ->>> import idna ->>> idna.encode('Königsgäßchen') -... -idna.core.InvalidCodepoint: Codepoint U+004B at position 1 of 'Königsgäßchen' not allowed ->>> idna.encode('Königsgäßchen', uts46=True) -b'xn--knigsgchen-b4a3dun' ->>> idna.decode('xn--knigsgchen-b4a3dun') -'königsgäßchen' -``` - -When performing a decode operation for display purposes, `decode()` -accepts a `display=True` argument that leaves any `xn--` label that -fails to decode unchanged. This is useful for user interface display -where a domain is in use, the A-label form can be presented when it -is not a valid IDN. - - -## Exceptions - -All errors raised during conversion derive from the `idna.IDNAError` -base class. The more specific exceptions are: - -* `idna.IDNABidiError` — raised when a label contains an illegal - combination of left-to-right and right-to-left characters. -* `idna.InvalidCodepoint` — raised when a label contains a codepoint - that is INVALID for IDNA. -* `idna.InvalidCodepointContext` — raised when a CONTEXTO or CONTEXTJ - codepoint appears in a position whose contextual requirements are - not satisfied. - - -## Command-line tool - -The package supports command-line usage to convert domain names -between their Unicode and ASCII-compatible forms. It can be run either -as a module (`python3 -m idna`) or, once installed (such as with `uv -tool` or `pipx`), via the `idna` script: - -```bash -$ uv tool install idna -$ idna xn--e1afmkfd.xn--p1ai -пример.рф -$ idna пример.рф -xn--e1afmkfd.xn--p1ai -``` - -Mode can be specified with `-e`/`--encode` or `-d`/`--decode`, otherwise -it will be chosen automatically based on the first input. Multiple -domains can be supplied either as arguments or through standard input. -UTS #46 mapping is applied by default, which lets the tool accept -inputs that aren't strictly valid IDNA 2008 by normalising them first, -pass `--strict` to disable UTS #46. - -Conversion failures are reported on stderr together with the -offending input; processing continues with the remaining domains and -the tool exits with a non-zero status if any conversion failed. - - -## Additional Notes - -* **Version support**. This library supports Python 3.9 and higher. - As this library serves as a low-level toolkit for a variety of - applications, we strive to support all versions of Python that are - not beyond end-of-life. - -* **Emoji**. It is an occasional request to support emoji domains in - this library. Encoding of symbols like emoji is expressly prohibited by - the IDNA technical standard, and emoji domains are broadly phased - out across the domain industry due to associated security risks. - -* **Regenerating lookup tables**. The IDNA and UTS 46 functionality - relies upon pre-calculated lookup tables, generated using the - `idna-data` script in [`tools/`](tools/README.md). - diff --git a/functions/python/data-processor/idna-3.18.dist-info/RECORD b/functions/python/data-processor/idna-3.18.dist-info/RECORD deleted file mode 100644 index f646669a4..000000000 --- a/functions/python/data-processor/idna-3.18.dist-info/RECORD +++ /dev/null @@ -1,28 +0,0 @@ -../../bin/idna,sha256=oYSi4_yQRE322GUNQ4DtCFgFmvdM0g2ZcWWbcSBfH1s,183 -idna-3.18.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -idna-3.18.dist-info/METADATA,sha256=Rt_m5axGLQ9oDs2avPZugptqIzSCS02eOXmzETXK8oE,6119 -idna-3.18.dist-info/RECORD,, -idna-3.18.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -idna-3.18.dist-info/entry_points.txt,sha256=7H3nGOHap3jnLE5e7q7Ywr9Vq8axB7WIj5-C_4N2vhw,38 -idna-3.18.dist-info/licenses/LICENSE.md,sha256=GppPDj1HmickDd1ZqRN6ZqtKD539yMphiMwL_YUYfwQ,1541 -idna/__init__.py,sha256=MPqNDLZbXqGaNdXxAFhiqFPKEQXju2jNQhCey6-5eJM,868 -idna/__main__.py,sha256=4JMK66Wj4uLZTKbF-sT3LAxOsr6buig77PmOkJCRRxw,83 -idna/__pycache__/__init__.cpython-314.pyc,, -idna/__pycache__/__main__.cpython-314.pyc,, -idna/__pycache__/cli.cpython-314.pyc,, -idna/__pycache__/codec.cpython-314.pyc,, -idna/__pycache__/compat.cpython-314.pyc,, -idna/__pycache__/core.cpython-314.pyc,, -idna/__pycache__/idnadata.cpython-314.pyc,, -idna/__pycache__/intranges.cpython-314.pyc,, -idna/__pycache__/package_data.cpython-314.pyc,, -idna/__pycache__/uts46data.cpython-314.pyc,, -idna/cli.py,sha256=swqJLMNc8Uzs60KziNpbWnHuqlG3WRQwJSbo4n8xDAo,4139 -idna/codec.py,sha256=JRbo-f7pEkLdWeiH89Z72UR4VBYhvKDFrQBeNX6sRDE,5040 -idna/compat.py,sha256=AepA39ceRHxkfHP41-FvKW5Ki-f4PfUZ90RUMlCNdmo,1353 -idna/core.py,sha256=SfOr1xO3PoE0RDYx7bMciAnjiyjJPbPw_93AB5IUYOw,24685 -idna/idnadata.py,sha256=Af-mo8WBmkhAK6TyXKOQH88OX0mQNDKtdL7UWtQpppk,44862 -idna/intranges.py,sha256=g49scLSkqJtAhLmOODa7hVHriSjmb60tiTsEoocJdBI,1851 -idna/package_data.py,sha256=TeI94EqAFAFaXfBJwsOPUMLn2969uirPa-DaeaceAyU,21 -idna/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -idna/uts46data.py,sha256=jujNz5QqWMcJf-XYLv4X1jBvb5FlI0t6-e1mILsgbPk,234325 diff --git a/functions/python/data-processor/idna-3.18.dist-info/WHEEL b/functions/python/data-processor/idna-3.18.dist-info/WHEEL deleted file mode 100644 index d8b9936da..000000000 --- a/functions/python/data-processor/idna-3.18.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/idna-3.18.dist-info/entry_points.txt b/functions/python/data-processor/idna-3.18.dist-info/entry_points.txt deleted file mode 100644 index 59ca7ac02..000000000 --- a/functions/python/data-processor/idna-3.18.dist-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -idna=idna.cli:main - diff --git a/functions/python/data-processor/idna-3.18.dist-info/licenses/LICENSE.md b/functions/python/data-processor/idna-3.18.dist-info/licenses/LICENSE.md deleted file mode 100644 index f706835ab..000000000 --- a/functions/python/data-processor/idna-3.18.dist-info/licenses/LICENSE.md +++ /dev/null @@ -1,31 +0,0 @@ -BSD 3-Clause License - -Copyright (c) 2013-2026, Kim Davies and contributors. -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - -1. Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - -2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - -3. Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED -TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR -PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF -LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING -NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/functions/python/data-processor/idna/__init__.py b/functions/python/data-processor/idna/__init__.py deleted file mode 100644 index cfdc030a7..000000000 --- a/functions/python/data-processor/idna/__init__.py +++ /dev/null @@ -1,45 +0,0 @@ -from .core import ( - IDNABidiError, - IDNAError, - InvalidCodepoint, - InvalidCodepointContext, - alabel, - check_bidi, - check_hyphen_ok, - check_initial_combiner, - check_label, - check_nfc, - decode, - encode, - ulabel, - uts46_remap, - valid_contextj, - valid_contexto, - valid_label_length, - valid_string_length, -) -from .intranges import intranges_contain -from .package_data import __version__ - -__all__ = [ - "__version__", - "IDNABidiError", - "IDNAError", - "InvalidCodepoint", - "InvalidCodepointContext", - "alabel", - "check_bidi", - "check_hyphen_ok", - "check_initial_combiner", - "check_label", - "check_nfc", - "decode", - "encode", - "intranges_contain", - "ulabel", - "uts46_remap", - "valid_contextj", - "valid_contexto", - "valid_label_length", - "valid_string_length", -] diff --git a/functions/python/data-processor/idna/__main__.py b/functions/python/data-processor/idna/__main__.py deleted file mode 100644 index dbdd06617..000000000 --- a/functions/python/data-processor/idna/__main__.py +++ /dev/null @@ -1,6 +0,0 @@ -import sys - -from .cli import main - -if __name__ == "__main__": - sys.exit(main()) diff --git a/functions/python/data-processor/idna/cli.py b/functions/python/data-processor/idna/cli.py deleted file mode 100644 index 4acda2c0f..000000000 --- a/functions/python/data-processor/idna/cli.py +++ /dev/null @@ -1,128 +0,0 @@ -"""Command-line interface for the :mod:`idna` package. - -Invoked via ``python -m idna``. See :func:`main` for the entry point. -""" - -import argparse -import sys -from collections.abc import Iterable -from itertools import chain -from typing import IO, Optional - -from . import IDNAError, decode, encode -from .core import _alabel_prefix, _unicode_dots_re -from .package_data import __version__ - - -def _looks_like_alabel(s: str) -> bool: - """Return True if any label in ``s`` carries the ``xn--`` ACE prefix.""" - prefix = _alabel_prefix.decode("ascii") - return any(label.lower().startswith(prefix) for label in _unicode_dots_re.split(s)) - - -def _build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - prog="python -m idna", - description=( - "Convert a domain name between its Unicode (U-label) and " - "ASCII-compatible (A-label) forms. With no mode flag, the " - "direction is chosen from the first input — if it contains " - "an xn-- label the stream is decoded, otherwise it is " - "encoded — and the same mode is applied to every remaining " - "input. UTS #46 mapping is applied by default; pass " - "--strict to disable it. When no domains are given on the " - "command line and stdin is piped, one domain per line is " - "read from stdin." - ), - ) - mode = parser.add_mutually_exclusive_group() - mode.add_argument( - "-e", - "--encode", - dest="mode", - action="store_const", - const="encode", - help="Encode the input to its ASCII A-label form.", - ) - mode.add_argument( - "-d", - "--decode", - dest="mode", - action="store_const", - const="decode", - help="Decode the input from its ASCII A-label form.", - ) - parser.add_argument( - "--strict", - action="store_true", - help="Disable the default UTS #46 mapping and apply IDNA 2008 rules verbatim.", - ) - parser.add_argument( - "--version", - action="version", - version=f"idna {__version__}", - ) - parser.add_argument( - "domain", - nargs="*", - help="One or more domain names to convert. Omit to read from stdin.", - ) - return parser - - -def _iter_stdin(stream: IO[str]) -> Iterable[str]: - """Yield non-empty stripped lines from ``stream``, ignoring blanks.""" - for line in stream: - stripped = line.strip() - if stripped: - yield stripped - - -def _convert_one(domain: str, mode: str, uts46: bool) -> bool: - """Convert ``domain`` and write the result; return ``False`` on failure.""" - try: - if mode == "decode": - print(decode(domain, uts46=uts46)) - else: - print(encode(domain, uts46=uts46).decode("ascii")) - except IDNAError as err: - print(f"idna: {mode} failed for {domain!r}: {err}", file=sys.stderr) - return False - return True - - -def main(argv: Optional[list[str]] = None) -> int: - """Entry point for ``python -m idna``. - - When more than one domain is supplied (via positional arguments or - piped stdin) and no mode flag is given, the first input determines - the direction and that mode is applied uniformly to the rest. - - :param argv: Argument list excluding the program name. Defaults to - :data:`sys.argv` when ``None``. - :returns: ``0`` on success, ``1`` if any conversion fails. - """ - parser = _build_parser() - args = parser.parse_args(argv) - uts46 = not args.strict - - if args.domain: - domains: Iterable[str] = args.domain - elif not sys.stdin.isatty(): - domains = _iter_stdin(sys.stdin) - else: - parser.error("a domain argument is required when stdin is a terminal") - - iterator = iter(domains) - first = next(iterator, None) - if first is None: - return 0 - - mode = args.mode or ("decode" if _looks_like_alabel(first) else "encode") - - results = [_convert_one(domain, mode, uts46) for domain in chain([first], iterator)] - return 0 if all(results) else 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/functions/python/data-processor/idna/codec.py b/functions/python/data-processor/idna/codec.py deleted file mode 100644 index 83b42fe42..000000000 --- a/functions/python/data-processor/idna/codec.py +++ /dev/null @@ -1,159 +0,0 @@ -import codecs -from typing import Any, Optional - -from .core import IDNAError, _unicode_dots_re, alabel, decode, encode, ulabel - - -class Codec(codecs.Codec): - """Stateless IDNA 2008 codec. - - Implements the :class:`codecs.Codec` protocol so that the whole-domain - encoder (:func:`idna.encode`) and decoder (:func:`idna.decode`) are - accessible through the standard codec machinery as ``"idna2008"``. - - Only the ``"strict"`` error handler is supported; any other handler - raises :exc:`~idna.IDNAError`. - """ - - def encode(self, data: str, errors: str = "strict") -> tuple[bytes, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return b"", 0 - - return encode(data), len(data) - - def decode(self, data: bytes, errors: str = "strict") -> tuple[str, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return "", 0 - - return decode(data), len(data) - - -class IncrementalEncoder(codecs.BufferedIncrementalEncoder): - """Incremental IDNA 2008 encoder. - - Buffers a partial trailing label across calls until either the next - label separator is seen or ``final=True``, so that streamed input is - encoded one whole label at a time. Any of the four Unicode label - separators (``U+002E``, ``U+3002``, ``U+FF0E``, ``U+FF61``) ends a - label; the result always uses ``U+002E`` as the separator. - - Only the ``"strict"`` error handler is supported. - """ - - def _buffer_encode(self, data: str, errors: str, final: bool) -> tuple[bytes, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return b"", 0 - - labels = _unicode_dots_re.split(data) - trailing_dot = b"" - if labels: - if not labels[-1]: - trailing_dot = b"." - del labels[-1] - elif not final: - # Keep potentially unfinished label until the next call - del labels[-1] - if labels: - trailing_dot = b"." - - result = [] - size = 0 - for label in labels: - result.append(alabel(label)) - if size: - size += 1 - size += len(label) - - # Join with U+002E - result_bytes = b".".join(result) + trailing_dot - size += len(trailing_dot) - return result_bytes, size - - -class IncrementalDecoder(codecs.BufferedIncrementalDecoder): - """Incremental IDNA 2008 decoder. - - Buffers a partial trailing label across calls until either the next - label separator is seen or ``final=True``, so that streamed input is - decoded one whole label at a time. - - Only the ``"strict"`` error handler is supported. - """ - - def _buffer_decode(self, data: Any, errors: str, final: bool) -> tuple[str, int]: # ty: ignore[invalid-method-override] - if errors != "strict": - raise IDNAError(f'Unsupported error handling "{errors}"') - - if not data: - return ("", 0) - - if not isinstance(data, str): - data = str(data, "ascii") - - labels = _unicode_dots_re.split(data) - trailing_dot = "" - if labels: - if not labels[-1]: - trailing_dot = "." - del labels[-1] - elif not final: - # Keep potentially unfinished label until the next call - del labels[-1] - if labels: - trailing_dot = "." - - result = [] - size = 0 - for label in labels: - result.append(ulabel(label)) - if size: - size += 1 - size += len(label) - - result_str = ".".join(result) + trailing_dot - size += len(trailing_dot) - return (result_str, size) - - -class StreamWriter(Codec, codecs.StreamWriter): - pass - - -class StreamReader(Codec, codecs.StreamReader): - pass - - -def search_function(name: str) -> Optional[codecs.CodecInfo]: - """Codec search function registered with :mod:`codecs`. - - Returns a :class:`codecs.CodecInfo` for the ``"idna2008"`` codec name - so that ``str.encode("idna2008")`` and ``bytes.decode("idna2008")`` - invoke the IDNA 2008 codec defined in this module. - - :param name: The codec name being looked up. - :returns: A :class:`codecs.CodecInfo` instance if ``name`` is - ``"idna2008"``, otherwise ``None``. - """ - if name != "idna2008": - return None - return codecs.CodecInfo( - name=name, - encode=Codec().encode, - decode=Codec().decode, # type: ignore - incrementalencoder=IncrementalEncoder, - incrementaldecoder=IncrementalDecoder, - streamwriter=StreamWriter, - streamreader=StreamReader, - ) - - -codecs.register(search_function) diff --git a/functions/python/data-processor/idna/compat.py b/functions/python/data-processor/idna/compat.py deleted file mode 100644 index 1d01e3d97..000000000 --- a/functions/python/data-processor/idna/compat.py +++ /dev/null @@ -1,41 +0,0 @@ -from typing import Any, Union - -from .core import decode, encode - - -def ToASCII(label: str) -> bytes: - """Compatibility shim for :rfc:`3490` ``ToASCII``. - - Delegates to :func:`idna.encode` (IDNA 2008). Provided to ease porting - of code written against the legacy :mod:`encodings.idna` API; new code - should call :func:`idna.encode` directly. - - :param label: The label or domain to encode. - :returns: The encoded form as ASCII :class:`bytes`. - """ - return encode(label) - - -def ToUnicode(label: Union[bytes, bytearray]) -> str: - """Compatibility shim for :rfc:`3490` ``ToUnicode``. - - Delegates to :func:`idna.decode` (IDNA 2008). Provided to ease porting - of code written against the legacy :mod:`encodings.idna` API; new code - should call :func:`idna.decode` directly. - - :param label: The label or domain to decode. - :returns: The decoded Unicode form. - """ - return decode(label) - - -def nameprep(s: Any) -> None: - """Stub for :rfc:`3491` Nameprep, which is not used by IDNA 2008. - - IDNA 2008 (:rfc:`5891`) replaces Nameprep with the per-codepoint - validity classes from :rfc:`5892`; this function exists only to - return a clear error if legacy code attempts to call it. - - :raises NotImplementedError: Always. - """ - raise NotImplementedError("IDNA 2008 does not utilise nameprep protocol") diff --git a/functions/python/data-processor/idna/core.py b/functions/python/data-processor/idna/core.py deleted file mode 100644 index 1ccbd1f33..000000000 --- a/functions/python/data-processor/idna/core.py +++ /dev/null @@ -1,648 +0,0 @@ -import bisect -import re -import unicodedata -import warnings -from typing import Optional, Union - -from . import idnadata -from .intranges import intranges_contain - -_virama_combining_class = 9 -_alabel_prefix = b"xn--" -_max_input_length = 1024 -_unicode_dots_re = re.compile("[\u002e\u3002\uff0e\uff61]") - - -# Bidi category sets from RFC 5893, hoisted out of the per-codepoint loop -_bidi_rtl_first = frozenset({"R", "AL"}) -_bidi_rtl_categories = frozenset({"R", "AL", "AN"}) -_bidi_rtl_allowed = frozenset({"R", "AL", "AN", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) -_bidi_rtl_valid_ending = frozenset({"R", "AL", "EN", "AN"}) -_bidi_rtl_numeric = frozenset({"AN", "EN"}) -_bidi_ltr_allowed = frozenset({"L", "EN", "ES", "CS", "ET", "ON", "BN", "NSM"}) -_bidi_ltr_valid_ending = frozenset({"L", "EN"}) -_bidi_joiner_l_or_d = frozenset({"L", "D"}) -_bidi_joiner_r_or_d = frozenset({"R", "D"}) - - -def _joining_type(cp: int) -> Optional[str]: - for jt, ranges in idnadata.joining_types.items(): - if intranges_contain(cp, ranges): - return jt - return None - - -class IDNAError(UnicodeError): - """Base exception for all IDNA-encoding related problems""" - - -class IDNABidiError(IDNAError): - """Exception when bidirectional requirements are not satisfied""" - - -class InvalidCodepoint(IDNAError): - """Exception when a disallowed or unallocated codepoint is used""" - - -class InvalidCodepointContext(IDNAError): - """Exception when the codepoint is not valid in the context it is used""" - - -def _combining_class(cp: int) -> int: - v = unicodedata.combining(chr(cp)) - if v == 0 and not unicodedata.name(chr(cp)): - raise ValueError("Unknown character in unicodedata") - return v - - -def _is_script(cp: str, script: str) -> bool: - return intranges_contain(ord(cp), idnadata.scripts[script]) - - -def _punycode(s: str) -> bytes: - return s.encode("punycode") - - -def _unot(s: int) -> str: - return f"U+{s:04X}" - - -def valid_label_length(label: Union[bytes, str]) -> bool: - """Check that a label does not exceed the maximum permitted length. - - Per :rfc:`1035` (and :rfc:`5891` §4.2.4) a DNS label must not exceed - 63 octets. The argument may be either a :class:`str` (a U-label, where - length is measured in characters) or :class:`bytes` (an A-label, where - length is measured in octets). - - :param label: The label to check. - :returns: ``True`` if the label is within the length limit, otherwise - ``False``. - """ - return len(label) <= 63 - - -def valid_string_length(domain: Union[bytes, str], trailing_dot: bool) -> bool: - """Check that a full domain name does not exceed the maximum length. - - Per :rfc:`1035`, a domain name is limited to 253 octets when no trailing - dot is present, or 254 octets when one is included. - - :param domain: The full (possibly multi-label) domain name. - :param trailing_dot: ``True`` if ``domain`` includes a trailing ``.``. - :returns: ``True`` if the domain is within the length limit, otherwise - ``False``. - """ - return len(domain) <= (254 if trailing_dot else 253) - - -def check_bidi(label: str, check_ltr: bool = False) -> bool: - """Validate the Bidi Rule from :rfc:`5893` for a single label. - - The Bidi Rule constrains how bidirectional characters (Hebrew, Arabic, - etc.) may appear within a label. By default the check is only applied - when the label contains at least one right-to-left character (Unicode - bidirectional categories ``R``, ``AL``, or ``AN``); set ``check_ltr`` - to ``True`` to apply it to LTR-only labels as well. - - :param label: The label to validate, as a Unicode string. - :param check_ltr: If ``True``, apply the rules even when the label - contains no RTL characters. - :returns: ``True`` if the label satisfies the Bidi Rule. - :raises IDNABidiError: If any of Bidi Rule conditions 1-6 are violated, - or if the directional category of a codepoint cannot be determined. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - # Bidi rules should only be applied if string contains RTL characters - bidi_label = False - for idx, cp in enumerate(label, 1): - direction = unicodedata.bidirectional(cp) - if direction == "": - # String likely comes from a newer version of Unicode - raise IDNABidiError(f"Unknown directionality in label {label!r} at position {idx}") - if direction in _bidi_rtl_categories: - bidi_label = True - if not bidi_label and not check_ltr: - return True - - # Bidi rule 1 - direction = unicodedata.bidirectional(label[0]) - if direction in _bidi_rtl_first: - rtl = True - elif direction == "L": - rtl = False - else: - raise IDNABidiError(f"First codepoint in label {label!r} must be directionality L, R or AL") - - valid_ending = False - number_type: Optional[str] = None - for idx, cp in enumerate(label, 1): - direction = unicodedata.bidirectional(cp) - - if rtl: - # Bidi rule 2 - if direction not in _bidi_rtl_allowed: - raise IDNABidiError(f"Invalid direction for codepoint at position {idx} in a right-to-left label") - # Bidi rule 3 - if direction in _bidi_rtl_valid_ending: - valid_ending = True - elif direction != "NSM": - valid_ending = False - # Bidi rule 4 - if direction in _bidi_rtl_numeric: - if not number_type: - number_type = direction - elif number_type != direction: - raise IDNABidiError("Can not mix numeral types in a right-to-left label") - else: - # Bidi rule 5 - if direction not in _bidi_ltr_allowed: - raise IDNABidiError(f"Invalid direction for codepoint at position {idx} in a left-to-right label") - # Bidi rule 6 - if direction in _bidi_ltr_valid_ending: - valid_ending = True - elif direction != "NSM": - valid_ending = False - - if not valid_ending: - raise IDNABidiError("Label ends with illegal codepoint directionality") - - return True - - -def check_initial_combiner(label: str) -> bool: - """Reject labels that begin with a combining mark. - - Per :rfc:`5891` §4.2.3.2 a label must not start with a character of - Unicode general category ``M`` (Mark). - - :param label: The label to check. - :returns: ``True`` if the first character is not a combining mark. - :raises IDNAError: If the label begins with a combining character. - """ - if unicodedata.category(label[0])[0] == "M": - raise IDNAError("Label begins with an illegal combining character") - return True - - -def check_hyphen_ok(label: str) -> bool: - """Validate the hyphen restrictions for a label. - - Per :rfc:`5891` §4.2.3.1 a label must not start or end with a hyphen - (``U+002D``), and must not have hyphens in both the third and fourth - positions (the prefix reserved for A-labels). - - :param label: The label to check. - :returns: ``True`` if the hyphen restrictions are satisfied. - :raises IDNAError: If any of the hyphen restrictions are violated. - """ - if label[2:4] == "--": - raise IDNAError("Label has disallowed hyphens in 3rd and 4th position") - if label[0] == "-" or label[-1] == "-": - raise IDNAError("Label must not start or end with a hyphen") - return True - - -def check_nfc(label: str) -> None: - """Require that a label is in Unicode Normalization Form C. - - :param label: The label to check. - :raises IDNAError: If ``label`` differs from its NFC normalisation. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - if unicodedata.normalize("NFC", label) != label: - raise IDNAError("Label must be in Normalization Form C") - - -def valid_contextj(label: str, pos: int) -> bool: - """Validate the CONTEXTJ rules from :rfc:`5892` Appendix A. - - These rules govern the contextual use of the joiner codepoints - ``U+200C`` (ZERO WIDTH NON-JOINER, Appendix A.1) and ``U+200D`` - (ZERO WIDTH JOINER, Appendix A.2) within a label. - - :param label: The label containing the codepoint. - :param pos: Index of the joiner codepoint within ``label``. - :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTJ - rule, ``False`` otherwise (including when the codepoint at - ``pos`` is not a recognised joiner). - :raises ValueError: If an adjacent codepoint has no Unicode name when - determining its combining class. - :raises IDNAError: If ``label`` exceeds the defensive input length limit. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - cp_value = ord(label[pos]) - - if cp_value == 0x200C: - if pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class: - return True - - ok = False - for i in range(pos - 1, -1, -1): - joining_type = _joining_type(ord(label[i])) - if joining_type == "T": - continue - if joining_type in _bidi_joiner_l_or_d: - ok = True - break - break - - if not ok: - return False - - ok = False - for i in range(pos + 1, len(label)): - joining_type = _joining_type(ord(label[i])) - if joining_type == "T": - continue - if joining_type in _bidi_joiner_r_or_d: - ok = True - break - break - return ok - - if cp_value == 0x200D: - return pos > 0 and _combining_class(ord(label[pos - 1])) == _virama_combining_class - - return False - - -def valid_contexto(label: str, pos: int, exception: bool = False) -> bool: - """Validate the CONTEXTO rules from :rfc:`5892` Appendix A. - - Covers the contextual rules for codepoints such as MIDDLE DOT - (``U+00B7``), Greek lower numeral sign, Hebrew punctuation, Katakana - middle dot, and the Arabic-Indic / Extended Arabic-Indic digit ranges. - - :param label: The label containing the codepoint. - :param pos: Index of the codepoint within ``label``. - :param exception: Reserved for forward compatibility; currently unused. - :returns: ``True`` if the codepoint at ``pos`` satisfies its CONTEXTO - rule, ``False`` otherwise (including when the codepoint is not a - recognised CONTEXTO codepoint). - :raises IDNAError: If ``label`` exceeds the defensive input length limit. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - cp_value = ord(label[pos]) - - if cp_value == 0x00B7: - return 0 < pos < len(label) - 1 and ord(label[pos - 1]) == 0x006C and ord(label[pos + 1]) == 0x006C - - if cp_value == 0x0375: - if pos < len(label) - 1 and len(label) > 1: - return _is_script(label[pos + 1], "Greek") - return False - - if cp_value in {0x05F3, 0x05F4}: - if pos > 0: - return _is_script(label[pos - 1], "Hebrew") - return False - - if cp_value == 0x30FB: - for cp in label: - if cp == "\u30fb": - continue - if _is_script(cp, "Hiragana") or _is_script(cp, "Katakana") or _is_script(cp, "Han"): - return True - return False - - if 0x660 <= cp_value <= 0x669: - return not any(0x6F0 <= ord(cp) <= 0x06F9 for cp in label) - - if 0x6F0 <= cp_value <= 0x6F9: - return not any(0x660 <= ord(cp) <= 0x0669 for cp in label) - - return False - - -def check_label(label: Union[str, bytes, bytearray]) -> None: - """Run the full set of IDNA 2008 validity checks on a single label. - - Applies, in order: NFC normalisation (:func:`check_nfc`), hyphen - restrictions (:func:`check_hyphen_ok`), the no-leading-combiner rule - (:func:`check_initial_combiner`), per-codepoint validity (PVALID, - CONTEXTJ, CONTEXTO classes from :rfc:`5892`), and the Bidi Rule - (:func:`check_bidi`). - - :param label: The label to validate. ``bytes`` or ``bytearray`` input - is decoded as UTF-8 first. - :raises IDNAError: If the label is empty or fails a structural rule. - :raises InvalidCodepoint: If the label contains a DISALLOWED or - UNASSIGNED codepoint. - :raises InvalidCodepointContext: If a CONTEXTJ or CONTEXTO codepoint - is not valid in its context. - :raises IDNABidiError: If the Bidi Rule is violated. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - if isinstance(label, (bytes, bytearray)): - label = label.decode("utf-8") - if len(label) == 0: - raise IDNAError("Empty Label") - - # Reject on domain length rather than label length so support some UTS 46 - # use cases, still reducing processing of label contextual rules - if not valid_string_length(label, trailing_dot=True): - raise IDNAError("Label too long") - - check_nfc(label) - check_hyphen_ok(label) - check_initial_combiner(label) - - for pos, cp in enumerate(label): - cp_value = ord(cp) - if intranges_contain(cp_value, idnadata.codepoint_classes["PVALID"]): - continue - if intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTJ"]): - try: - if not valid_contextj(label, pos): - raise InvalidCodepointContext(f"Joiner {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") - except ValueError as err: - raise IDNAError( - f"Unknown codepoint adjacent to joiner {_unot(cp_value)} at position {pos + 1} in {label!r}" - ) from err - elif intranges_contain(cp_value, idnadata.codepoint_classes["CONTEXTO"]): - if not valid_contexto(label, pos): - raise InvalidCodepointContext(f"Codepoint {_unot(cp_value)} not allowed at position {pos + 1} in {label!r}") - else: - raise InvalidCodepoint(f"Codepoint {_unot(cp_value)} at position {pos + 1} of {label!r} not allowed") - - check_bidi(label) - - -def alabel(label: str) -> bytes: - """Convert a single U-label into its A-label form. - - The result is the ASCII-Compatible Encoding (ACE) form per :rfc:`5891` - §4: the label is validated, Punycode-encoded, and prefixed with - ``xn--``. Pure ASCII labels that are already valid IDNA labels are - returned unchanged (as :class:`bytes`). - - :param label: The label to convert, as a Unicode string. - :returns: The A-label as ASCII-encoded :class:`bytes`. - :raises IDNAError: If the label is invalid or the resulting A-label - exceeds 63 octets. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - try: - label_bytes = label.encode("ascii") - except UnicodeEncodeError: - pass - else: - ulabel(label_bytes) - if not valid_label_length(label_bytes): - raise IDNAError("Label too long") - return label_bytes - - check_label(label) - label_bytes = _alabel_prefix + _punycode(label) - - if not valid_label_length(label_bytes): - raise IDNAError("Label too long") - - return label_bytes - - -def ulabel(label: Union[str, bytes, bytearray]) -> str: - """Convert a single A-label into its U-label form. - - Performs the inverse of :func:`alabel`: an ``xn--``-prefixed label is - Punycode-decoded and validated. Labels that are already Unicode (or - plain ASCII without the ACE prefix) are validated and returned as a - Unicode string. - - :param label: The label to convert. ``bytes`` or ``bytearray`` input - is treated as ASCII. - :returns: The U-label as a Unicode string. - :raises IDNAError: If the label is malformed or fails validation. - """ - if len(label) > _max_input_length: - raise IDNAError("Label too long") - if not isinstance(label, (bytes, bytearray)): - try: - label_bytes = label.encode("ascii") - except UnicodeEncodeError: - check_label(label) - return label - else: - label_bytes = bytes(label) - - label_bytes = label_bytes.lower() - if label_bytes.startswith(_alabel_prefix): - label_bytes = label_bytes[len(_alabel_prefix) :] - if not label_bytes: - raise IDNAError("Malformed A-label, no Punycode eligible content found") - if label_bytes.endswith(b"-"): - raise IDNAError("A-label must not end with a hyphen") - else: - check_label(label_bytes) - return label_bytes.decode("ascii") - - try: - label = label_bytes.decode("punycode") - except UnicodeError as err: - raise IDNAError("Invalid A-label") from err - check_label(label) - return label - - -def uts46_remap(domain: str, std3_rules: bool = True, transitional: bool = False) -> str: - """Apply the UTS #46 character mapping to a domain string. - - Implements the mapping table from `UTS #46 §4 - `_: each character is kept, - replaced, or rejected based on its status (``V``, ``M``, ``D``, ``3``, - ``I``). The result is returned in Normalisation Form C. - - :param domain: The full domain name to remap. - :param std3_rules: If ``True``, apply the stricter STD3 ASCII rules - (status ``3`` codepoints raise instead of being kept or mapped). - :param transitional: If ``True``, use transitional processing (status - ``D`` codepoints are mapped instead of kept). Transitional - processing has been removed from UTS #46 and this option is - retained only for backwards compatibility. - :returns: The remapped domain, in Normalisation Form C. - :raises InvalidCodepoint: If the domain contains a disallowed - codepoint under the chosen rules. - :raises IDNAError: If ``domain`` exceeds the defensive input length limit. - """ - if len(domain) > _max_input_length: - raise IDNAError("Domain too long") - from .uts46data import uts46_replacements, uts46_starts, uts46_statuses - - output = "" - - for pos, char in enumerate(domain): - code_point = ord(char) - i = code_point if code_point < 256 else bisect.bisect_right(uts46_starts, code_point) - 1 - status = chr(uts46_statuses[i]) - replacement: Optional[str] = uts46_replacements[i] - - # UTS #46 §4: V is always valid, D is deviation (kept unless transitional), - # 3 is disallowed-STD3 (kept unmapped if std3_rules is off and no mapping). - keep_as_is = ( - status == "V" or (status == "D" and not transitional) or (status == "3" and not std3_rules and replacement is None) - ) - # M is mapped, 3-with-replacement and transitional D fall through to the - # same replacement output path. - use_replacement = replacement is not None and ( - status == "M" or (status == "3" and not std3_rules) or (status == "D" and transitional) - ) - - if keep_as_is: - output += char - elif use_replacement: - assert replacement is not None # narrowed by use_replacement - output += replacement - elif status == "I": - continue - else: - raise InvalidCodepoint(f"Codepoint {_unot(code_point)} not allowed at position {pos + 1} in {domain!r}") - - return unicodedata.normalize("NFC", output) - - -def encode( - s: Union[str, bytes, bytearray], - strict: bool = False, - uts46: bool = False, - std3_rules: bool = False, - transitional: bool = False, -) -> bytes: - """Encode a Unicode domain name into its ASCII (A-label) form. - - Splits the input on label separators (only ``U+002E`` if ``strict`` is - set; otherwise also IDEOGRAPHIC FULL STOP ``U+3002``, FULLWIDTH FULL - STOP ``U+FF0E``, and HALFWIDTH IDEOGRAPHIC FULL STOP ``U+FF61``), - encodes each label with :func:`alabel`, and rejoins them with ``.``. - Optionally pre-processes the input through :func:`uts46_remap`. - - :param s: The domain name to encode. - :param strict: If ``True``, only ``U+002E`` is recognised as a label - separator. - :param uts46: If ``True``, apply UTS #46 mapping before encoding. - :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is - ``True``. - :param transitional: Forwarded to :func:`uts46_remap` when ``uts46`` - is ``True``. Deprecated: emits a :class:`DeprecationWarning` and - will be removed in a future version. - :returns: The encoded domain as ASCII :class:`bytes`. - :raises IDNAError: If the domain is empty, contains an invalid label, - or exceeds the maximum domain length. - """ - if transitional: - warnings.warn( - "Transitional processing has been removed from UTS #46. " - "The transitional argument will be removed in a future version.", - DeprecationWarning, - stacklevel=2, - ) - if not isinstance(s, str): - try: - s = str(s, "ascii") - except (UnicodeDecodeError, TypeError) as err: - raise IDNAError("should pass a unicode string to the function rather than a byte string.") from err - if len(s) > _max_input_length: - raise IDNAError("Domain too long") - if uts46: - s = uts46_remap(s, std3_rules, transitional) - - # Reject inputs that exceed the maximum DNS domain length up-front - # to avoid expensive computation on long inputs. - if not valid_string_length(s, trailing_dot=True): - raise IDNAError("Domain too long") - - trailing_dot = False - result = [] - labels = s.split(".") if strict else _unicode_dots_re.split(s) - if not labels or labels == [""]: - raise IDNAError("Empty domain") - if labels[-1] == "": - del labels[-1] - trailing_dot = True - for label in labels: - s = alabel(label) - if s: - result.append(s) - else: - raise IDNAError("Empty label") - if trailing_dot: - result.append(b"") - s = b".".join(result) - if not valid_string_length(s, trailing_dot): - raise IDNAError("Domain too long") - return s - - -def decode( - s: Union[str, bytes, bytearray], - strict: bool = False, - uts46: bool = False, - std3_rules: bool = False, - display: bool = False, -) -> str: - """Decode an A-label-encoded domain name back to Unicode. - - Splits the input on label separators (see :func:`encode` for the - rules), decodes each label with :func:`ulabel`, and rejoins them - with ``.``. Optionally pre-processes the input through - :func:`uts46_remap`. - - :param s: The domain name to decode. - :param strict: If ``True``, only ``U+002E`` is recognised as a label - separator. - :param uts46: If ``True``, apply UTS #46 mapping before decoding. - :param std3_rules: Forwarded to :func:`uts46_remap` when ``uts46`` is - ``True``. - :param display: If ``True``, any ``xn--`` label that fails IDNA - validation is passed through unchanged (lowercased) rather than - aborting the whole call. Intended for "decode for display" - consumers (e.g. URL libraries, HTTP clients) that want to show - the user the label as it appears on the wire when it cannot be - rendered as Unicode. Matches the per-label recovery prescribed - by UTS #46 §4 and the WHATWG URL "domain to Unicode" algorithm. - :returns: The decoded domain as a Unicode string. - :raises IDNAError: If the input is not valid ASCII, contains an - invalid label, or is empty. - """ - if not isinstance(s, str): - try: - s = str(s, "ascii") - except (UnicodeDecodeError, TypeError) as err: - raise IDNAError("Invalid ASCII in A-label") from err - if len(s) > _max_input_length: - raise IDNAError("Domain too long") - if uts46: - s = uts46_remap(s, std3_rules, False) - # Reject inputs that exceed the maximum DNS domain length up-front - # to avoid expensive computation on long inputs. - if not valid_string_length(s, trailing_dot=True): - raise IDNAError("Domain too long") - trailing_dot = False - result = [] - labels = s.split(".") if strict else _unicode_dots_re.split(s) - if not labels or labels == [""]: - raise IDNAError("Empty domain") - if not labels[-1]: - del labels[-1] - trailing_dot = True - for label in labels: - try: - u = ulabel(label) - except IDNAError: - if display and label[:4].lower() == "xn--": - u = label.lower() - else: - raise - if u: - result.append(u) - else: - raise IDNAError("Empty label") - if trailing_dot: - result.append("") - return ".".join(result) diff --git a/functions/python/data-processor/idna/idnadata.py b/functions/python/data-processor/idna/idnadata.py deleted file mode 100644 index f2ab38897..000000000 --- a/functions/python/data-processor/idna/idnadata.py +++ /dev/null @@ -1,1897 +0,0 @@ -# This file is automatically generated by tools/idna-data - -__version__ = "17.0.0" - -scripts = { - "Greek": ( - 0x37000000374, - 0x37500000378, - 0x37A0000037E, - 0x37F00000380, - 0x38400000385, - 0x38600000387, - 0x3880000038B, - 0x38C0000038D, - 0x38E000003A2, - 0x3A3000003E2, - 0x3F000000400, - 0x1D2600001D2B, - 0x1D5D00001D62, - 0x1D6600001D6B, - 0x1DBF00001DC0, - 0x1F0000001F16, - 0x1F1800001F1E, - 0x1F2000001F46, - 0x1F4800001F4E, - 0x1F5000001F58, - 0x1F5900001F5A, - 0x1F5B00001F5C, - 0x1F5D00001F5E, - 0x1F5F00001F7E, - 0x1F8000001FB5, - 0x1FB600001FC5, - 0x1FC600001FD4, - 0x1FD600001FDC, - 0x1FDD00001FF0, - 0x1FF200001FF5, - 0x1FF600001FFF, - 0x212600002127, - 0xAB650000AB66, - 0x101400001018F, - 0x101A0000101A1, - 0x1D2000001D246, - ), - "Han": ( - 0x2E8000002E9A, - 0x2E9B00002EF4, - 0x2F0000002FD6, - 0x300500003006, - 0x300700003008, - 0x30210000302A, - 0x30380000303C, - 0x340000004DC0, - 0x4E000000A000, - 0xF9000000FA6E, - 0xFA700000FADA, - 0x16FE200016FE4, - 0x16FF000016FF7, - 0x200000002A6E0, - 0x2A7000002B81E, - 0x2B8200002CEAE, - 0x2CEB00002EBE1, - 0x2EBF00002EE5E, - 0x2F8000002FA1E, - 0x300000003134B, - 0x313500003347A, - ), - "Hebrew": ( - 0x591000005C8, - 0x5D0000005EB, - 0x5EF000005F5, - 0xFB1D0000FB37, - 0xFB380000FB3D, - 0xFB3E0000FB3F, - 0xFB400000FB42, - 0xFB430000FB45, - 0xFB460000FB50, - ), - "Hiragana": ( - 0x304100003097, - 0x309D000030A0, - 0x1B0010001B120, - 0x1B1320001B133, - 0x1B1500001B153, - 0x1F2000001F201, - ), - "Katakana": ( - 0x30A1000030FB, - 0x30FD00003100, - 0x31F000003200, - 0x32D0000032FF, - 0x330000003358, - 0xFF660000FF70, - 0xFF710000FF9E, - 0x1AFF00001AFF4, - 0x1AFF50001AFFC, - 0x1AFFD0001AFFF, - 0x1B0000001B001, - 0x1B1200001B123, - 0x1B1550001B156, - 0x1B1640001B168, - ), -} - - -joining_types = { - "C": ( - 0x64000000641, - 0x7FA000007FB, - 0x88300000886, - 0x180A0000180B, - 0x200D0000200E, - ), - "D": ( - 0x62000000621, - 0x62600000627, - 0x62800000629, - 0x62A0000062F, - 0x63300000640, - 0x64100000648, - 0x6490000064B, - 0x66E00000670, - 0x67800000688, - 0x69A000006C0, - 0x6C1000006C3, - 0x6CC000006CD, - 0x6CE000006CF, - 0x6D0000006D2, - 0x6FA000006FD, - 0x6FF00000700, - 0x71200000715, - 0x71A0000071E, - 0x71F00000728, - 0x7290000072A, - 0x72B0000072C, - 0x72D0000072F, - 0x74E00000759, - 0x75C0000076B, - 0x76D00000771, - 0x77200000773, - 0x77500000778, - 0x77A00000780, - 0x7CA000007EB, - 0x84100000846, - 0x84800000849, - 0x84A00000854, - 0x85500000856, - 0x86000000861, - 0x86200000866, - 0x86800000869, - 0x88600000887, - 0x8890000088E, - 0x88F00000890, - 0x8A0000008AA, - 0x8AF000008B1, - 0x8B3000008B9, - 0x8BA000008C9, - 0x180700001808, - 0x182000001879, - 0x1887000018A9, - 0x18AA000018AB, - 0xA8400000A872, - 0x10AC000010AC5, - 0x10AD300010AD7, - 0x10AD800010ADD, - 0x10ADE00010AE1, - 0x10AEB00010AEF, - 0x10B8000010B81, - 0x10B8200010B83, - 0x10B8600010B89, - 0x10B8A00010B8C, - 0x10B8D00010B8E, - 0x10B9000010B91, - 0x10BAD00010BAF, - 0x10D0100010D22, - 0x10D2300010D24, - 0x10EC300010EC5, - 0x10EC600010EC8, - 0x10F3000010F33, - 0x10F3400010F45, - 0x10F5100010F54, - 0x10F7000010F74, - 0x10F7600010F82, - 0x10FB000010FB1, - 0x10FB200010FB4, - 0x10FB800010FB9, - 0x10FBB00010FBD, - 0x10FBE00010FC0, - 0x10FC100010FC2, - 0x10FC400010FC5, - 0x10FCA00010FCB, - 0x1E9000001E944, - ), - "L": ( - 0xA8720000A873, - 0x10ACD00010ACE, - 0x10AD700010AD8, - 0x10D0000010D01, - 0x10FCB00010FCC, - ), - "R": ( - 0x62200000626, - 0x62700000628, - 0x6290000062A, - 0x62F00000633, - 0x64800000649, - 0x67100000674, - 0x67500000678, - 0x6880000069A, - 0x6C0000006C1, - 0x6C3000006CC, - 0x6CD000006CE, - 0x6CF000006D0, - 0x6D2000006D4, - 0x6D5000006D6, - 0x6EE000006F0, - 0x71000000711, - 0x7150000071A, - 0x71E0000071F, - 0x72800000729, - 0x72A0000072B, - 0x72C0000072D, - 0x72F00000730, - 0x74D0000074E, - 0x7590000075C, - 0x76B0000076D, - 0x77100000772, - 0x77300000775, - 0x7780000077A, - 0x84000000841, - 0x84600000848, - 0x8490000084A, - 0x85400000855, - 0x85600000859, - 0x86700000868, - 0x8690000086B, - 0x87000000883, - 0x88E0000088F, - 0x8AA000008AD, - 0x8AE000008AF, - 0x8B1000008B3, - 0x8B9000008BA, - 0x10AC500010AC6, - 0x10AC700010AC8, - 0x10AC900010ACB, - 0x10ACE00010AD3, - 0x10ADD00010ADE, - 0x10AE100010AE2, - 0x10AE400010AE5, - 0x10AEF00010AF0, - 0x10B8100010B82, - 0x10B8300010B86, - 0x10B8900010B8A, - 0x10B8C00010B8D, - 0x10B8E00010B90, - 0x10B9100010B92, - 0x10BA900010BAD, - 0x10D2200010D23, - 0x10EC200010EC3, - 0x10F3300010F34, - 0x10F5400010F55, - 0x10F7400010F76, - 0x10FB400010FB7, - 0x10FB900010FBB, - 0x10FBD00010FBE, - 0x10FC200010FC4, - 0x10FC900010FCA, - ), - "T": ( - 0xAD000000AE, - 0x30000000370, - 0x4830000048A, - 0x591000005BE, - 0x5BF000005C0, - 0x5C1000005C3, - 0x5C4000005C6, - 0x5C7000005C8, - 0x6100000061B, - 0x61C0000061D, - 0x64B00000660, - 0x67000000671, - 0x6D6000006DD, - 0x6DF000006E5, - 0x6E7000006E9, - 0x6EA000006EE, - 0x70F00000710, - 0x71100000712, - 0x7300000074B, - 0x7A6000007B1, - 0x7EB000007F4, - 0x7FD000007FE, - 0x8160000081A, - 0x81B00000824, - 0x82500000828, - 0x8290000082E, - 0x8590000085C, - 0x897000008A0, - 0x8CA000008E2, - 0x8E300000903, - 0x93A0000093B, - 0x93C0000093D, - 0x94100000949, - 0x94D0000094E, - 0x95100000958, - 0x96200000964, - 0x98100000982, - 0x9BC000009BD, - 0x9C1000009C5, - 0x9CD000009CE, - 0x9E2000009E4, - 0x9FE000009FF, - 0xA0100000A03, - 0xA3C00000A3D, - 0xA4100000A43, - 0xA4700000A49, - 0xA4B00000A4E, - 0xA5100000A52, - 0xA7000000A72, - 0xA7500000A76, - 0xA8100000A83, - 0xABC00000ABD, - 0xAC100000AC6, - 0xAC700000AC9, - 0xACD00000ACE, - 0xAE200000AE4, - 0xAFA00000B00, - 0xB0100000B02, - 0xB3C00000B3D, - 0xB3F00000B40, - 0xB4100000B45, - 0xB4D00000B4E, - 0xB5500000B57, - 0xB6200000B64, - 0xB8200000B83, - 0xBC000000BC1, - 0xBCD00000BCE, - 0xC0000000C01, - 0xC0400000C05, - 0xC3C00000C3D, - 0xC3E00000C41, - 0xC4600000C49, - 0xC4A00000C4E, - 0xC5500000C57, - 0xC6200000C64, - 0xC8100000C82, - 0xCBC00000CBD, - 0xCBF00000CC0, - 0xCC600000CC7, - 0xCCC00000CCE, - 0xCE200000CE4, - 0xD0000000D02, - 0xD3B00000D3D, - 0xD4100000D45, - 0xD4D00000D4E, - 0xD6200000D64, - 0xD8100000D82, - 0xDCA00000DCB, - 0xDD200000DD5, - 0xDD600000DD7, - 0xE3100000E32, - 0xE3400000E3B, - 0xE4700000E4F, - 0xEB100000EB2, - 0xEB400000EBD, - 0xEC800000ECF, - 0xF1800000F1A, - 0xF3500000F36, - 0xF3700000F38, - 0xF3900000F3A, - 0xF7100000F7F, - 0xF8000000F85, - 0xF8600000F88, - 0xF8D00000F98, - 0xF9900000FBD, - 0xFC600000FC7, - 0x102D00001031, - 0x103200001038, - 0x10390000103B, - 0x103D0000103F, - 0x10580000105A, - 0x105E00001061, - 0x107100001075, - 0x108200001083, - 0x108500001087, - 0x108D0000108E, - 0x109D0000109E, - 0x135D00001360, - 0x171200001715, - 0x173200001734, - 0x175200001754, - 0x177200001774, - 0x17B4000017B6, - 0x17B7000017BE, - 0x17C6000017C7, - 0x17C9000017D4, - 0x17DD000017DE, - 0x180B0000180E, - 0x180F00001810, - 0x188500001887, - 0x18A9000018AA, - 0x192000001923, - 0x192700001929, - 0x193200001933, - 0x19390000193C, - 0x1A1700001A19, - 0x1A1B00001A1C, - 0x1A5600001A57, - 0x1A5800001A5F, - 0x1A6000001A61, - 0x1A6200001A63, - 0x1A6500001A6D, - 0x1A7300001A7D, - 0x1A7F00001A80, - 0x1AB000001ADE, - 0x1AE000001AEC, - 0x1B0000001B04, - 0x1B3400001B35, - 0x1B3600001B3B, - 0x1B3C00001B3D, - 0x1B4200001B43, - 0x1B6B00001B74, - 0x1B8000001B82, - 0x1BA200001BA6, - 0x1BA800001BAA, - 0x1BAB00001BAE, - 0x1BE600001BE7, - 0x1BE800001BEA, - 0x1BED00001BEE, - 0x1BEF00001BF2, - 0x1C2C00001C34, - 0x1C3600001C38, - 0x1CD000001CD3, - 0x1CD400001CE1, - 0x1CE200001CE9, - 0x1CED00001CEE, - 0x1CF400001CF5, - 0x1CF800001CFA, - 0x1DC000001E00, - 0x200B0000200C, - 0x200E00002010, - 0x202A0000202F, - 0x206000002065, - 0x206A00002070, - 0x20D0000020F1, - 0x2CEF00002CF2, - 0x2D7F00002D80, - 0x2DE000002E00, - 0x302A0000302E, - 0x30990000309B, - 0xA66F0000A673, - 0xA6740000A67E, - 0xA69E0000A6A0, - 0xA6F00000A6F2, - 0xA8020000A803, - 0xA8060000A807, - 0xA80B0000A80C, - 0xA8250000A827, - 0xA82C0000A82D, - 0xA8C40000A8C6, - 0xA8E00000A8F2, - 0xA8FF0000A900, - 0xA9260000A92E, - 0xA9470000A952, - 0xA9800000A983, - 0xA9B30000A9B4, - 0xA9B60000A9BA, - 0xA9BC0000A9BE, - 0xA9E50000A9E6, - 0xAA290000AA2F, - 0xAA310000AA33, - 0xAA350000AA37, - 0xAA430000AA44, - 0xAA4C0000AA4D, - 0xAA7C0000AA7D, - 0xAAB00000AAB1, - 0xAAB20000AAB5, - 0xAAB70000AAB9, - 0xAABE0000AAC0, - 0xAAC10000AAC2, - 0xAAEC0000AAEE, - 0xAAF60000AAF7, - 0xABE50000ABE6, - 0xABE80000ABE9, - 0xABED0000ABEE, - 0xFB1E0000FB1F, - 0xFE000000FE10, - 0xFE200000FE30, - 0xFEFF0000FF00, - 0xFFF90000FFFC, - 0x101FD000101FE, - 0x102E0000102E1, - 0x103760001037B, - 0x10A0100010A04, - 0x10A0500010A07, - 0x10A0C00010A10, - 0x10A3800010A3B, - 0x10A3F00010A40, - 0x10AE500010AE7, - 0x10D2400010D28, - 0x10D6900010D6E, - 0x10EAB00010EAD, - 0x10EFA00010F00, - 0x10F4600010F51, - 0x10F8200010F86, - 0x1100100011002, - 0x1103800011047, - 0x1107000011071, - 0x1107300011075, - 0x1107F00011082, - 0x110B3000110B7, - 0x110B9000110BB, - 0x110C2000110C3, - 0x1110000011103, - 0x111270001112C, - 0x1112D00011135, - 0x1117300011174, - 0x1118000011182, - 0x111B6000111BF, - 0x111C9000111CD, - 0x111CF000111D0, - 0x1122F00011232, - 0x1123400011235, - 0x1123600011238, - 0x1123E0001123F, - 0x1124100011242, - 0x112DF000112E0, - 0x112E3000112EB, - 0x1130000011302, - 0x1133B0001133D, - 0x1134000011341, - 0x113660001136D, - 0x1137000011375, - 0x113BB000113C1, - 0x113CE000113CF, - 0x113D0000113D1, - 0x113D2000113D3, - 0x113E1000113E3, - 0x1143800011440, - 0x1144200011445, - 0x1144600011447, - 0x1145E0001145F, - 0x114B3000114B9, - 0x114BA000114BB, - 0x114BF000114C1, - 0x114C2000114C4, - 0x115B2000115B6, - 0x115BC000115BE, - 0x115BF000115C1, - 0x115DC000115DE, - 0x116330001163B, - 0x1163D0001163E, - 0x1163F00011641, - 0x116AB000116AC, - 0x116AD000116AE, - 0x116B0000116B6, - 0x116B7000116B8, - 0x1171D0001171E, - 0x1171F00011720, - 0x1172200011726, - 0x117270001172C, - 0x1182F00011838, - 0x118390001183B, - 0x1193B0001193D, - 0x1193E0001193F, - 0x1194300011944, - 0x119D4000119D8, - 0x119DA000119DC, - 0x119E0000119E1, - 0x11A0100011A0B, - 0x11A3300011A39, - 0x11A3B00011A3F, - 0x11A4700011A48, - 0x11A5100011A57, - 0x11A5900011A5C, - 0x11A8A00011A97, - 0x11A9800011A9A, - 0x11B6000011B61, - 0x11B6200011B65, - 0x11B6600011B67, - 0x11C3000011C37, - 0x11C3800011C3E, - 0x11C3F00011C40, - 0x11C9200011CA8, - 0x11CAA00011CB1, - 0x11CB200011CB4, - 0x11CB500011CB7, - 0x11D3100011D37, - 0x11D3A00011D3B, - 0x11D3C00011D3E, - 0x11D3F00011D46, - 0x11D4700011D48, - 0x11D9000011D92, - 0x11D9500011D96, - 0x11D9700011D98, - 0x11EF300011EF5, - 0x11F0000011F02, - 0x11F3600011F3B, - 0x11F4000011F41, - 0x11F4200011F43, - 0x11F5A00011F5B, - 0x1343000013441, - 0x1344700013456, - 0x1611E0001612A, - 0x1612D00016130, - 0x16AF000016AF5, - 0x16B3000016B37, - 0x16F4F00016F50, - 0x16F8F00016F93, - 0x16FE400016FE5, - 0x1BC9D0001BC9F, - 0x1BCA00001BCA4, - 0x1CF000001CF2E, - 0x1CF300001CF47, - 0x1D1670001D16A, - 0x1D1730001D183, - 0x1D1850001D18C, - 0x1D1AA0001D1AE, - 0x1D2420001D245, - 0x1DA000001DA37, - 0x1DA3B0001DA6D, - 0x1DA750001DA76, - 0x1DA840001DA85, - 0x1DA9B0001DAA0, - 0x1DAA10001DAB0, - 0x1E0000001E007, - 0x1E0080001E019, - 0x1E01B0001E022, - 0x1E0230001E025, - 0x1E0260001E02B, - 0x1E08F0001E090, - 0x1E1300001E137, - 0x1E2AE0001E2AF, - 0x1E2EC0001E2F0, - 0x1E4EC0001E4F0, - 0x1E5EE0001E5F0, - 0x1E6E30001E6E4, - 0x1E6E60001E6E7, - 0x1E6EE0001E6F0, - 0x1E6F50001E6F6, - 0x1E8D00001E8D7, - 0x1E9440001E94C, - 0xE0001000E0002, - 0xE0020000E0080, - 0xE0100000E01F0, - ), -} - - -codepoint_classes = { - "PVALID": ( - 0x2D0000002E, - 0x300000003A, - 0x610000007B, - 0xDF000000F7, - 0xF800000100, - 0x10100000102, - 0x10300000104, - 0x10500000106, - 0x10700000108, - 0x1090000010A, - 0x10B0000010C, - 0x10D0000010E, - 0x10F00000110, - 0x11100000112, - 0x11300000114, - 0x11500000116, - 0x11700000118, - 0x1190000011A, - 0x11B0000011C, - 0x11D0000011E, - 0x11F00000120, - 0x12100000122, - 0x12300000124, - 0x12500000126, - 0x12700000128, - 0x1290000012A, - 0x12B0000012C, - 0x12D0000012E, - 0x12F00000130, - 0x13100000132, - 0x13500000136, - 0x13700000139, - 0x13A0000013B, - 0x13C0000013D, - 0x13E0000013F, - 0x14200000143, - 0x14400000145, - 0x14600000147, - 0x14800000149, - 0x14B0000014C, - 0x14D0000014E, - 0x14F00000150, - 0x15100000152, - 0x15300000154, - 0x15500000156, - 0x15700000158, - 0x1590000015A, - 0x15B0000015C, - 0x15D0000015E, - 0x15F00000160, - 0x16100000162, - 0x16300000164, - 0x16500000166, - 0x16700000168, - 0x1690000016A, - 0x16B0000016C, - 0x16D0000016E, - 0x16F00000170, - 0x17100000172, - 0x17300000174, - 0x17500000176, - 0x17700000178, - 0x17A0000017B, - 0x17C0000017D, - 0x17E0000017F, - 0x18000000181, - 0x18300000184, - 0x18500000186, - 0x18800000189, - 0x18C0000018E, - 0x19200000193, - 0x19500000196, - 0x1990000019C, - 0x19E0000019F, - 0x1A1000001A2, - 0x1A3000001A4, - 0x1A5000001A6, - 0x1A8000001A9, - 0x1AA000001AC, - 0x1AD000001AE, - 0x1B0000001B1, - 0x1B4000001B5, - 0x1B6000001B7, - 0x1B9000001BC, - 0x1BD000001C4, - 0x1CE000001CF, - 0x1D0000001D1, - 0x1D2000001D3, - 0x1D4000001D5, - 0x1D6000001D7, - 0x1D8000001D9, - 0x1DA000001DB, - 0x1DC000001DE, - 0x1DF000001E0, - 0x1E1000001E2, - 0x1E3000001E4, - 0x1E5000001E6, - 0x1E7000001E8, - 0x1E9000001EA, - 0x1EB000001EC, - 0x1ED000001EE, - 0x1EF000001F1, - 0x1F5000001F6, - 0x1F9000001FA, - 0x1FB000001FC, - 0x1FD000001FE, - 0x1FF00000200, - 0x20100000202, - 0x20300000204, - 0x20500000206, - 0x20700000208, - 0x2090000020A, - 0x20B0000020C, - 0x20D0000020E, - 0x20F00000210, - 0x21100000212, - 0x21300000214, - 0x21500000216, - 0x21700000218, - 0x2190000021A, - 0x21B0000021C, - 0x21D0000021E, - 0x21F00000220, - 0x22100000222, - 0x22300000224, - 0x22500000226, - 0x22700000228, - 0x2290000022A, - 0x22B0000022C, - 0x22D0000022E, - 0x22F00000230, - 0x23100000232, - 0x2330000023A, - 0x23C0000023D, - 0x23F00000241, - 0x24200000243, - 0x24700000248, - 0x2490000024A, - 0x24B0000024C, - 0x24D0000024E, - 0x24F000002B0, - 0x2B9000002C2, - 0x2C6000002D2, - 0x2EC000002ED, - 0x2EE000002EF, - 0x30000000340, - 0x34200000343, - 0x3460000034F, - 0x35000000370, - 0x37100000372, - 0x37300000374, - 0x37700000378, - 0x37B0000037E, - 0x39000000391, - 0x3AC000003CF, - 0x3D7000003D8, - 0x3D9000003DA, - 0x3DB000003DC, - 0x3DD000003DE, - 0x3DF000003E0, - 0x3E1000003E2, - 0x3E3000003E4, - 0x3E5000003E6, - 0x3E7000003E8, - 0x3E9000003EA, - 0x3EB000003EC, - 0x3ED000003EE, - 0x3EF000003F0, - 0x3F3000003F4, - 0x3F8000003F9, - 0x3FB000003FD, - 0x43000000460, - 0x46100000462, - 0x46300000464, - 0x46500000466, - 0x46700000468, - 0x4690000046A, - 0x46B0000046C, - 0x46D0000046E, - 0x46F00000470, - 0x47100000472, - 0x47300000474, - 0x47500000476, - 0x47700000478, - 0x4790000047A, - 0x47B0000047C, - 0x47D0000047E, - 0x47F00000480, - 0x48100000482, - 0x48300000488, - 0x48B0000048C, - 0x48D0000048E, - 0x48F00000490, - 0x49100000492, - 0x49300000494, - 0x49500000496, - 0x49700000498, - 0x4990000049A, - 0x49B0000049C, - 0x49D0000049E, - 0x49F000004A0, - 0x4A1000004A2, - 0x4A3000004A4, - 0x4A5000004A6, - 0x4A7000004A8, - 0x4A9000004AA, - 0x4AB000004AC, - 0x4AD000004AE, - 0x4AF000004B0, - 0x4B1000004B2, - 0x4B3000004B4, - 0x4B5000004B6, - 0x4B7000004B8, - 0x4B9000004BA, - 0x4BB000004BC, - 0x4BD000004BE, - 0x4BF000004C0, - 0x4C2000004C3, - 0x4C4000004C5, - 0x4C6000004C7, - 0x4C8000004C9, - 0x4CA000004CB, - 0x4CC000004CD, - 0x4CE000004D0, - 0x4D1000004D2, - 0x4D3000004D4, - 0x4D5000004D6, - 0x4D7000004D8, - 0x4D9000004DA, - 0x4DB000004DC, - 0x4DD000004DE, - 0x4DF000004E0, - 0x4E1000004E2, - 0x4E3000004E4, - 0x4E5000004E6, - 0x4E7000004E8, - 0x4E9000004EA, - 0x4EB000004EC, - 0x4ED000004EE, - 0x4EF000004F0, - 0x4F1000004F2, - 0x4F3000004F4, - 0x4F5000004F6, - 0x4F7000004F8, - 0x4F9000004FA, - 0x4FB000004FC, - 0x4FD000004FE, - 0x4FF00000500, - 0x50100000502, - 0x50300000504, - 0x50500000506, - 0x50700000508, - 0x5090000050A, - 0x50B0000050C, - 0x50D0000050E, - 0x50F00000510, - 0x51100000512, - 0x51300000514, - 0x51500000516, - 0x51700000518, - 0x5190000051A, - 0x51B0000051C, - 0x51D0000051E, - 0x51F00000520, - 0x52100000522, - 0x52300000524, - 0x52500000526, - 0x52700000528, - 0x5290000052A, - 0x52B0000052C, - 0x52D0000052E, - 0x52F00000530, - 0x5590000055A, - 0x56000000587, - 0x58800000589, - 0x591000005BE, - 0x5BF000005C0, - 0x5C1000005C3, - 0x5C4000005C6, - 0x5C7000005C8, - 0x5D0000005EB, - 0x5EF000005F3, - 0x6100000061B, - 0x62000000640, - 0x64100000660, - 0x66E00000675, - 0x679000006D4, - 0x6D5000006DD, - 0x6DF000006E9, - 0x6EA000006F0, - 0x6FA00000700, - 0x7100000074B, - 0x74D000007B2, - 0x7C0000007F6, - 0x7FD000007FE, - 0x8000000082E, - 0x8400000085C, - 0x8600000086B, - 0x87000000888, - 0x88900000890, - 0x897000008E2, - 0x8E300000958, - 0x96000000964, - 0x96600000970, - 0x97100000984, - 0x9850000098D, - 0x98F00000991, - 0x993000009A9, - 0x9AA000009B1, - 0x9B2000009B3, - 0x9B6000009BA, - 0x9BC000009C5, - 0x9C7000009C9, - 0x9CB000009CF, - 0x9D7000009D8, - 0x9E0000009E4, - 0x9E6000009F2, - 0x9FC000009FD, - 0x9FE000009FF, - 0xA0100000A04, - 0xA0500000A0B, - 0xA0F00000A11, - 0xA1300000A29, - 0xA2A00000A31, - 0xA3200000A33, - 0xA3500000A36, - 0xA3800000A3A, - 0xA3C00000A3D, - 0xA3E00000A43, - 0xA4700000A49, - 0xA4B00000A4E, - 0xA5100000A52, - 0xA5C00000A5D, - 0xA6600000A76, - 0xA8100000A84, - 0xA8500000A8E, - 0xA8F00000A92, - 0xA9300000AA9, - 0xAAA00000AB1, - 0xAB200000AB4, - 0xAB500000ABA, - 0xABC00000AC6, - 0xAC700000ACA, - 0xACB00000ACE, - 0xAD000000AD1, - 0xAE000000AE4, - 0xAE600000AF0, - 0xAF900000B00, - 0xB0100000B04, - 0xB0500000B0D, - 0xB0F00000B11, - 0xB1300000B29, - 0xB2A00000B31, - 0xB3200000B34, - 0xB3500000B3A, - 0xB3C00000B45, - 0xB4700000B49, - 0xB4B00000B4E, - 0xB5500000B58, - 0xB5F00000B64, - 0xB6600000B70, - 0xB7100000B72, - 0xB8200000B84, - 0xB8500000B8B, - 0xB8E00000B91, - 0xB9200000B96, - 0xB9900000B9B, - 0xB9C00000B9D, - 0xB9E00000BA0, - 0xBA300000BA5, - 0xBA800000BAB, - 0xBAE00000BBA, - 0xBBE00000BC3, - 0xBC600000BC9, - 0xBCA00000BCE, - 0xBD000000BD1, - 0xBD700000BD8, - 0xBE600000BF0, - 0xC0000000C0D, - 0xC0E00000C11, - 0xC1200000C29, - 0xC2A00000C3A, - 0xC3C00000C45, - 0xC4600000C49, - 0xC4A00000C4E, - 0xC5500000C57, - 0xC5800000C5B, - 0xC5C00000C5E, - 0xC6000000C64, - 0xC6600000C70, - 0xC8000000C84, - 0xC8500000C8D, - 0xC8E00000C91, - 0xC9200000CA9, - 0xCAA00000CB4, - 0xCB500000CBA, - 0xCBC00000CC5, - 0xCC600000CC9, - 0xCCA00000CCE, - 0xCD500000CD7, - 0xCDC00000CDF, - 0xCE000000CE4, - 0xCE600000CF0, - 0xCF100000CF4, - 0xD0000000D0D, - 0xD0E00000D11, - 0xD1200000D45, - 0xD4600000D49, - 0xD4A00000D4F, - 0xD5400000D58, - 0xD5F00000D64, - 0xD6600000D70, - 0xD7A00000D80, - 0xD8100000D84, - 0xD8500000D97, - 0xD9A00000DB2, - 0xDB300000DBC, - 0xDBD00000DBE, - 0xDC000000DC7, - 0xDCA00000DCB, - 0xDCF00000DD5, - 0xDD600000DD7, - 0xDD800000DE0, - 0xDE600000DF0, - 0xDF200000DF4, - 0xE0100000E33, - 0xE3400000E3B, - 0xE4000000E4F, - 0xE5000000E5A, - 0xE8100000E83, - 0xE8400000E85, - 0xE8600000E8B, - 0xE8C00000EA4, - 0xEA500000EA6, - 0xEA700000EB3, - 0xEB400000EBE, - 0xEC000000EC5, - 0xEC600000EC7, - 0xEC800000ECF, - 0xED000000EDA, - 0xEDE00000EE0, - 0xF0000000F01, - 0xF0B00000F0C, - 0xF1800000F1A, - 0xF2000000F2A, - 0xF3500000F36, - 0xF3700000F38, - 0xF3900000F3A, - 0xF3E00000F43, - 0xF4400000F48, - 0xF4900000F4D, - 0xF4E00000F52, - 0xF5300000F57, - 0xF5800000F5C, - 0xF5D00000F69, - 0xF6A00000F6D, - 0xF7100000F73, - 0xF7400000F75, - 0xF7A00000F81, - 0xF8200000F85, - 0xF8600000F93, - 0xF9400000F98, - 0xF9900000F9D, - 0xF9E00000FA2, - 0xFA300000FA7, - 0xFA800000FAC, - 0xFAD00000FB9, - 0xFBA00000FBD, - 0xFC600000FC7, - 0x10000000104A, - 0x10500000109E, - 0x10D0000010FB, - 0x10FD00001100, - 0x120000001249, - 0x124A0000124E, - 0x125000001257, - 0x125800001259, - 0x125A0000125E, - 0x126000001289, - 0x128A0000128E, - 0x1290000012B1, - 0x12B2000012B6, - 0x12B8000012BF, - 0x12C0000012C1, - 0x12C2000012C6, - 0x12C8000012D7, - 0x12D800001311, - 0x131200001316, - 0x13180000135B, - 0x135D00001360, - 0x138000001390, - 0x13A0000013F6, - 0x14010000166D, - 0x166F00001680, - 0x16810000169B, - 0x16A0000016EB, - 0x16F1000016F9, - 0x170000001716, - 0x171F00001735, - 0x174000001754, - 0x17600000176D, - 0x176E00001771, - 0x177200001774, - 0x1780000017B4, - 0x17B6000017D4, - 0x17D7000017D8, - 0x17DC000017DE, - 0x17E0000017EA, - 0x18100000181A, - 0x182000001879, - 0x1880000018AB, - 0x18B0000018F6, - 0x19000000191F, - 0x19200000192C, - 0x19300000193C, - 0x19460000196E, - 0x197000001975, - 0x1980000019AC, - 0x19B0000019CA, - 0x19D0000019DA, - 0x1A0000001A1C, - 0x1A2000001A5F, - 0x1A6000001A7D, - 0x1A7F00001A8A, - 0x1A9000001A9A, - 0x1AA700001AA8, - 0x1AB000001ABE, - 0x1ABF00001ADE, - 0x1AE000001AEC, - 0x1B0000001B4D, - 0x1B5000001B5A, - 0x1B6B00001B74, - 0x1B8000001BF4, - 0x1C0000001C38, - 0x1C4000001C4A, - 0x1C4D00001C7E, - 0x1C8A00001C8B, - 0x1CD000001CD3, - 0x1CD400001CFB, - 0x1D0000001D2C, - 0x1D2F00001D30, - 0x1D3B00001D3C, - 0x1D4E00001D4F, - 0x1D6B00001D78, - 0x1D7900001D9B, - 0x1DC000001E00, - 0x1E0100001E02, - 0x1E0300001E04, - 0x1E0500001E06, - 0x1E0700001E08, - 0x1E0900001E0A, - 0x1E0B00001E0C, - 0x1E0D00001E0E, - 0x1E0F00001E10, - 0x1E1100001E12, - 0x1E1300001E14, - 0x1E1500001E16, - 0x1E1700001E18, - 0x1E1900001E1A, - 0x1E1B00001E1C, - 0x1E1D00001E1E, - 0x1E1F00001E20, - 0x1E2100001E22, - 0x1E2300001E24, - 0x1E2500001E26, - 0x1E2700001E28, - 0x1E2900001E2A, - 0x1E2B00001E2C, - 0x1E2D00001E2E, - 0x1E2F00001E30, - 0x1E3100001E32, - 0x1E3300001E34, - 0x1E3500001E36, - 0x1E3700001E38, - 0x1E3900001E3A, - 0x1E3B00001E3C, - 0x1E3D00001E3E, - 0x1E3F00001E40, - 0x1E4100001E42, - 0x1E4300001E44, - 0x1E4500001E46, - 0x1E4700001E48, - 0x1E4900001E4A, - 0x1E4B00001E4C, - 0x1E4D00001E4E, - 0x1E4F00001E50, - 0x1E5100001E52, - 0x1E5300001E54, - 0x1E5500001E56, - 0x1E5700001E58, - 0x1E5900001E5A, - 0x1E5B00001E5C, - 0x1E5D00001E5E, - 0x1E5F00001E60, - 0x1E6100001E62, - 0x1E6300001E64, - 0x1E6500001E66, - 0x1E6700001E68, - 0x1E6900001E6A, - 0x1E6B00001E6C, - 0x1E6D00001E6E, - 0x1E6F00001E70, - 0x1E7100001E72, - 0x1E7300001E74, - 0x1E7500001E76, - 0x1E7700001E78, - 0x1E7900001E7A, - 0x1E7B00001E7C, - 0x1E7D00001E7E, - 0x1E7F00001E80, - 0x1E8100001E82, - 0x1E8300001E84, - 0x1E8500001E86, - 0x1E8700001E88, - 0x1E8900001E8A, - 0x1E8B00001E8C, - 0x1E8D00001E8E, - 0x1E8F00001E90, - 0x1E9100001E92, - 0x1E9300001E94, - 0x1E9500001E9A, - 0x1E9C00001E9E, - 0x1E9F00001EA0, - 0x1EA100001EA2, - 0x1EA300001EA4, - 0x1EA500001EA6, - 0x1EA700001EA8, - 0x1EA900001EAA, - 0x1EAB00001EAC, - 0x1EAD00001EAE, - 0x1EAF00001EB0, - 0x1EB100001EB2, - 0x1EB300001EB4, - 0x1EB500001EB6, - 0x1EB700001EB8, - 0x1EB900001EBA, - 0x1EBB00001EBC, - 0x1EBD00001EBE, - 0x1EBF00001EC0, - 0x1EC100001EC2, - 0x1EC300001EC4, - 0x1EC500001EC6, - 0x1EC700001EC8, - 0x1EC900001ECA, - 0x1ECB00001ECC, - 0x1ECD00001ECE, - 0x1ECF00001ED0, - 0x1ED100001ED2, - 0x1ED300001ED4, - 0x1ED500001ED6, - 0x1ED700001ED8, - 0x1ED900001EDA, - 0x1EDB00001EDC, - 0x1EDD00001EDE, - 0x1EDF00001EE0, - 0x1EE100001EE2, - 0x1EE300001EE4, - 0x1EE500001EE6, - 0x1EE700001EE8, - 0x1EE900001EEA, - 0x1EEB00001EEC, - 0x1EED00001EEE, - 0x1EEF00001EF0, - 0x1EF100001EF2, - 0x1EF300001EF4, - 0x1EF500001EF6, - 0x1EF700001EF8, - 0x1EF900001EFA, - 0x1EFB00001EFC, - 0x1EFD00001EFE, - 0x1EFF00001F08, - 0x1F1000001F16, - 0x1F2000001F28, - 0x1F3000001F38, - 0x1F4000001F46, - 0x1F5000001F58, - 0x1F6000001F68, - 0x1F7000001F71, - 0x1F7200001F73, - 0x1F7400001F75, - 0x1F7600001F77, - 0x1F7800001F79, - 0x1F7A00001F7B, - 0x1F7C00001F7D, - 0x1FB000001FB2, - 0x1FB600001FB7, - 0x1FC600001FC7, - 0x1FD000001FD3, - 0x1FD600001FD8, - 0x1FE000001FE3, - 0x1FE400001FE8, - 0x1FF600001FF7, - 0x214E0000214F, - 0x218400002185, - 0x2C3000002C60, - 0x2C6100002C62, - 0x2C6500002C67, - 0x2C6800002C69, - 0x2C6A00002C6B, - 0x2C6C00002C6D, - 0x2C7100002C72, - 0x2C7300002C75, - 0x2C7600002C7C, - 0x2C8100002C82, - 0x2C8300002C84, - 0x2C8500002C86, - 0x2C8700002C88, - 0x2C8900002C8A, - 0x2C8B00002C8C, - 0x2C8D00002C8E, - 0x2C8F00002C90, - 0x2C9100002C92, - 0x2C9300002C94, - 0x2C9500002C96, - 0x2C9700002C98, - 0x2C9900002C9A, - 0x2C9B00002C9C, - 0x2C9D00002C9E, - 0x2C9F00002CA0, - 0x2CA100002CA2, - 0x2CA300002CA4, - 0x2CA500002CA6, - 0x2CA700002CA8, - 0x2CA900002CAA, - 0x2CAB00002CAC, - 0x2CAD00002CAE, - 0x2CAF00002CB0, - 0x2CB100002CB2, - 0x2CB300002CB4, - 0x2CB500002CB6, - 0x2CB700002CB8, - 0x2CB900002CBA, - 0x2CBB00002CBC, - 0x2CBD00002CBE, - 0x2CBF00002CC0, - 0x2CC100002CC2, - 0x2CC300002CC4, - 0x2CC500002CC6, - 0x2CC700002CC8, - 0x2CC900002CCA, - 0x2CCB00002CCC, - 0x2CCD00002CCE, - 0x2CCF00002CD0, - 0x2CD100002CD2, - 0x2CD300002CD4, - 0x2CD500002CD6, - 0x2CD700002CD8, - 0x2CD900002CDA, - 0x2CDB00002CDC, - 0x2CDD00002CDE, - 0x2CDF00002CE0, - 0x2CE100002CE2, - 0x2CE300002CE5, - 0x2CEC00002CED, - 0x2CEE00002CF2, - 0x2CF300002CF4, - 0x2D0000002D26, - 0x2D2700002D28, - 0x2D2D00002D2E, - 0x2D3000002D68, - 0x2D7F00002D97, - 0x2DA000002DA7, - 0x2DA800002DAF, - 0x2DB000002DB7, - 0x2DB800002DBF, - 0x2DC000002DC7, - 0x2DC800002DCF, - 0x2DD000002DD7, - 0x2DD800002DDF, - 0x2DE000002E00, - 0x2E2F00002E30, - 0x300500003008, - 0x302A0000302E, - 0x303C0000303D, - 0x304100003097, - 0x30990000309B, - 0x309D0000309F, - 0x30A1000030FB, - 0x30FC000030FF, - 0x310500003130, - 0x31A0000031C0, - 0x31F000003200, - 0x340000004DC0, - 0x4E000000A48D, - 0xA4D00000A4FE, - 0xA5000000A60D, - 0xA6100000A62C, - 0xA6410000A642, - 0xA6430000A644, - 0xA6450000A646, - 0xA6470000A648, - 0xA6490000A64A, - 0xA64B0000A64C, - 0xA64D0000A64E, - 0xA64F0000A650, - 0xA6510000A652, - 0xA6530000A654, - 0xA6550000A656, - 0xA6570000A658, - 0xA6590000A65A, - 0xA65B0000A65C, - 0xA65D0000A65E, - 0xA65F0000A660, - 0xA6610000A662, - 0xA6630000A664, - 0xA6650000A666, - 0xA6670000A668, - 0xA6690000A66A, - 0xA66B0000A66C, - 0xA66D0000A670, - 0xA6740000A67E, - 0xA67F0000A680, - 0xA6810000A682, - 0xA6830000A684, - 0xA6850000A686, - 0xA6870000A688, - 0xA6890000A68A, - 0xA68B0000A68C, - 0xA68D0000A68E, - 0xA68F0000A690, - 0xA6910000A692, - 0xA6930000A694, - 0xA6950000A696, - 0xA6970000A698, - 0xA6990000A69A, - 0xA69B0000A69C, - 0xA69E0000A6E6, - 0xA6F00000A6F2, - 0xA7170000A720, - 0xA7230000A724, - 0xA7250000A726, - 0xA7270000A728, - 0xA7290000A72A, - 0xA72B0000A72C, - 0xA72D0000A72E, - 0xA72F0000A732, - 0xA7330000A734, - 0xA7350000A736, - 0xA7370000A738, - 0xA7390000A73A, - 0xA73B0000A73C, - 0xA73D0000A73E, - 0xA73F0000A740, - 0xA7410000A742, - 0xA7430000A744, - 0xA7450000A746, - 0xA7470000A748, - 0xA7490000A74A, - 0xA74B0000A74C, - 0xA74D0000A74E, - 0xA74F0000A750, - 0xA7510000A752, - 0xA7530000A754, - 0xA7550000A756, - 0xA7570000A758, - 0xA7590000A75A, - 0xA75B0000A75C, - 0xA75D0000A75E, - 0xA75F0000A760, - 0xA7610000A762, - 0xA7630000A764, - 0xA7650000A766, - 0xA7670000A768, - 0xA7690000A76A, - 0xA76B0000A76C, - 0xA76D0000A76E, - 0xA76F0000A770, - 0xA7710000A779, - 0xA77A0000A77B, - 0xA77C0000A77D, - 0xA77F0000A780, - 0xA7810000A782, - 0xA7830000A784, - 0xA7850000A786, - 0xA7870000A789, - 0xA78C0000A78D, - 0xA78E0000A790, - 0xA7910000A792, - 0xA7930000A796, - 0xA7970000A798, - 0xA7990000A79A, - 0xA79B0000A79C, - 0xA79D0000A79E, - 0xA79F0000A7A0, - 0xA7A10000A7A2, - 0xA7A30000A7A4, - 0xA7A50000A7A6, - 0xA7A70000A7A8, - 0xA7A90000A7AA, - 0xA7AF0000A7B0, - 0xA7B50000A7B6, - 0xA7B70000A7B8, - 0xA7B90000A7BA, - 0xA7BB0000A7BC, - 0xA7BD0000A7BE, - 0xA7BF0000A7C0, - 0xA7C10000A7C2, - 0xA7C30000A7C4, - 0xA7C80000A7C9, - 0xA7CA0000A7CB, - 0xA7CD0000A7CE, - 0xA7CF0000A7D0, - 0xA7D10000A7D2, - 0xA7D30000A7D4, - 0xA7D50000A7D6, - 0xA7D70000A7D8, - 0xA7D90000A7DA, - 0xA7DB0000A7DC, - 0xA7F60000A7F8, - 0xA7FA0000A828, - 0xA82C0000A82D, - 0xA8400000A874, - 0xA8800000A8C6, - 0xA8D00000A8DA, - 0xA8E00000A8F8, - 0xA8FB0000A8FC, - 0xA8FD0000A92E, - 0xA9300000A954, - 0xA9800000A9C1, - 0xA9CF0000A9DA, - 0xA9E00000A9FF, - 0xAA000000AA37, - 0xAA400000AA4E, - 0xAA500000AA5A, - 0xAA600000AA77, - 0xAA7A0000AAC3, - 0xAADB0000AADE, - 0xAAE00000AAF0, - 0xAAF20000AAF7, - 0xAB010000AB07, - 0xAB090000AB0F, - 0xAB110000AB17, - 0xAB200000AB27, - 0xAB280000AB2F, - 0xAB300000AB5B, - 0xAB600000AB69, - 0xABC00000ABEB, - 0xABEC0000ABEE, - 0xABF00000ABFA, - 0xAC000000D7A4, - 0xFA0E0000FA10, - 0xFA110000FA12, - 0xFA130000FA15, - 0xFA1F0000FA20, - 0xFA210000FA22, - 0xFA230000FA25, - 0xFA270000FA2A, - 0xFB1E0000FB1F, - 0xFE200000FE30, - 0xFE730000FE74, - 0x100000001000C, - 0x1000D00010027, - 0x100280001003B, - 0x1003C0001003E, - 0x1003F0001004E, - 0x100500001005E, - 0x10080000100FB, - 0x101FD000101FE, - 0x102800001029D, - 0x102A0000102D1, - 0x102E0000102E1, - 0x1030000010320, - 0x1032D00010341, - 0x103420001034A, - 0x103500001037B, - 0x103800001039E, - 0x103A0000103C4, - 0x103C8000103D0, - 0x104280001049E, - 0x104A0000104AA, - 0x104D8000104FC, - 0x1050000010528, - 0x1053000010564, - 0x10597000105A2, - 0x105A3000105B2, - 0x105B3000105BA, - 0x105BB000105BD, - 0x105C0000105F4, - 0x1060000010737, - 0x1074000010756, - 0x1076000010768, - 0x1078000010781, - 0x1080000010806, - 0x1080800010809, - 0x1080A00010836, - 0x1083700010839, - 0x1083C0001083D, - 0x1083F00010856, - 0x1086000010877, - 0x108800001089F, - 0x108E0000108F3, - 0x108F4000108F6, - 0x1090000010916, - 0x109200001093A, - 0x109400001095A, - 0x10980000109B8, - 0x109BE000109C0, - 0x10A0000010A04, - 0x10A0500010A07, - 0x10A0C00010A14, - 0x10A1500010A18, - 0x10A1900010A36, - 0x10A3800010A3B, - 0x10A3F00010A40, - 0x10A6000010A7D, - 0x10A8000010A9D, - 0x10AC000010AC8, - 0x10AC900010AE7, - 0x10B0000010B36, - 0x10B4000010B56, - 0x10B6000010B73, - 0x10B8000010B92, - 0x10C0000010C49, - 0x10CC000010CF3, - 0x10D0000010D28, - 0x10D3000010D3A, - 0x10D4000010D50, - 0x10D6900010D6E, - 0x10D6F00010D86, - 0x10E8000010EAA, - 0x10EAB00010EAD, - 0x10EB000010EB2, - 0x10EC200010EC8, - 0x10EFA00010F1D, - 0x10F2700010F28, - 0x10F3000010F51, - 0x10F7000010F86, - 0x10FB000010FC5, - 0x10FE000010FF7, - 0x1100000011047, - 0x1106600011076, - 0x1107F000110BB, - 0x110C2000110C3, - 0x110D0000110E9, - 0x110F0000110FA, - 0x1110000011135, - 0x1113600011140, - 0x1114400011148, - 0x1115000011174, - 0x1117600011177, - 0x11180000111C5, - 0x111C9000111CD, - 0x111CE000111DB, - 0x111DC000111DD, - 0x1120000011212, - 0x1121300011238, - 0x1123E00011242, - 0x1128000011287, - 0x1128800011289, - 0x1128A0001128E, - 0x1128F0001129E, - 0x1129F000112A9, - 0x112B0000112EB, - 0x112F0000112FA, - 0x1130000011304, - 0x113050001130D, - 0x1130F00011311, - 0x1131300011329, - 0x1132A00011331, - 0x1133200011334, - 0x113350001133A, - 0x1133B00011345, - 0x1134700011349, - 0x1134B0001134E, - 0x1135000011351, - 0x1135700011358, - 0x1135D00011364, - 0x113660001136D, - 0x1137000011375, - 0x113800001138A, - 0x1138B0001138C, - 0x1138E0001138F, - 0x11390000113B6, - 0x113B7000113C1, - 0x113C2000113C3, - 0x113C5000113C6, - 0x113C7000113CB, - 0x113CC000113D4, - 0x113E1000113E3, - 0x114000001144B, - 0x114500001145A, - 0x1145E00011462, - 0x11480000114C6, - 0x114C7000114C8, - 0x114D0000114DA, - 0x11580000115B6, - 0x115B8000115C1, - 0x115D8000115DE, - 0x1160000011641, - 0x1164400011645, - 0x116500001165A, - 0x11680000116B9, - 0x116C0000116CA, - 0x116D0000116E4, - 0x117000001171B, - 0x1171D0001172C, - 0x117300001173A, - 0x1174000011747, - 0x118000001183B, - 0x118C0000118EA, - 0x118FF00011907, - 0x119090001190A, - 0x1190C00011914, - 0x1191500011917, - 0x1191800011936, - 0x1193700011939, - 0x1193B00011944, - 0x119500001195A, - 0x119A0000119A8, - 0x119AA000119D8, - 0x119DA000119E2, - 0x119E3000119E5, - 0x11A0000011A3F, - 0x11A4700011A48, - 0x11A5000011A9A, - 0x11A9D00011A9E, - 0x11AB000011AF9, - 0x11B6000011B68, - 0x11BC000011BE1, - 0x11BF000011BFA, - 0x11C0000011C09, - 0x11C0A00011C37, - 0x11C3800011C41, - 0x11C5000011C5A, - 0x11C7200011C90, - 0x11C9200011CA8, - 0x11CA900011CB7, - 0x11D0000011D07, - 0x11D0800011D0A, - 0x11D0B00011D37, - 0x11D3A00011D3B, - 0x11D3C00011D3E, - 0x11D3F00011D48, - 0x11D5000011D5A, - 0x11D6000011D66, - 0x11D6700011D69, - 0x11D6A00011D8F, - 0x11D9000011D92, - 0x11D9300011D99, - 0x11DA000011DAA, - 0x11DB000011DDC, - 0x11DE000011DEA, - 0x11EE000011EF7, - 0x11F0000011F11, - 0x11F1200011F3B, - 0x11F3E00011F43, - 0x11F5000011F5B, - 0x11FB000011FB1, - 0x120000001239A, - 0x1248000012544, - 0x12F9000012FF1, - 0x1300000013430, - 0x1344000013456, - 0x13460000143FB, - 0x1440000014647, - 0x161000001613A, - 0x1680000016A39, - 0x16A4000016A5F, - 0x16A6000016A6A, - 0x16A7000016ABF, - 0x16AC000016ACA, - 0x16AD000016AEE, - 0x16AF000016AF5, - 0x16B0000016B37, - 0x16B4000016B44, - 0x16B5000016B5A, - 0x16B6300016B78, - 0x16B7D00016B90, - 0x16D4000016D6D, - 0x16D7000016D7A, - 0x16E6000016E80, - 0x16EBB00016ED4, - 0x16F0000016F4B, - 0x16F4F00016F88, - 0x16F8F00016FA0, - 0x16FE000016FE2, - 0x16FE300016FE5, - 0x16FF000016FF4, - 0x1700000018CD6, - 0x18CFF00018D1F, - 0x18D8000018DF3, - 0x1AFF00001AFF4, - 0x1AFF50001AFFC, - 0x1AFFD0001AFFF, - 0x1B0000001B123, - 0x1B1320001B133, - 0x1B1500001B153, - 0x1B1550001B156, - 0x1B1640001B168, - 0x1B1700001B2FC, - 0x1BC000001BC6B, - 0x1BC700001BC7D, - 0x1BC800001BC89, - 0x1BC900001BC9A, - 0x1BC9D0001BC9F, - 0x1CF000001CF2E, - 0x1CF300001CF47, - 0x1DA000001DA37, - 0x1DA3B0001DA6D, - 0x1DA750001DA76, - 0x1DA840001DA85, - 0x1DA9B0001DAA0, - 0x1DAA10001DAB0, - 0x1DF000001DF1F, - 0x1DF250001DF2B, - 0x1E0000001E007, - 0x1E0080001E019, - 0x1E01B0001E022, - 0x1E0230001E025, - 0x1E0260001E02B, - 0x1E08F0001E090, - 0x1E1000001E12D, - 0x1E1300001E13E, - 0x1E1400001E14A, - 0x1E14E0001E14F, - 0x1E2900001E2AF, - 0x1E2C00001E2FA, - 0x1E4D00001E4FA, - 0x1E5D00001E5FB, - 0x1E6C00001E6DF, - 0x1E6E00001E6F6, - 0x1E6FE0001E700, - 0x1E7E00001E7E7, - 0x1E7E80001E7EC, - 0x1E7ED0001E7EF, - 0x1E7F00001E7FF, - 0x1E8000001E8C5, - 0x1E8D00001E8D7, - 0x1E9220001E94C, - 0x1E9500001E95A, - 0x200000002A6E0, - 0x2A7000002B81E, - 0x2B8200002CEAE, - 0x2CEB00002EBE1, - 0x2EBF00002EE5E, - 0x300000003134B, - 0x313500003347A, - ), - "CONTEXTJ": (0x200C0000200E,), - "CONTEXTO": ( - 0xB7000000B8, - 0x37500000376, - 0x5F3000005F5, - 0x6600000066A, - 0x6F0000006FA, - 0x30FB000030FC, - ), -} diff --git a/functions/python/data-processor/idna/intranges.py b/functions/python/data-processor/idna/intranges.py deleted file mode 100644 index 19d77810c..000000000 --- a/functions/python/data-processor/idna/intranges.py +++ /dev/null @@ -1,55 +0,0 @@ -""" -Given a list of integers, made up of (hopefully) a small number of long runs -of consecutive integers, compute a representation of the form -((start1, end1), (start2, end2) ...). Then answer the question "was x present -in the original list?" in time O(log(# runs)). -""" - -import bisect - - -def intranges_from_list(list_: list[int]) -> tuple[int, ...]: - """Represent a list of integers as a sequence of ranges: - ((start_0, end_0), (start_1, end_1), ...), such that the original - integers are exactly those x such that start_i <= x < end_i for some i. - - Ranges are encoded as single integers (start << 32 | end), not as tuples. - """ - - sorted_list = sorted(list_) - ranges = [] - last_write = -1 - for i in range(len(sorted_list)): - if i + 1 < len(sorted_list) and sorted_list[i] == sorted_list[i + 1] - 1: - continue - current_range = sorted_list[last_write + 1 : i + 1] - ranges.append(_encode_range(current_range[0], current_range[-1] + 1)) - last_write = i - - return tuple(ranges) - - -def _encode_range(start: int, end: int) -> int: - return (start << 32) | end - - -def _decode_range(r: int) -> tuple[int, int]: - return (r >> 32), (r & ((1 << 32) - 1)) - - -def intranges_contain(int_: int, ranges: tuple[int, ...]) -> bool: - """Determine if `int_` falls into one of the ranges in `ranges`.""" - tuple_ = _encode_range(int_, 0) - pos = bisect.bisect_left(ranges, tuple_) - # we could be immediately ahead of a tuple (start, end) - # with start < int_ <= end - if pos > 0: - left, right = _decode_range(ranges[pos - 1]) - if left <= int_ < right: - return True - # or we could be immediately behind a tuple (int_, end) - if pos < len(ranges): - left, _ = _decode_range(ranges[pos]) - if left == int_: - return True - return False diff --git a/functions/python/data-processor/idna/package_data.py b/functions/python/data-processor/idna/package_data.py deleted file mode 100644 index 94e40398c..000000000 --- a/functions/python/data-processor/idna/package_data.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "3.18" diff --git a/functions/python/data-processor/idna/py.typed b/functions/python/data-processor/idna/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/idna/uts46data.py b/functions/python/data-processor/idna/uts46data.py deleted file mode 100644 index f2d931fe5..000000000 --- a/functions/python/data-processor/idna/uts46data.py +++ /dev/null @@ -1,16896 +0,0 @@ -# This file is automatically generated by tools/idna-data - -from array import array -from typing import Optional - -"""IDNA Mapping Table from UTS46.""" - - -__version__ = "17.0.0" - -uts46_starts: "array[int]" = array( - "I", - ( - 0x0, - 0x1, - 0x2, - 0x3, - 0x4, - 0x5, - 0x6, - 0x7, - 0x8, - 0x9, - 0xA, - 0xB, - 0xC, - 0xD, - 0xE, - 0xF, - 0x10, - 0x11, - 0x12, - 0x13, - 0x14, - 0x15, - 0x16, - 0x17, - 0x18, - 0x19, - 0x1A, - 0x1B, - 0x1C, - 0x1D, - 0x1E, - 0x1F, - 0x20, - 0x21, - 0x22, - 0x23, - 0x24, - 0x25, - 0x26, - 0x27, - 0x28, - 0x29, - 0x2A, - 0x2B, - 0x2C, - 0x2D, - 0x2E, - 0x2F, - 0x30, - 0x31, - 0x32, - 0x33, - 0x34, - 0x35, - 0x36, - 0x37, - 0x38, - 0x39, - 0x3A, - 0x3B, - 0x3C, - 0x3D, - 0x3E, - 0x3F, - 0x40, - 0x41, - 0x42, - 0x43, - 0x44, - 0x45, - 0x46, - 0x47, - 0x48, - 0x49, - 0x4A, - 0x4B, - 0x4C, - 0x4D, - 0x4E, - 0x4F, - 0x50, - 0x51, - 0x52, - 0x53, - 0x54, - 0x55, - 0x56, - 0x57, - 0x58, - 0x59, - 0x5A, - 0x5B, - 0x5C, - 0x5D, - 0x5E, - 0x5F, - 0x60, - 0x61, - 0x62, - 0x63, - 0x64, - 0x65, - 0x66, - 0x67, - 0x68, - 0x69, - 0x6A, - 0x6B, - 0x6C, - 0x6D, - 0x6E, - 0x6F, - 0x70, - 0x71, - 0x72, - 0x73, - 0x74, - 0x75, - 0x76, - 0x77, - 0x78, - 0x79, - 0x7A, - 0x7B, - 0x7C, - 0x7D, - 0x7E, - 0x7F, - 0x80, - 0x81, - 0x82, - 0x83, - 0x84, - 0x85, - 0x86, - 0x87, - 0x88, - 0x89, - 0x8A, - 0x8B, - 0x8C, - 0x8D, - 0x8E, - 0x8F, - 0x90, - 0x91, - 0x92, - 0x93, - 0x94, - 0x95, - 0x96, - 0x97, - 0x98, - 0x99, - 0x9A, - 0x9B, - 0x9C, - 0x9D, - 0x9E, - 0x9F, - 0xA0, - 0xA1, - 0xA2, - 0xA3, - 0xA4, - 0xA5, - 0xA6, - 0xA7, - 0xA8, - 0xA9, - 0xAA, - 0xAB, - 0xAC, - 0xAD, - 0xAE, - 0xAF, - 0xB0, - 0xB1, - 0xB2, - 0xB3, - 0xB4, - 0xB5, - 0xB6, - 0xB7, - 0xB8, - 0xB9, - 0xBA, - 0xBB, - 0xBC, - 0xBD, - 0xBE, - 0xBF, - 0xC0, - 0xC1, - 0xC2, - 0xC3, - 0xC4, - 0xC5, - 0xC6, - 0xC7, - 0xC8, - 0xC9, - 0xCA, - 0xCB, - 0xCC, - 0xCD, - 0xCE, - 0xCF, - 0xD0, - 0xD1, - 0xD2, - 0xD3, - 0xD4, - 0xD5, - 0xD6, - 0xD7, - 0xD8, - 0xD9, - 0xDA, - 0xDB, - 0xDC, - 0xDD, - 0xDE, - 0xDF, - 0xE0, - 0xE1, - 0xE2, - 0xE3, - 0xE4, - 0xE5, - 0xE6, - 0xE7, - 0xE8, - 0xE9, - 0xEA, - 0xEB, - 0xEC, - 0xED, - 0xEE, - 0xEF, - 0xF0, - 0xF1, - 0xF2, - 0xF3, - 0xF4, - 0xF5, - 0xF6, - 0xF7, - 0xF8, - 0xF9, - 0xFA, - 0xFB, - 0xFC, - 0xFD, - 0xFE, - 0xFF, - 0x100, - 0x101, - 0x102, - 0x103, - 0x104, - 0x105, - 0x106, - 0x107, - 0x108, - 0x109, - 0x10A, - 0x10B, - 0x10C, - 0x10D, - 0x10E, - 0x10F, - 0x110, - 0x111, - 0x112, - 0x113, - 0x114, - 0x115, - 0x116, - 0x117, - 0x118, - 0x119, - 0x11A, - 0x11B, - 0x11C, - 0x11D, - 0x11E, - 0x11F, - 0x120, - 0x121, - 0x122, - 0x123, - 0x124, - 0x125, - 0x126, - 0x127, - 0x128, - 0x129, - 0x12A, - 0x12B, - 0x12C, - 0x12D, - 0x12E, - 0x12F, - 0x130, - 0x131, - 0x132, - 0x134, - 0x135, - 0x136, - 0x137, - 0x139, - 0x13A, - 0x13B, - 0x13C, - 0x13D, - 0x13E, - 0x13F, - 0x141, - 0x142, - 0x143, - 0x144, - 0x145, - 0x146, - 0x147, - 0x148, - 0x149, - 0x14A, - 0x14B, - 0x14C, - 0x14D, - 0x14E, - 0x14F, - 0x150, - 0x151, - 0x152, - 0x153, - 0x154, - 0x155, - 0x156, - 0x157, - 0x158, - 0x159, - 0x15A, - 0x15B, - 0x15C, - 0x15D, - 0x15E, - 0x15F, - 0x160, - 0x161, - 0x162, - 0x163, - 0x164, - 0x165, - 0x166, - 0x167, - 0x168, - 0x169, - 0x16A, - 0x16B, - 0x16C, - 0x16D, - 0x16E, - 0x16F, - 0x170, - 0x171, - 0x172, - 0x173, - 0x174, - 0x175, - 0x176, - 0x177, - 0x178, - 0x179, - 0x17A, - 0x17B, - 0x17C, - 0x17D, - 0x17E, - 0x17F, - 0x180, - 0x181, - 0x182, - 0x183, - 0x184, - 0x185, - 0x186, - 0x187, - 0x188, - 0x189, - 0x18A, - 0x18B, - 0x18C, - 0x18E, - 0x18F, - 0x190, - 0x191, - 0x192, - 0x193, - 0x194, - 0x195, - 0x196, - 0x197, - 0x198, - 0x199, - 0x19C, - 0x19D, - 0x19E, - 0x19F, - 0x1A0, - 0x1A1, - 0x1A2, - 0x1A3, - 0x1A4, - 0x1A5, - 0x1A6, - 0x1A7, - 0x1A8, - 0x1A9, - 0x1AA, - 0x1AC, - 0x1AD, - 0x1AE, - 0x1AF, - 0x1B0, - 0x1B1, - 0x1B2, - 0x1B3, - 0x1B4, - 0x1B5, - 0x1B6, - 0x1B7, - 0x1B8, - 0x1B9, - 0x1BC, - 0x1BD, - 0x1C4, - 0x1C7, - 0x1CA, - 0x1CD, - 0x1CE, - 0x1CF, - 0x1D0, - 0x1D1, - 0x1D2, - 0x1D3, - 0x1D4, - 0x1D5, - 0x1D6, - 0x1D7, - 0x1D8, - 0x1D9, - 0x1DA, - 0x1DB, - 0x1DC, - 0x1DE, - 0x1DF, - 0x1E0, - 0x1E1, - 0x1E2, - 0x1E3, - 0x1E4, - 0x1E5, - 0x1E6, - 0x1E7, - 0x1E8, - 0x1E9, - 0x1EA, - 0x1EB, - 0x1EC, - 0x1ED, - 0x1EE, - 0x1EF, - 0x1F1, - 0x1F4, - 0x1F5, - 0x1F6, - 0x1F7, - 0x1F8, - 0x1F9, - 0x1FA, - 0x1FB, - 0x1FC, - 0x1FD, - 0x1FE, - 0x1FF, - 0x200, - 0x201, - 0x202, - 0x203, - 0x204, - 0x205, - 0x206, - 0x207, - 0x208, - 0x209, - 0x20A, - 0x20B, - 0x20C, - 0x20D, - 0x20E, - 0x20F, - 0x210, - 0x211, - 0x212, - 0x213, - 0x214, - 0x215, - 0x216, - 0x217, - 0x218, - 0x219, - 0x21A, - 0x21B, - 0x21C, - 0x21D, - 0x21E, - 0x21F, - 0x220, - 0x221, - 0x222, - 0x223, - 0x224, - 0x225, - 0x226, - 0x227, - 0x228, - 0x229, - 0x22A, - 0x22B, - 0x22C, - 0x22D, - 0x22E, - 0x22F, - 0x230, - 0x231, - 0x232, - 0x233, - 0x23A, - 0x23B, - 0x23C, - 0x23D, - 0x23E, - 0x23F, - 0x241, - 0x242, - 0x243, - 0x244, - 0x245, - 0x246, - 0x247, - 0x248, - 0x249, - 0x24A, - 0x24B, - 0x24C, - 0x24D, - 0x24E, - 0x24F, - 0x2B0, - 0x2B1, - 0x2B2, - 0x2B3, - 0x2B4, - 0x2B5, - 0x2B6, - 0x2B7, - 0x2B8, - 0x2B9, - 0x2D8, - 0x2D9, - 0x2DA, - 0x2DB, - 0x2DC, - 0x2DD, - 0x2DE, - 0x2E0, - 0x2E1, - 0x2E2, - 0x2E3, - 0x2E4, - 0x2E5, - 0x340, - 0x341, - 0x342, - 0x343, - 0x344, - 0x345, - 0x346, - 0x34F, - 0x350, - 0x370, - 0x371, - 0x372, - 0x373, - 0x374, - 0x375, - 0x376, - 0x377, - 0x378, - 0x37A, - 0x37B, - 0x37E, - 0x37F, - 0x380, - 0x384, - 0x385, - 0x386, - 0x387, - 0x388, - 0x389, - 0x38A, - 0x38B, - 0x38C, - 0x38D, - 0x38E, - 0x38F, - 0x390, - 0x391, - 0x392, - 0x393, - 0x394, - 0x395, - 0x396, - 0x397, - 0x398, - 0x399, - 0x39A, - 0x39B, - 0x39C, - 0x39D, - 0x39E, - 0x39F, - 0x3A0, - 0x3A1, - 0x3A2, - 0x3A3, - 0x3A4, - 0x3A5, - 0x3A6, - 0x3A7, - 0x3A8, - 0x3A9, - 0x3AA, - 0x3AB, - 0x3AC, - 0x3C2, - 0x3C3, - 0x3CF, - 0x3D0, - 0x3D1, - 0x3D2, - 0x3D3, - 0x3D4, - 0x3D5, - 0x3D6, - 0x3D7, - 0x3D8, - 0x3D9, - 0x3DA, - 0x3DB, - 0x3DC, - 0x3DD, - 0x3DE, - 0x3DF, - 0x3E0, - 0x3E1, - 0x3E2, - 0x3E3, - 0x3E4, - 0x3E5, - 0x3E6, - 0x3E7, - 0x3E8, - 0x3E9, - 0x3EA, - 0x3EB, - 0x3EC, - 0x3ED, - 0x3EE, - 0x3EF, - 0x3F0, - 0x3F1, - 0x3F2, - 0x3F3, - 0x3F4, - 0x3F5, - 0x3F6, - 0x3F7, - 0x3F8, - 0x3F9, - 0x3FA, - 0x3FB, - 0x3FD, - 0x3FE, - 0x3FF, - 0x400, - 0x401, - 0x402, - 0x403, - 0x404, - 0x405, - 0x406, - 0x407, - 0x408, - 0x409, - 0x40A, - 0x40B, - 0x40C, - 0x40D, - 0x40E, - 0x40F, - 0x410, - 0x411, - 0x412, - 0x413, - 0x414, - 0x415, - 0x416, - 0x417, - 0x418, - 0x419, - 0x41A, - 0x41B, - 0x41C, - 0x41D, - 0x41E, - 0x41F, - 0x420, - 0x421, - 0x422, - 0x423, - 0x424, - 0x425, - 0x426, - 0x427, - 0x428, - 0x429, - 0x42A, - 0x42B, - 0x42C, - 0x42D, - 0x42E, - 0x42F, - 0x430, - 0x460, - 0x461, - 0x462, - 0x463, - 0x464, - 0x465, - 0x466, - 0x467, - 0x468, - 0x469, - 0x46A, - 0x46B, - 0x46C, - 0x46D, - 0x46E, - 0x46F, - 0x470, - 0x471, - 0x472, - 0x473, - 0x474, - 0x475, - 0x476, - 0x477, - 0x478, - 0x479, - 0x47A, - 0x47B, - 0x47C, - 0x47D, - 0x47E, - 0x47F, - 0x480, - 0x481, - 0x48A, - 0x48B, - 0x48C, - 0x48D, - 0x48E, - 0x48F, - 0x490, - 0x491, - 0x492, - 0x493, - 0x494, - 0x495, - 0x496, - 0x497, - 0x498, - 0x499, - 0x49A, - 0x49B, - 0x49C, - 0x49D, - 0x49E, - 0x49F, - 0x4A0, - 0x4A1, - 0x4A2, - 0x4A3, - 0x4A4, - 0x4A5, - 0x4A6, - 0x4A7, - 0x4A8, - 0x4A9, - 0x4AA, - 0x4AB, - 0x4AC, - 0x4AD, - 0x4AE, - 0x4AF, - 0x4B0, - 0x4B1, - 0x4B2, - 0x4B3, - 0x4B4, - 0x4B5, - 0x4B6, - 0x4B7, - 0x4B8, - 0x4B9, - 0x4BA, - 0x4BB, - 0x4BC, - 0x4BD, - 0x4BE, - 0x4BF, - 0x4C0, - 0x4C1, - 0x4C2, - 0x4C3, - 0x4C4, - 0x4C5, - 0x4C6, - 0x4C7, - 0x4C8, - 0x4C9, - 0x4CA, - 0x4CB, - 0x4CC, - 0x4CD, - 0x4CE, - 0x4D0, - 0x4D1, - 0x4D2, - 0x4D3, - 0x4D4, - 0x4D5, - 0x4D6, - 0x4D7, - 0x4D8, - 0x4D9, - 0x4DA, - 0x4DB, - 0x4DC, - 0x4DD, - 0x4DE, - 0x4DF, - 0x4E0, - 0x4E1, - 0x4E2, - 0x4E3, - 0x4E4, - 0x4E5, - 0x4E6, - 0x4E7, - 0x4E8, - 0x4E9, - 0x4EA, - 0x4EB, - 0x4EC, - 0x4ED, - 0x4EE, - 0x4EF, - 0x4F0, - 0x4F1, - 0x4F2, - 0x4F3, - 0x4F4, - 0x4F5, - 0x4F6, - 0x4F7, - 0x4F8, - 0x4F9, - 0x4FA, - 0x4FB, - 0x4FC, - 0x4FD, - 0x4FE, - 0x4FF, - 0x500, - 0x501, - 0x502, - 0x503, - 0x504, - 0x505, - 0x506, - 0x507, - 0x508, - 0x509, - 0x50A, - 0x50B, - 0x50C, - 0x50D, - 0x50E, - 0x50F, - 0x510, - 0x511, - 0x512, - 0x513, - 0x514, - 0x515, - 0x516, - 0x517, - 0x518, - 0x519, - 0x51A, - 0x51B, - 0x51C, - 0x51D, - 0x51E, - 0x51F, - 0x520, - 0x521, - 0x522, - 0x523, - 0x524, - 0x525, - 0x526, - 0x527, - 0x528, - 0x529, - 0x52A, - 0x52B, - 0x52C, - 0x52D, - 0x52E, - 0x52F, - 0x530, - 0x531, - 0x532, - 0x533, - 0x534, - 0x535, - 0x536, - 0x537, - 0x538, - 0x539, - 0x53A, - 0x53B, - 0x53C, - 0x53D, - 0x53E, - 0x53F, - 0x540, - 0x541, - 0x542, - 0x543, - 0x544, - 0x545, - 0x546, - 0x547, - 0x548, - 0x549, - 0x54A, - 0x54B, - 0x54C, - 0x54D, - 0x54E, - 0x54F, - 0x550, - 0x551, - 0x552, - 0x553, - 0x554, - 0x555, - 0x556, - 0x557, - 0x559, - 0x587, - 0x588, - 0x58B, - 0x58D, - 0x590, - 0x591, - 0x5C8, - 0x5D0, - 0x5EB, - 0x5EF, - 0x5F5, - 0x606, - 0x61C, - 0x61D, - 0x675, - 0x676, - 0x677, - 0x678, - 0x679, - 0x6DD, - 0x6DE, - 0x70E, - 0x710, - 0x74B, - 0x74D, - 0x7B2, - 0x7C0, - 0x7FB, - 0x7FD, - 0x82E, - 0x830, - 0x83F, - 0x840, - 0x85C, - 0x85E, - 0x85F, - 0x860, - 0x86B, - 0x870, - 0x890, - 0x897, - 0x8E2, - 0x8E3, - 0x958, - 0x959, - 0x95A, - 0x95B, - 0x95C, - 0x95D, - 0x95E, - 0x95F, - 0x960, - 0x984, - 0x985, - 0x98D, - 0x98F, - 0x991, - 0x993, - 0x9A9, - 0x9AA, - 0x9B1, - 0x9B2, - 0x9B3, - 0x9B6, - 0x9BA, - 0x9BC, - 0x9C5, - 0x9C7, - 0x9C9, - 0x9CB, - 0x9CF, - 0x9D7, - 0x9D8, - 0x9DC, - 0x9DD, - 0x9DE, - 0x9DF, - 0x9E0, - 0x9E4, - 0x9E6, - 0x9FF, - 0xA01, - 0xA04, - 0xA05, - 0xA0B, - 0xA0F, - 0xA11, - 0xA13, - 0xA29, - 0xA2A, - 0xA31, - 0xA32, - 0xA33, - 0xA34, - 0xA35, - 0xA36, - 0xA37, - 0xA38, - 0xA3A, - 0xA3C, - 0xA3D, - 0xA3E, - 0xA43, - 0xA47, - 0xA49, - 0xA4B, - 0xA4E, - 0xA51, - 0xA52, - 0xA59, - 0xA5A, - 0xA5B, - 0xA5C, - 0xA5D, - 0xA5E, - 0xA5F, - 0xA66, - 0xA77, - 0xA81, - 0xA84, - 0xA85, - 0xA8E, - 0xA8F, - 0xA92, - 0xA93, - 0xAA9, - 0xAAA, - 0xAB1, - 0xAB2, - 0xAB4, - 0xAB5, - 0xABA, - 0xABC, - 0xAC6, - 0xAC7, - 0xACA, - 0xACB, - 0xACE, - 0xAD0, - 0xAD1, - 0xAE0, - 0xAE4, - 0xAE6, - 0xAF2, - 0xAF9, - 0xB00, - 0xB01, - 0xB04, - 0xB05, - 0xB0D, - 0xB0F, - 0xB11, - 0xB13, - 0xB29, - 0xB2A, - 0xB31, - 0xB32, - 0xB34, - 0xB35, - 0xB3A, - 0xB3C, - 0xB45, - 0xB47, - 0xB49, - 0xB4B, - 0xB4E, - 0xB55, - 0xB58, - 0xB5C, - 0xB5D, - 0xB5E, - 0xB5F, - 0xB64, - 0xB66, - 0xB78, - 0xB82, - 0xB84, - 0xB85, - 0xB8B, - 0xB8E, - 0xB91, - 0xB92, - 0xB96, - 0xB99, - 0xB9B, - 0xB9C, - 0xB9D, - 0xB9E, - 0xBA0, - 0xBA3, - 0xBA5, - 0xBA8, - 0xBAB, - 0xBAE, - 0xBBA, - 0xBBE, - 0xBC3, - 0xBC6, - 0xBC9, - 0xBCA, - 0xBCE, - 0xBD0, - 0xBD1, - 0xBD7, - 0xBD8, - 0xBE6, - 0xBFB, - 0xC00, - 0xC0D, - 0xC0E, - 0xC11, - 0xC12, - 0xC29, - 0xC2A, - 0xC3A, - 0xC3C, - 0xC45, - 0xC46, - 0xC49, - 0xC4A, - 0xC4E, - 0xC55, - 0xC57, - 0xC58, - 0xC5B, - 0xC5C, - 0xC5E, - 0xC60, - 0xC64, - 0xC66, - 0xC70, - 0xC77, - 0xC8D, - 0xC8E, - 0xC91, - 0xC92, - 0xCA9, - 0xCAA, - 0xCB4, - 0xCB5, - 0xCBA, - 0xCBC, - 0xCC5, - 0xCC6, - 0xCC9, - 0xCCA, - 0xCCE, - 0xCD5, - 0xCD7, - 0xCDC, - 0xCDF, - 0xCE0, - 0xCE4, - 0xCE6, - 0xCF0, - 0xCF1, - 0xCF4, - 0xD00, - 0xD0D, - 0xD0E, - 0xD11, - 0xD12, - 0xD45, - 0xD46, - 0xD49, - 0xD4A, - 0xD50, - 0xD54, - 0xD64, - 0xD66, - 0xD80, - 0xD81, - 0xD84, - 0xD85, - 0xD97, - 0xD9A, - 0xDB2, - 0xDB3, - 0xDBC, - 0xDBD, - 0xDBE, - 0xDC0, - 0xDC7, - 0xDCA, - 0xDCB, - 0xDCF, - 0xDD5, - 0xDD6, - 0xDD7, - 0xDD8, - 0xDE0, - 0xDE6, - 0xDF0, - 0xDF2, - 0xDF5, - 0xE01, - 0xE33, - 0xE34, - 0xE3B, - 0xE3F, - 0xE5C, - 0xE81, - 0xE83, - 0xE84, - 0xE85, - 0xE86, - 0xE8B, - 0xE8C, - 0xEA4, - 0xEA5, - 0xEA6, - 0xEA7, - 0xEB3, - 0xEB4, - 0xEBE, - 0xEC0, - 0xEC5, - 0xEC6, - 0xEC7, - 0xEC8, - 0xECF, - 0xED0, - 0xEDA, - 0xEDC, - 0xEDD, - 0xEDE, - 0xEE0, - 0xF00, - 0xF0C, - 0xF0D, - 0xF43, - 0xF44, - 0xF48, - 0xF49, - 0xF4D, - 0xF4E, - 0xF52, - 0xF53, - 0xF57, - 0xF58, - 0xF5C, - 0xF5D, - 0xF69, - 0xF6A, - 0xF6D, - 0xF71, - 0xF73, - 0xF74, - 0xF75, - 0xF76, - 0xF77, - 0xF78, - 0xF79, - 0xF7A, - 0xF81, - 0xF82, - 0xF93, - 0xF94, - 0xF98, - 0xF99, - 0xF9D, - 0xF9E, - 0xFA2, - 0xFA3, - 0xFA7, - 0xFA8, - 0xFAC, - 0xFAD, - 0xFB9, - 0xFBA, - 0xFBD, - 0xFBE, - 0xFCD, - 0xFCE, - 0xFDB, - 0x1000, - 0x10A0, - 0x10A1, - 0x10A2, - 0x10A3, - 0x10A4, - 0x10A5, - 0x10A6, - 0x10A7, - 0x10A8, - 0x10A9, - 0x10AA, - 0x10AB, - 0x10AC, - 0x10AD, - 0x10AE, - 0x10AF, - 0x10B0, - 0x10B1, - 0x10B2, - 0x10B3, - 0x10B4, - 0x10B5, - 0x10B6, - 0x10B7, - 0x10B8, - 0x10B9, - 0x10BA, - 0x10BB, - 0x10BC, - 0x10BD, - 0x10BE, - 0x10BF, - 0x10C0, - 0x10C1, - 0x10C2, - 0x10C3, - 0x10C4, - 0x10C5, - 0x10C6, - 0x10C7, - 0x10C8, - 0x10CD, - 0x10CE, - 0x10D0, - 0x10FC, - 0x10FD, - 0x115F, - 0x1161, - 0x1249, - 0x124A, - 0x124E, - 0x1250, - 0x1257, - 0x1258, - 0x1259, - 0x125A, - 0x125E, - 0x1260, - 0x1289, - 0x128A, - 0x128E, - 0x1290, - 0x12B1, - 0x12B2, - 0x12B6, - 0x12B8, - 0x12BF, - 0x12C0, - 0x12C1, - 0x12C2, - 0x12C6, - 0x12C8, - 0x12D7, - 0x12D8, - 0x1311, - 0x1312, - 0x1316, - 0x1318, - 0x135B, - 0x135D, - 0x137D, - 0x1380, - 0x139A, - 0x13A0, - 0x13F6, - 0x13F8, - 0x13F9, - 0x13FA, - 0x13FB, - 0x13FC, - 0x13FD, - 0x13FE, - 0x1400, - 0x1680, - 0x1681, - 0x169D, - 0x16A0, - 0x16F9, - 0x1700, - 0x1716, - 0x171F, - 0x1737, - 0x1740, - 0x1754, - 0x1760, - 0x176D, - 0x176E, - 0x1771, - 0x1772, - 0x1774, - 0x1780, - 0x17B4, - 0x17B6, - 0x17DE, - 0x17E0, - 0x17EA, - 0x17F0, - 0x17FA, - 0x1800, - 0x180B, - 0x1810, - 0x181A, - 0x1820, - 0x1879, - 0x1880, - 0x18AB, - 0x18B0, - 0x18F6, - 0x1900, - 0x191F, - 0x1920, - 0x192C, - 0x1930, - 0x193C, - 0x1940, - 0x1941, - 0x1944, - 0x196E, - 0x1970, - 0x1975, - 0x1980, - 0x19AC, - 0x19B0, - 0x19CA, - 0x19D0, - 0x19DB, - 0x19DE, - 0x1A1C, - 0x1A1E, - 0x1A5F, - 0x1A60, - 0x1A7D, - 0x1A7F, - 0x1A8A, - 0x1A90, - 0x1A9A, - 0x1AA0, - 0x1AAE, - 0x1AB0, - 0x1ADE, - 0x1AE0, - 0x1AEC, - 0x1B00, - 0x1B4D, - 0x1B4E, - 0x1BF4, - 0x1BFC, - 0x1C38, - 0x1C3B, - 0x1C4A, - 0x1C4D, - 0x1C80, - 0x1C81, - 0x1C82, - 0x1C83, - 0x1C84, - 0x1C86, - 0x1C87, - 0x1C88, - 0x1C89, - 0x1C8A, - 0x1C8B, - 0x1C90, - 0x1C91, - 0x1C92, - 0x1C93, - 0x1C94, - 0x1C95, - 0x1C96, - 0x1C97, - 0x1C98, - 0x1C99, - 0x1C9A, - 0x1C9B, - 0x1C9C, - 0x1C9D, - 0x1C9E, - 0x1C9F, - 0x1CA0, - 0x1CA1, - 0x1CA2, - 0x1CA3, - 0x1CA4, - 0x1CA5, - 0x1CA6, - 0x1CA7, - 0x1CA8, - 0x1CA9, - 0x1CAA, - 0x1CAB, - 0x1CAC, - 0x1CAD, - 0x1CAE, - 0x1CAF, - 0x1CB0, - 0x1CB1, - 0x1CB2, - 0x1CB3, - 0x1CB4, - 0x1CB5, - 0x1CB6, - 0x1CB7, - 0x1CB8, - 0x1CB9, - 0x1CBA, - 0x1CBB, - 0x1CBD, - 0x1CBE, - 0x1CBF, - 0x1CC0, - 0x1CC8, - 0x1CD0, - 0x1CFB, - 0x1D00, - 0x1D2C, - 0x1D2D, - 0x1D2E, - 0x1D2F, - 0x1D30, - 0x1D31, - 0x1D32, - 0x1D33, - 0x1D34, - 0x1D35, - 0x1D36, - 0x1D37, - 0x1D38, - 0x1D39, - 0x1D3A, - 0x1D3B, - 0x1D3C, - 0x1D3D, - 0x1D3E, - 0x1D3F, - 0x1D40, - 0x1D41, - 0x1D42, - 0x1D43, - 0x1D44, - 0x1D45, - 0x1D46, - 0x1D47, - 0x1D48, - 0x1D49, - 0x1D4A, - 0x1D4B, - 0x1D4C, - 0x1D4D, - 0x1D4E, - 0x1D4F, - 0x1D50, - 0x1D51, - 0x1D52, - 0x1D53, - 0x1D54, - 0x1D55, - 0x1D56, - 0x1D57, - 0x1D58, - 0x1D59, - 0x1D5A, - 0x1D5B, - 0x1D5C, - 0x1D5D, - 0x1D5E, - 0x1D5F, - 0x1D60, - 0x1D61, - 0x1D62, - 0x1D63, - 0x1D64, - 0x1D65, - 0x1D66, - 0x1D67, - 0x1D68, - 0x1D69, - 0x1D6A, - 0x1D6B, - 0x1D78, - 0x1D79, - 0x1D9B, - 0x1D9C, - 0x1D9D, - 0x1D9E, - 0x1D9F, - 0x1DA0, - 0x1DA1, - 0x1DA2, - 0x1DA3, - 0x1DA4, - 0x1DA5, - 0x1DA6, - 0x1DA7, - 0x1DA8, - 0x1DA9, - 0x1DAA, - 0x1DAB, - 0x1DAC, - 0x1DAD, - 0x1DAE, - 0x1DAF, - 0x1DB0, - 0x1DB1, - 0x1DB2, - 0x1DB3, - 0x1DB4, - 0x1DB5, - 0x1DB6, - 0x1DB7, - 0x1DB8, - 0x1DB9, - 0x1DBA, - 0x1DBB, - 0x1DBC, - 0x1DBD, - 0x1DBE, - 0x1DBF, - 0x1DC0, - 0x1E00, - 0x1E01, - 0x1E02, - 0x1E03, - 0x1E04, - 0x1E05, - 0x1E06, - 0x1E07, - 0x1E08, - 0x1E09, - 0x1E0A, - 0x1E0B, - 0x1E0C, - 0x1E0D, - 0x1E0E, - 0x1E0F, - 0x1E10, - 0x1E11, - 0x1E12, - 0x1E13, - 0x1E14, - 0x1E15, - 0x1E16, - 0x1E17, - 0x1E18, - 0x1E19, - 0x1E1A, - 0x1E1B, - 0x1E1C, - 0x1E1D, - 0x1E1E, - 0x1E1F, - 0x1E20, - 0x1E21, - 0x1E22, - 0x1E23, - 0x1E24, - 0x1E25, - 0x1E26, - 0x1E27, - 0x1E28, - 0x1E29, - 0x1E2A, - 0x1E2B, - 0x1E2C, - 0x1E2D, - 0x1E2E, - 0x1E2F, - 0x1E30, - 0x1E31, - 0x1E32, - 0x1E33, - 0x1E34, - 0x1E35, - 0x1E36, - 0x1E37, - 0x1E38, - 0x1E39, - 0x1E3A, - 0x1E3B, - 0x1E3C, - 0x1E3D, - 0x1E3E, - 0x1E3F, - 0x1E40, - 0x1E41, - 0x1E42, - 0x1E43, - 0x1E44, - 0x1E45, - 0x1E46, - 0x1E47, - 0x1E48, - 0x1E49, - 0x1E4A, - 0x1E4B, - 0x1E4C, - 0x1E4D, - 0x1E4E, - 0x1E4F, - 0x1E50, - 0x1E51, - 0x1E52, - 0x1E53, - 0x1E54, - 0x1E55, - 0x1E56, - 0x1E57, - 0x1E58, - 0x1E59, - 0x1E5A, - 0x1E5B, - 0x1E5C, - 0x1E5D, - 0x1E5E, - 0x1E5F, - 0x1E60, - 0x1E61, - 0x1E62, - 0x1E63, - 0x1E64, - 0x1E65, - 0x1E66, - 0x1E67, - 0x1E68, - 0x1E69, - 0x1E6A, - 0x1E6B, - 0x1E6C, - 0x1E6D, - 0x1E6E, - 0x1E6F, - 0x1E70, - 0x1E71, - 0x1E72, - 0x1E73, - 0x1E74, - 0x1E75, - 0x1E76, - 0x1E77, - 0x1E78, - 0x1E79, - 0x1E7A, - 0x1E7B, - 0x1E7C, - 0x1E7D, - 0x1E7E, - 0x1E7F, - 0x1E80, - 0x1E81, - 0x1E82, - 0x1E83, - 0x1E84, - 0x1E85, - 0x1E86, - 0x1E87, - 0x1E88, - 0x1E89, - 0x1E8A, - 0x1E8B, - 0x1E8C, - 0x1E8D, - 0x1E8E, - 0x1E8F, - 0x1E90, - 0x1E91, - 0x1E92, - 0x1E93, - 0x1E94, - 0x1E95, - 0x1E9A, - 0x1E9B, - 0x1E9C, - 0x1E9E, - 0x1E9F, - 0x1EA0, - 0x1EA1, - 0x1EA2, - 0x1EA3, - 0x1EA4, - 0x1EA5, - 0x1EA6, - 0x1EA7, - 0x1EA8, - 0x1EA9, - 0x1EAA, - 0x1EAB, - 0x1EAC, - 0x1EAD, - 0x1EAE, - 0x1EAF, - 0x1EB0, - 0x1EB1, - 0x1EB2, - 0x1EB3, - 0x1EB4, - 0x1EB5, - 0x1EB6, - 0x1EB7, - 0x1EB8, - 0x1EB9, - 0x1EBA, - 0x1EBB, - 0x1EBC, - 0x1EBD, - 0x1EBE, - 0x1EBF, - 0x1EC0, - 0x1EC1, - 0x1EC2, - 0x1EC3, - 0x1EC4, - 0x1EC5, - 0x1EC6, - 0x1EC7, - 0x1EC8, - 0x1EC9, - 0x1ECA, - 0x1ECB, - 0x1ECC, - 0x1ECD, - 0x1ECE, - 0x1ECF, - 0x1ED0, - 0x1ED1, - 0x1ED2, - 0x1ED3, - 0x1ED4, - 0x1ED5, - 0x1ED6, - 0x1ED7, - 0x1ED8, - 0x1ED9, - 0x1EDA, - 0x1EDB, - 0x1EDC, - 0x1EDD, - 0x1EDE, - 0x1EDF, - 0x1EE0, - 0x1EE1, - 0x1EE2, - 0x1EE3, - 0x1EE4, - 0x1EE5, - 0x1EE6, - 0x1EE7, - 0x1EE8, - 0x1EE9, - 0x1EEA, - 0x1EEB, - 0x1EEC, - 0x1EED, - 0x1EEE, - 0x1EEF, - 0x1EF0, - 0x1EF1, - 0x1EF2, - 0x1EF3, - 0x1EF4, - 0x1EF5, - 0x1EF6, - 0x1EF7, - 0x1EF8, - 0x1EF9, - 0x1EFA, - 0x1EFB, - 0x1EFC, - 0x1EFD, - 0x1EFE, - 0x1EFF, - 0x1F08, - 0x1F09, - 0x1F0A, - 0x1F0B, - 0x1F0C, - 0x1F0D, - 0x1F0E, - 0x1F0F, - 0x1F10, - 0x1F16, - 0x1F18, - 0x1F19, - 0x1F1A, - 0x1F1B, - 0x1F1C, - 0x1F1D, - 0x1F1E, - 0x1F20, - 0x1F28, - 0x1F29, - 0x1F2A, - 0x1F2B, - 0x1F2C, - 0x1F2D, - 0x1F2E, - 0x1F2F, - 0x1F30, - 0x1F38, - 0x1F39, - 0x1F3A, - 0x1F3B, - 0x1F3C, - 0x1F3D, - 0x1F3E, - 0x1F3F, - 0x1F40, - 0x1F46, - 0x1F48, - 0x1F49, - 0x1F4A, - 0x1F4B, - 0x1F4C, - 0x1F4D, - 0x1F4E, - 0x1F50, - 0x1F58, - 0x1F59, - 0x1F5A, - 0x1F5B, - 0x1F5C, - 0x1F5D, - 0x1F5E, - 0x1F5F, - 0x1F60, - 0x1F68, - 0x1F69, - 0x1F6A, - 0x1F6B, - 0x1F6C, - 0x1F6D, - 0x1F6E, - 0x1F6F, - 0x1F70, - 0x1F71, - 0x1F72, - 0x1F73, - 0x1F74, - 0x1F75, - 0x1F76, - 0x1F77, - 0x1F78, - 0x1F79, - 0x1F7A, - 0x1F7B, - 0x1F7C, - 0x1F7D, - 0x1F7E, - 0x1F80, - 0x1F81, - 0x1F82, - 0x1F83, - 0x1F84, - 0x1F85, - 0x1F86, - 0x1F87, - 0x1F88, - 0x1F89, - 0x1F8A, - 0x1F8B, - 0x1F8C, - 0x1F8D, - 0x1F8E, - 0x1F8F, - 0x1F90, - 0x1F91, - 0x1F92, - 0x1F93, - 0x1F94, - 0x1F95, - 0x1F96, - 0x1F97, - 0x1F98, - 0x1F99, - 0x1F9A, - 0x1F9B, - 0x1F9C, - 0x1F9D, - 0x1F9E, - 0x1F9F, - 0x1FA0, - 0x1FA1, - 0x1FA2, - 0x1FA3, - 0x1FA4, - 0x1FA5, - 0x1FA6, - 0x1FA7, - 0x1FA8, - 0x1FA9, - 0x1FAA, - 0x1FAB, - 0x1FAC, - 0x1FAD, - 0x1FAE, - 0x1FAF, - 0x1FB0, - 0x1FB2, - 0x1FB3, - 0x1FB4, - 0x1FB5, - 0x1FB6, - 0x1FB7, - 0x1FB8, - 0x1FB9, - 0x1FBA, - 0x1FBB, - 0x1FBC, - 0x1FBD, - 0x1FBE, - 0x1FBF, - 0x1FC0, - 0x1FC1, - 0x1FC2, - 0x1FC3, - 0x1FC4, - 0x1FC5, - 0x1FC6, - 0x1FC7, - 0x1FC8, - 0x1FC9, - 0x1FCA, - 0x1FCB, - 0x1FCC, - 0x1FCD, - 0x1FCE, - 0x1FCF, - 0x1FD0, - 0x1FD3, - 0x1FD4, - 0x1FD6, - 0x1FD8, - 0x1FD9, - 0x1FDA, - 0x1FDB, - 0x1FDC, - 0x1FDD, - 0x1FDE, - 0x1FDF, - 0x1FE0, - 0x1FE3, - 0x1FE4, - 0x1FE8, - 0x1FE9, - 0x1FEA, - 0x1FEB, - 0x1FEC, - 0x1FED, - 0x1FEE, - 0x1FEF, - 0x1FF0, - 0x1FF2, - 0x1FF3, - 0x1FF4, - 0x1FF5, - 0x1FF6, - 0x1FF7, - 0x1FF8, - 0x1FF9, - 0x1FFA, - 0x1FFB, - 0x1FFC, - 0x1FFD, - 0x1FFE, - 0x1FFF, - 0x2000, - 0x200B, - 0x200C, - 0x200E, - 0x2010, - 0x2011, - 0x2012, - 0x2017, - 0x2018, - 0x2024, - 0x2027, - 0x2028, - 0x202F, - 0x2030, - 0x2033, - 0x2034, - 0x2035, - 0x2036, - 0x2037, - 0x2038, - 0x203C, - 0x203D, - 0x203E, - 0x203F, - 0x2047, - 0x2048, - 0x2049, - 0x204A, - 0x2057, - 0x2058, - 0x205F, - 0x2060, - 0x2065, - 0x206A, - 0x2070, - 0x2071, - 0x2072, - 0x2074, - 0x2075, - 0x2076, - 0x2077, - 0x2078, - 0x2079, - 0x207A, - 0x207B, - 0x207C, - 0x207D, - 0x207E, - 0x207F, - 0x2080, - 0x2081, - 0x2082, - 0x2083, - 0x2084, - 0x2085, - 0x2086, - 0x2087, - 0x2088, - 0x2089, - 0x208A, - 0x208B, - 0x208C, - 0x208D, - 0x208E, - 0x208F, - 0x2090, - 0x2091, - 0x2092, - 0x2093, - 0x2094, - 0x2095, - 0x2096, - 0x2097, - 0x2098, - 0x2099, - 0x209A, - 0x209B, - 0x209C, - 0x209D, - 0x20A0, - 0x20A8, - 0x20A9, - 0x20C2, - 0x20D0, - 0x20F1, - 0x2100, - 0x2101, - 0x2102, - 0x2103, - 0x2104, - 0x2105, - 0x2106, - 0x2107, - 0x2108, - 0x2109, - 0x210A, - 0x210B, - 0x210F, - 0x2110, - 0x2112, - 0x2114, - 0x2115, - 0x2116, - 0x2117, - 0x2119, - 0x211A, - 0x211B, - 0x211E, - 0x2120, - 0x2121, - 0x2122, - 0x2123, - 0x2124, - 0x2125, - 0x2126, - 0x2127, - 0x2128, - 0x2129, - 0x212A, - 0x212B, - 0x212C, - 0x212D, - 0x212E, - 0x212F, - 0x2131, - 0x2132, - 0x2133, - 0x2134, - 0x2135, - 0x2136, - 0x2137, - 0x2138, - 0x2139, - 0x213A, - 0x213B, - 0x213C, - 0x213D, - 0x213F, - 0x2140, - 0x2141, - 0x2145, - 0x2147, - 0x2148, - 0x2149, - 0x214A, - 0x2150, - 0x2151, - 0x2152, - 0x2153, - 0x2154, - 0x2155, - 0x2156, - 0x2157, - 0x2158, - 0x2159, - 0x215A, - 0x215B, - 0x215C, - 0x215D, - 0x215E, - 0x215F, - 0x2160, - 0x2161, - 0x2162, - 0x2163, - 0x2164, - 0x2165, - 0x2166, - 0x2167, - 0x2168, - 0x2169, - 0x216A, - 0x216B, - 0x216C, - 0x216D, - 0x216E, - 0x216F, - 0x2170, - 0x2171, - 0x2172, - 0x2173, - 0x2174, - 0x2175, - 0x2176, - 0x2177, - 0x2178, - 0x2179, - 0x217A, - 0x217B, - 0x217C, - 0x217D, - 0x217E, - 0x217F, - 0x2180, - 0x2183, - 0x2184, - 0x2189, - 0x218A, - 0x218C, - 0x2190, - 0x222C, - 0x222D, - 0x222E, - 0x222F, - 0x2230, - 0x2231, - 0x2329, - 0x232A, - 0x232B, - 0x242A, - 0x2440, - 0x244B, - 0x2460, - 0x2461, - 0x2462, - 0x2463, - 0x2464, - 0x2465, - 0x2466, - 0x2467, - 0x2468, - 0x2469, - 0x246A, - 0x246B, - 0x246C, - 0x246D, - 0x246E, - 0x246F, - 0x2470, - 0x2471, - 0x2472, - 0x2473, - 0x2474, - 0x2475, - 0x2476, - 0x2477, - 0x2478, - 0x2479, - 0x247A, - 0x247B, - 0x247C, - 0x247D, - 0x247E, - 0x247F, - 0x2480, - 0x2481, - 0x2482, - 0x2483, - 0x2484, - 0x2485, - 0x2486, - 0x2487, - 0x2488, - 0x249C, - 0x249D, - 0x249E, - 0x249F, - 0x24A0, - 0x24A1, - 0x24A2, - 0x24A3, - 0x24A4, - 0x24A5, - 0x24A6, - 0x24A7, - 0x24A8, - 0x24A9, - 0x24AA, - 0x24AB, - 0x24AC, - 0x24AD, - 0x24AE, - 0x24AF, - 0x24B0, - 0x24B1, - 0x24B2, - 0x24B3, - 0x24B4, - 0x24B5, - 0x24B6, - 0x24B7, - 0x24B8, - 0x24B9, - 0x24BA, - 0x24BB, - 0x24BC, - 0x24BD, - 0x24BE, - 0x24BF, - 0x24C0, - 0x24C1, - 0x24C2, - 0x24C3, - 0x24C4, - 0x24C5, - 0x24C6, - 0x24C7, - 0x24C8, - 0x24C9, - 0x24CA, - 0x24CB, - 0x24CC, - 0x24CD, - 0x24CE, - 0x24CF, - 0x24D0, - 0x24D1, - 0x24D2, - 0x24D3, - 0x24D4, - 0x24D5, - 0x24D6, - 0x24D7, - 0x24D8, - 0x24D9, - 0x24DA, - 0x24DB, - 0x24DC, - 0x24DD, - 0x24DE, - 0x24DF, - 0x24E0, - 0x24E1, - 0x24E2, - 0x24E3, - 0x24E4, - 0x24E5, - 0x24E6, - 0x24E7, - 0x24E8, - 0x24E9, - 0x24EA, - 0x24EB, - 0x2A0C, - 0x2A0D, - 0x2A74, - 0x2A75, - 0x2A76, - 0x2A77, - 0x2ADC, - 0x2ADD, - 0x2B74, - 0x2B76, - 0x2C00, - 0x2C01, - 0x2C02, - 0x2C03, - 0x2C04, - 0x2C05, - 0x2C06, - 0x2C07, - 0x2C08, - 0x2C09, - 0x2C0A, - 0x2C0B, - 0x2C0C, - 0x2C0D, - 0x2C0E, - 0x2C0F, - 0x2C10, - 0x2C11, - 0x2C12, - 0x2C13, - 0x2C14, - 0x2C15, - 0x2C16, - 0x2C17, - 0x2C18, - 0x2C19, - 0x2C1A, - 0x2C1B, - 0x2C1C, - 0x2C1D, - 0x2C1E, - 0x2C1F, - 0x2C20, - 0x2C21, - 0x2C22, - 0x2C23, - 0x2C24, - 0x2C25, - 0x2C26, - 0x2C27, - 0x2C28, - 0x2C29, - 0x2C2A, - 0x2C2B, - 0x2C2C, - 0x2C2D, - 0x2C2E, - 0x2C2F, - 0x2C30, - 0x2C60, - 0x2C61, - 0x2C62, - 0x2C63, - 0x2C64, - 0x2C65, - 0x2C67, - 0x2C68, - 0x2C69, - 0x2C6A, - 0x2C6B, - 0x2C6C, - 0x2C6D, - 0x2C6E, - 0x2C6F, - 0x2C70, - 0x2C71, - 0x2C72, - 0x2C73, - 0x2C75, - 0x2C76, - 0x2C7C, - 0x2C7D, - 0x2C7E, - 0x2C7F, - 0x2C80, - 0x2C81, - 0x2C82, - 0x2C83, - 0x2C84, - 0x2C85, - 0x2C86, - 0x2C87, - 0x2C88, - 0x2C89, - 0x2C8A, - 0x2C8B, - 0x2C8C, - 0x2C8D, - 0x2C8E, - 0x2C8F, - 0x2C90, - 0x2C91, - 0x2C92, - 0x2C93, - 0x2C94, - 0x2C95, - 0x2C96, - 0x2C97, - 0x2C98, - 0x2C99, - 0x2C9A, - 0x2C9B, - 0x2C9C, - 0x2C9D, - 0x2C9E, - 0x2C9F, - 0x2CA0, - 0x2CA1, - 0x2CA2, - 0x2CA3, - 0x2CA4, - 0x2CA5, - 0x2CA6, - 0x2CA7, - 0x2CA8, - 0x2CA9, - 0x2CAA, - 0x2CAB, - 0x2CAC, - 0x2CAD, - 0x2CAE, - 0x2CAF, - 0x2CB0, - 0x2CB1, - 0x2CB2, - 0x2CB3, - 0x2CB4, - 0x2CB5, - 0x2CB6, - 0x2CB7, - 0x2CB8, - 0x2CB9, - 0x2CBA, - 0x2CBB, - 0x2CBC, - 0x2CBD, - 0x2CBE, - 0x2CBF, - 0x2CC0, - 0x2CC1, - 0x2CC2, - 0x2CC3, - 0x2CC4, - 0x2CC5, - 0x2CC6, - 0x2CC7, - 0x2CC8, - 0x2CC9, - 0x2CCA, - 0x2CCB, - 0x2CCC, - 0x2CCD, - 0x2CCE, - 0x2CCF, - 0x2CD0, - 0x2CD1, - 0x2CD2, - 0x2CD3, - 0x2CD4, - 0x2CD5, - 0x2CD6, - 0x2CD7, - 0x2CD8, - 0x2CD9, - 0x2CDA, - 0x2CDB, - 0x2CDC, - 0x2CDD, - 0x2CDE, - 0x2CDF, - 0x2CE0, - 0x2CE1, - 0x2CE2, - 0x2CE3, - 0x2CEB, - 0x2CEC, - 0x2CED, - 0x2CEE, - 0x2CF2, - 0x2CF3, - 0x2CF4, - 0x2CF9, - 0x2D26, - 0x2D27, - 0x2D28, - 0x2D2D, - 0x2D2E, - 0x2D30, - 0x2D68, - 0x2D6F, - 0x2D70, - 0x2D71, - 0x2D7F, - 0x2D97, - 0x2DA0, - 0x2DA7, - 0x2DA8, - 0x2DAF, - 0x2DB0, - 0x2DB7, - 0x2DB8, - 0x2DBF, - 0x2DC0, - 0x2DC7, - 0x2DC8, - 0x2DCF, - 0x2DD0, - 0x2DD7, - 0x2DD8, - 0x2DDF, - 0x2DE0, - 0x2E5E, - 0x2E80, - 0x2E9A, - 0x2E9B, - 0x2E9F, - 0x2EA0, - 0x2EF3, - 0x2EF4, - 0x2F00, - 0x2F01, - 0x2F02, - 0x2F03, - 0x2F04, - 0x2F05, - 0x2F06, - 0x2F07, - 0x2F08, - 0x2F09, - 0x2F0A, - 0x2F0B, - 0x2F0C, - 0x2F0D, - 0x2F0E, - 0x2F0F, - 0x2F10, - 0x2F11, - 0x2F12, - 0x2F13, - 0x2F14, - 0x2F15, - 0x2F16, - 0x2F17, - 0x2F18, - 0x2F19, - 0x2F1A, - 0x2F1B, - 0x2F1C, - 0x2F1D, - 0x2F1E, - 0x2F1F, - 0x2F20, - 0x2F21, - 0x2F22, - 0x2F23, - 0x2F24, - 0x2F25, - 0x2F26, - 0x2F27, - 0x2F28, - 0x2F29, - 0x2F2A, - 0x2F2B, - 0x2F2C, - 0x2F2D, - 0x2F2E, - 0x2F2F, - 0x2F30, - 0x2F31, - 0x2F32, - 0x2F33, - 0x2F34, - 0x2F35, - 0x2F36, - 0x2F37, - 0x2F38, - 0x2F39, - 0x2F3A, - 0x2F3B, - 0x2F3C, - 0x2F3D, - 0x2F3E, - 0x2F3F, - 0x2F40, - 0x2F41, - 0x2F42, - 0x2F43, - 0x2F44, - 0x2F45, - 0x2F46, - 0x2F47, - 0x2F48, - 0x2F49, - 0x2F4A, - 0x2F4B, - 0x2F4C, - 0x2F4D, - 0x2F4E, - 0x2F4F, - 0x2F50, - 0x2F51, - 0x2F52, - 0x2F53, - 0x2F54, - 0x2F55, - 0x2F56, - 0x2F57, - 0x2F58, - 0x2F59, - 0x2F5A, - 0x2F5B, - 0x2F5C, - 0x2F5D, - 0x2F5E, - 0x2F5F, - 0x2F60, - 0x2F61, - 0x2F62, - 0x2F63, - 0x2F64, - 0x2F65, - 0x2F66, - 0x2F67, - 0x2F68, - 0x2F69, - 0x2F6A, - 0x2F6B, - 0x2F6C, - 0x2F6D, - 0x2F6E, - 0x2F6F, - 0x2F70, - 0x2F71, - 0x2F72, - 0x2F73, - 0x2F74, - 0x2F75, - 0x2F76, - 0x2F77, - 0x2F78, - 0x2F79, - 0x2F7A, - 0x2F7B, - 0x2F7C, - 0x2F7D, - 0x2F7E, - 0x2F7F, - 0x2F80, - 0x2F81, - 0x2F82, - 0x2F83, - 0x2F84, - 0x2F85, - 0x2F86, - 0x2F87, - 0x2F88, - 0x2F89, - 0x2F8A, - 0x2F8B, - 0x2F8C, - 0x2F8D, - 0x2F8E, - 0x2F8F, - 0x2F90, - 0x2F91, - 0x2F92, - 0x2F93, - 0x2F94, - 0x2F95, - 0x2F96, - 0x2F97, - 0x2F98, - 0x2F99, - 0x2F9A, - 0x2F9B, - 0x2F9C, - 0x2F9D, - 0x2F9E, - 0x2F9F, - 0x2FA0, - 0x2FA1, - 0x2FA2, - 0x2FA3, - 0x2FA4, - 0x2FA5, - 0x2FA6, - 0x2FA7, - 0x2FA8, - 0x2FA9, - 0x2FAA, - 0x2FAB, - 0x2FAC, - 0x2FAD, - 0x2FAE, - 0x2FAF, - 0x2FB0, - 0x2FB1, - 0x2FB2, - 0x2FB3, - 0x2FB4, - 0x2FB5, - 0x2FB6, - 0x2FB7, - 0x2FB8, - 0x2FB9, - 0x2FBA, - 0x2FBB, - 0x2FBC, - 0x2FBD, - 0x2FBE, - 0x2FBF, - 0x2FC0, - 0x2FC1, - 0x2FC2, - 0x2FC3, - 0x2FC4, - 0x2FC5, - 0x2FC6, - 0x2FC7, - 0x2FC8, - 0x2FC9, - 0x2FCA, - 0x2FCB, - 0x2FCC, - 0x2FCD, - 0x2FCE, - 0x2FCF, - 0x2FD0, - 0x2FD1, - 0x2FD2, - 0x2FD3, - 0x2FD4, - 0x2FD5, - 0x2FD6, - 0x3000, - 0x3001, - 0x3002, - 0x3003, - 0x3036, - 0x3037, - 0x3038, - 0x3039, - 0x303A, - 0x303B, - 0x3040, - 0x3041, - 0x3097, - 0x3099, - 0x309B, - 0x309C, - 0x309D, - 0x309F, - 0x30A0, - 0x30FF, - 0x3100, - 0x3105, - 0x3130, - 0x3131, - 0x3132, - 0x3133, - 0x3134, - 0x3135, - 0x3136, - 0x3137, - 0x3138, - 0x3139, - 0x313A, - 0x313B, - 0x313C, - 0x313D, - 0x313E, - 0x313F, - 0x3140, - 0x3141, - 0x3142, - 0x3143, - 0x3144, - 0x3145, - 0x3146, - 0x3147, - 0x3148, - 0x3149, - 0x314A, - 0x314B, - 0x314C, - 0x314D, - 0x314E, - 0x314F, - 0x3150, - 0x3151, - 0x3152, - 0x3153, - 0x3154, - 0x3155, - 0x3156, - 0x3157, - 0x3158, - 0x3159, - 0x315A, - 0x315B, - 0x315C, - 0x315D, - 0x315E, - 0x315F, - 0x3160, - 0x3161, - 0x3162, - 0x3163, - 0x3164, - 0x3165, - 0x3166, - 0x3167, - 0x3168, - 0x3169, - 0x316A, - 0x316B, - 0x316C, - 0x316D, - 0x316E, - 0x316F, - 0x3170, - 0x3171, - 0x3172, - 0x3173, - 0x3174, - 0x3175, - 0x3176, - 0x3177, - 0x3178, - 0x3179, - 0x317A, - 0x317B, - 0x317C, - 0x317D, - 0x317E, - 0x317F, - 0x3180, - 0x3181, - 0x3182, - 0x3183, - 0x3184, - 0x3185, - 0x3186, - 0x3187, - 0x3188, - 0x3189, - 0x318A, - 0x318B, - 0x318C, - 0x318D, - 0x318E, - 0x318F, - 0x3190, - 0x3192, - 0x3193, - 0x3194, - 0x3195, - 0x3196, - 0x3197, - 0x3198, - 0x3199, - 0x319A, - 0x319B, - 0x319C, - 0x319D, - 0x319E, - 0x319F, - 0x31A0, - 0x31E6, - 0x31F0, - 0x3200, - 0x3201, - 0x3202, - 0x3203, - 0x3204, - 0x3205, - 0x3206, - 0x3207, - 0x3208, - 0x3209, - 0x320A, - 0x320B, - 0x320C, - 0x320D, - 0x320E, - 0x320F, - 0x3210, - 0x3211, - 0x3212, - 0x3213, - 0x3214, - 0x3215, - 0x3216, - 0x3217, - 0x3218, - 0x3219, - 0x321A, - 0x321B, - 0x321C, - 0x321D, - 0x321E, - 0x321F, - 0x3220, - 0x3221, - 0x3222, - 0x3223, - 0x3224, - 0x3225, - 0x3226, - 0x3227, - 0x3228, - 0x3229, - 0x322A, - 0x322B, - 0x322C, - 0x322D, - 0x322E, - 0x322F, - 0x3230, - 0x3231, - 0x3232, - 0x3233, - 0x3234, - 0x3235, - 0x3236, - 0x3237, - 0x3238, - 0x3239, - 0x323A, - 0x323B, - 0x323C, - 0x323D, - 0x323E, - 0x323F, - 0x3240, - 0x3241, - 0x3242, - 0x3243, - 0x3244, - 0x3245, - 0x3246, - 0x3247, - 0x3248, - 0x3250, - 0x3251, - 0x3252, - 0x3253, - 0x3254, - 0x3255, - 0x3256, - 0x3257, - 0x3258, - 0x3259, - 0x325A, - 0x325B, - 0x325C, - 0x325D, - 0x325E, - 0x325F, - 0x3260, - 0x3261, - 0x3262, - 0x3263, - 0x3264, - 0x3265, - 0x3266, - 0x3267, - 0x3268, - 0x3269, - 0x326A, - 0x326B, - 0x326C, - 0x326D, - 0x326E, - 0x326F, - 0x3270, - 0x3271, - 0x3272, - 0x3273, - 0x3274, - 0x3275, - 0x3276, - 0x3277, - 0x3278, - 0x3279, - 0x327A, - 0x327B, - 0x327C, - 0x327D, - 0x327E, - 0x327F, - 0x3280, - 0x3281, - 0x3282, - 0x3283, - 0x3284, - 0x3285, - 0x3286, - 0x3287, - 0x3288, - 0x3289, - 0x328A, - 0x328B, - 0x328C, - 0x328D, - 0x328E, - 0x328F, - 0x3290, - 0x3291, - 0x3292, - 0x3293, - 0x3294, - 0x3295, - 0x3296, - 0x3297, - 0x3298, - 0x3299, - 0x329A, - 0x329B, - 0x329C, - 0x329D, - 0x329E, - 0x329F, - 0x32A0, - 0x32A1, - 0x32A2, - 0x32A3, - 0x32A4, - 0x32A5, - 0x32A6, - 0x32A7, - 0x32A8, - 0x32A9, - 0x32AA, - 0x32AB, - 0x32AC, - 0x32AD, - 0x32AE, - 0x32AF, - 0x32B0, - 0x32B1, - 0x32B2, - 0x32B3, - 0x32B4, - 0x32B5, - 0x32B6, - 0x32B7, - 0x32B8, - 0x32B9, - 0x32BA, - 0x32BB, - 0x32BC, - 0x32BD, - 0x32BE, - 0x32BF, - 0x32C0, - 0x32C1, - 0x32C2, - 0x32C3, - 0x32C4, - 0x32C5, - 0x32C6, - 0x32C7, - 0x32C8, - 0x32C9, - 0x32CA, - 0x32CB, - 0x32CC, - 0x32CD, - 0x32CE, - 0x32CF, - 0x32D0, - 0x32D1, - 0x32D2, - 0x32D3, - 0x32D4, - 0x32D5, - 0x32D6, - 0x32D7, - 0x32D8, - 0x32D9, - 0x32DA, - 0x32DB, - 0x32DC, - 0x32DD, - 0x32DE, - 0x32DF, - 0x32E0, - 0x32E1, - 0x32E2, - 0x32E3, - 0x32E4, - 0x32E5, - 0x32E6, - 0x32E7, - 0x32E8, - 0x32E9, - 0x32EA, - 0x32EB, - 0x32EC, - 0x32ED, - 0x32EE, - 0x32EF, - 0x32F0, - 0x32F1, - 0x32F2, - 0x32F3, - 0x32F4, - 0x32F5, - 0x32F6, - 0x32F7, - 0x32F8, - 0x32F9, - 0x32FA, - 0x32FB, - 0x32FC, - 0x32FD, - 0x32FE, - 0x32FF, - 0x3300, - 0x3301, - 0x3302, - 0x3303, - 0x3304, - 0x3305, - 0x3306, - 0x3307, - 0x3308, - 0x3309, - 0x330A, - 0x330B, - 0x330C, - 0x330D, - 0x330E, - 0x330F, - 0x3310, - 0x3311, - 0x3312, - 0x3313, - 0x3314, - 0x3315, - 0x3316, - 0x3317, - 0x3318, - 0x3319, - 0x331A, - 0x331B, - 0x331C, - 0x331D, - 0x331E, - 0x331F, - 0x3320, - 0x3321, - 0x3322, - 0x3323, - 0x3324, - 0x3325, - 0x3326, - 0x3327, - 0x3328, - 0x3329, - 0x332A, - 0x332B, - 0x332C, - 0x332D, - 0x332E, - 0x332F, - 0x3330, - 0x3331, - 0x3332, - 0x3333, - 0x3334, - 0x3335, - 0x3336, - 0x3337, - 0x3338, - 0x3339, - 0x333A, - 0x333B, - 0x333C, - 0x333D, - 0x333E, - 0x333F, - 0x3340, - 0x3341, - 0x3342, - 0x3343, - 0x3344, - 0x3345, - 0x3346, - 0x3347, - 0x3348, - 0x3349, - 0x334A, - 0x334B, - 0x334C, - 0x334D, - 0x334E, - 0x334F, - 0x3350, - 0x3351, - 0x3352, - 0x3353, - 0x3354, - 0x3355, - 0x3356, - 0x3357, - 0x3358, - 0x3359, - 0x335A, - 0x335B, - 0x335C, - 0x335D, - 0x335E, - 0x335F, - 0x3360, - 0x3361, - 0x3362, - 0x3363, - 0x3364, - 0x3365, - 0x3366, - 0x3367, - 0x3368, - 0x3369, - 0x336A, - 0x336B, - 0x336C, - 0x336D, - 0x336E, - 0x336F, - 0x3370, - 0x3371, - 0x3372, - 0x3373, - 0x3374, - 0x3375, - 0x3376, - 0x3377, - 0x3378, - 0x3379, - 0x337A, - 0x337B, - 0x337C, - 0x337D, - 0x337E, - 0x337F, - 0x3380, - 0x3381, - 0x3382, - 0x3383, - 0x3384, - 0x3385, - 0x3386, - 0x3387, - 0x3388, - 0x3389, - 0x338A, - 0x338B, - 0x338C, - 0x338D, - 0x338E, - 0x338F, - 0x3390, - 0x3391, - 0x3392, - 0x3393, - 0x3394, - 0x3395, - 0x3396, - 0x3397, - 0x3398, - 0x3399, - 0x339A, - 0x339B, - 0x339C, - 0x339D, - 0x339E, - 0x339F, - 0x33A0, - 0x33A1, - 0x33A2, - 0x33A3, - 0x33A4, - 0x33A5, - 0x33A6, - 0x33A7, - 0x33A8, - 0x33A9, - 0x33AA, - 0x33AB, - 0x33AC, - 0x33AD, - 0x33AE, - 0x33AF, - 0x33B0, - 0x33B1, - 0x33B2, - 0x33B3, - 0x33B4, - 0x33B5, - 0x33B6, - 0x33B7, - 0x33B8, - 0x33B9, - 0x33BA, - 0x33BB, - 0x33BC, - 0x33BD, - 0x33BE, - 0x33BF, - 0x33C0, - 0x33C1, - 0x33C2, - 0x33C3, - 0x33C4, - 0x33C5, - 0x33C6, - 0x33C7, - 0x33C8, - 0x33C9, - 0x33CA, - 0x33CB, - 0x33CC, - 0x33CD, - 0x33CE, - 0x33CF, - 0x33D0, - 0x33D1, - 0x33D2, - 0x33D3, - 0x33D4, - 0x33D5, - 0x33D6, - 0x33D7, - 0x33D8, - 0x33D9, - 0x33DA, - 0x33DB, - 0x33DC, - 0x33DD, - 0x33DE, - 0x33DF, - 0x33E0, - 0x33E1, - 0x33E2, - 0x33E3, - 0x33E4, - 0x33E5, - 0x33E6, - 0x33E7, - 0x33E8, - 0x33E9, - 0x33EA, - 0x33EB, - 0x33EC, - 0x33ED, - 0x33EE, - 0x33EF, - 0x33F0, - 0x33F1, - 0x33F2, - 0x33F3, - 0x33F4, - 0x33F5, - 0x33F6, - 0x33F7, - 0x33F8, - 0x33F9, - 0x33FA, - 0x33FB, - 0x33FC, - 0x33FD, - 0x33FE, - 0x33FF, - 0x3400, - 0xA48D, - 0xA490, - 0xA4C7, - 0xA4D0, - 0xA62C, - 0xA640, - 0xA641, - 0xA642, - 0xA643, - 0xA644, - 0xA645, - 0xA646, - 0xA647, - 0xA648, - 0xA649, - 0xA64A, - 0xA64B, - 0xA64C, - 0xA64D, - 0xA64E, - 0xA64F, - 0xA650, - 0xA651, - 0xA652, - 0xA653, - 0xA654, - 0xA655, - 0xA656, - 0xA657, - 0xA658, - 0xA659, - 0xA65A, - 0xA65B, - 0xA65C, - 0xA65D, - 0xA65E, - 0xA65F, - 0xA660, - 0xA661, - 0xA662, - 0xA663, - 0xA664, - 0xA665, - 0xA666, - 0xA667, - 0xA668, - 0xA669, - 0xA66A, - 0xA66B, - 0xA66C, - 0xA66D, - 0xA680, - 0xA681, - 0xA682, - 0xA683, - 0xA684, - 0xA685, - 0xA686, - 0xA687, - 0xA688, - 0xA689, - 0xA68A, - 0xA68B, - 0xA68C, - 0xA68D, - 0xA68E, - 0xA68F, - 0xA690, - 0xA691, - 0xA692, - 0xA693, - 0xA694, - 0xA695, - 0xA696, - 0xA697, - 0xA698, - 0xA699, - 0xA69A, - 0xA69B, - 0xA69C, - 0xA69D, - 0xA69E, - 0xA6F8, - 0xA700, - 0xA722, - 0xA723, - 0xA724, - 0xA725, - 0xA726, - 0xA727, - 0xA728, - 0xA729, - 0xA72A, - 0xA72B, - 0xA72C, - 0xA72D, - 0xA72E, - 0xA72F, - 0xA732, - 0xA733, - 0xA734, - 0xA735, - 0xA736, - 0xA737, - 0xA738, - 0xA739, - 0xA73A, - 0xA73B, - 0xA73C, - 0xA73D, - 0xA73E, - 0xA73F, - 0xA740, - 0xA741, - 0xA742, - 0xA743, - 0xA744, - 0xA745, - 0xA746, - 0xA747, - 0xA748, - 0xA749, - 0xA74A, - 0xA74B, - 0xA74C, - 0xA74D, - 0xA74E, - 0xA74F, - 0xA750, - 0xA751, - 0xA752, - 0xA753, - 0xA754, - 0xA755, - 0xA756, - 0xA757, - 0xA758, - 0xA759, - 0xA75A, - 0xA75B, - 0xA75C, - 0xA75D, - 0xA75E, - 0xA75F, - 0xA760, - 0xA761, - 0xA762, - 0xA763, - 0xA764, - 0xA765, - 0xA766, - 0xA767, - 0xA768, - 0xA769, - 0xA76A, - 0xA76B, - 0xA76C, - 0xA76D, - 0xA76E, - 0xA76F, - 0xA770, - 0xA771, - 0xA779, - 0xA77A, - 0xA77B, - 0xA77C, - 0xA77D, - 0xA77E, - 0xA77F, - 0xA780, - 0xA781, - 0xA782, - 0xA783, - 0xA784, - 0xA785, - 0xA786, - 0xA787, - 0xA78B, - 0xA78C, - 0xA78D, - 0xA78E, - 0xA790, - 0xA791, - 0xA792, - 0xA793, - 0xA796, - 0xA797, - 0xA798, - 0xA799, - 0xA79A, - 0xA79B, - 0xA79C, - 0xA79D, - 0xA79E, - 0xA79F, - 0xA7A0, - 0xA7A1, - 0xA7A2, - 0xA7A3, - 0xA7A4, - 0xA7A5, - 0xA7A6, - 0xA7A7, - 0xA7A8, - 0xA7A9, - 0xA7AA, - 0xA7AB, - 0xA7AC, - 0xA7AD, - 0xA7AE, - 0xA7AF, - 0xA7B0, - 0xA7B1, - 0xA7B2, - 0xA7B3, - 0xA7B4, - 0xA7B5, - 0xA7B6, - 0xA7B7, - 0xA7B8, - 0xA7B9, - 0xA7BA, - 0xA7BB, - 0xA7BC, - 0xA7BD, - 0xA7BE, - 0xA7BF, - 0xA7C0, - 0xA7C1, - 0xA7C2, - 0xA7C3, - 0xA7C4, - 0xA7C5, - 0xA7C6, - 0xA7C7, - 0xA7C8, - 0xA7C9, - 0xA7CA, - 0xA7CB, - 0xA7CC, - 0xA7CD, - 0xA7CE, - 0xA7CF, - 0xA7D0, - 0xA7D1, - 0xA7D2, - 0xA7D3, - 0xA7D4, - 0xA7D5, - 0xA7D6, - 0xA7D7, - 0xA7D8, - 0xA7D9, - 0xA7DA, - 0xA7DB, - 0xA7DC, - 0xA7DD, - 0xA7F1, - 0xA7F2, - 0xA7F3, - 0xA7F4, - 0xA7F5, - 0xA7F6, - 0xA7F8, - 0xA7F9, - 0xA7FA, - 0xA82D, - 0xA830, - 0xA83A, - 0xA840, - 0xA878, - 0xA880, - 0xA8C6, - 0xA8CE, - 0xA8DA, - 0xA8E0, - 0xA954, - 0xA95F, - 0xA97D, - 0xA980, - 0xA9CE, - 0xA9CF, - 0xA9DA, - 0xA9DE, - 0xA9FF, - 0xAA00, - 0xAA37, - 0xAA40, - 0xAA4E, - 0xAA50, - 0xAA5A, - 0xAA5C, - 0xAAC3, - 0xAADB, - 0xAAF7, - 0xAB01, - 0xAB07, - 0xAB09, - 0xAB0F, - 0xAB11, - 0xAB17, - 0xAB20, - 0xAB27, - 0xAB28, - 0xAB2F, - 0xAB30, - 0xAB5C, - 0xAB5D, - 0xAB5E, - 0xAB5F, - 0xAB60, - 0xAB69, - 0xAB6A, - 0xAB6C, - 0xAB70, - 0xAB71, - 0xAB72, - 0xAB73, - 0xAB74, - 0xAB75, - 0xAB76, - 0xAB77, - 0xAB78, - 0xAB79, - 0xAB7A, - 0xAB7B, - 0xAB7C, - 0xAB7D, - 0xAB7E, - 0xAB7F, - 0xAB80, - 0xAB81, - 0xAB82, - 0xAB83, - 0xAB84, - 0xAB85, - 0xAB86, - 0xAB87, - 0xAB88, - 0xAB89, - 0xAB8A, - 0xAB8B, - 0xAB8C, - 0xAB8D, - 0xAB8E, - 0xAB8F, - 0xAB90, - 0xAB91, - 0xAB92, - 0xAB93, - 0xAB94, - 0xAB95, - 0xAB96, - 0xAB97, - 0xAB98, - 0xAB99, - 0xAB9A, - 0xAB9B, - 0xAB9C, - 0xAB9D, - 0xAB9E, - 0xAB9F, - 0xABA0, - 0xABA1, - 0xABA2, - 0xABA3, - 0xABA4, - 0xABA5, - 0xABA6, - 0xABA7, - 0xABA8, - 0xABA9, - 0xABAA, - 0xABAB, - 0xABAC, - 0xABAD, - 0xABAE, - 0xABAF, - 0xABB0, - 0xABB1, - 0xABB2, - 0xABB3, - 0xABB4, - 0xABB5, - 0xABB6, - 0xABB7, - 0xABB8, - 0xABB9, - 0xABBA, - 0xABBB, - 0xABBC, - 0xABBD, - 0xABBE, - 0xABBF, - 0xABC0, - 0xABEE, - 0xABF0, - 0xABFA, - 0xAC00, - 0xD7A4, - 0xD7B0, - 0xD7C7, - 0xD7CB, - 0xD7FC, - 0xF900, - 0xF901, - 0xF902, - 0xF903, - 0xF904, - 0xF905, - 0xF906, - 0xF907, - 0xF909, - 0xF90A, - 0xF90B, - 0xF90C, - 0xF90D, - 0xF90E, - 0xF90F, - 0xF910, - 0xF911, - 0xF912, - 0xF913, - 0xF914, - 0xF915, - 0xF916, - 0xF917, - 0xF918, - 0xF919, - 0xF91A, - 0xF91B, - 0xF91C, - 0xF91D, - 0xF91E, - 0xF91F, - 0xF920, - 0xF921, - 0xF922, - 0xF923, - 0xF924, - 0xF925, - 0xF926, - 0xF927, - 0xF928, - 0xF929, - 0xF92A, - 0xF92B, - 0xF92C, - 0xF92D, - 0xF92E, - 0xF92F, - 0xF930, - 0xF931, - 0xF932, - 0xF933, - 0xF934, - 0xF935, - 0xF936, - 0xF937, - 0xF938, - 0xF939, - 0xF93A, - 0xF93B, - 0xF93C, - 0xF93D, - 0xF93E, - 0xF93F, - 0xF940, - 0xF941, - 0xF942, - 0xF943, - 0xF944, - 0xF945, - 0xF946, - 0xF947, - 0xF948, - 0xF949, - 0xF94A, - 0xF94B, - 0xF94C, - 0xF94D, - 0xF94E, - 0xF94F, - 0xF950, - 0xF951, - 0xF952, - 0xF953, - 0xF954, - 0xF955, - 0xF956, - 0xF957, - 0xF958, - 0xF959, - 0xF95A, - 0xF95B, - 0xF95C, - 0xF95D, - 0xF95E, - 0xF95F, - 0xF960, - 0xF961, - 0xF962, - 0xF963, - 0xF964, - 0xF965, - 0xF966, - 0xF967, - 0xF968, - 0xF969, - 0xF96A, - 0xF96B, - 0xF96C, - 0xF96D, - 0xF96E, - 0xF96F, - 0xF970, - 0xF971, - 0xF972, - 0xF973, - 0xF974, - 0xF975, - 0xF976, - 0xF977, - 0xF978, - 0xF979, - 0xF97A, - 0xF97B, - 0xF97C, - 0xF97D, - 0xF97E, - 0xF97F, - 0xF980, - 0xF981, - 0xF982, - 0xF983, - 0xF984, - 0xF985, - 0xF986, - 0xF987, - 0xF988, - 0xF989, - 0xF98A, - 0xF98B, - 0xF98C, - 0xF98D, - 0xF98E, - 0xF98F, - 0xF990, - 0xF991, - 0xF992, - 0xF993, - 0xF994, - 0xF995, - 0xF996, - 0xF997, - 0xF998, - 0xF999, - 0xF99A, - 0xF99B, - 0xF99C, - 0xF99D, - 0xF99E, - 0xF99F, - 0xF9A0, - 0xF9A1, - 0xF9A2, - 0xF9A3, - 0xF9A4, - 0xF9A5, - 0xF9A6, - 0xF9A7, - 0xF9A8, - 0xF9A9, - 0xF9AA, - 0xF9AB, - 0xF9AC, - 0xF9AD, - 0xF9AE, - 0xF9AF, - 0xF9B0, - 0xF9B1, - 0xF9B2, - 0xF9B3, - 0xF9B4, - 0xF9B5, - 0xF9B6, - 0xF9B7, - 0xF9B8, - 0xF9B9, - 0xF9BA, - 0xF9BB, - 0xF9BC, - 0xF9BD, - 0xF9BE, - 0xF9BF, - 0xF9C0, - 0xF9C1, - 0xF9C2, - 0xF9C3, - 0xF9C4, - 0xF9C5, - 0xF9C6, - 0xF9C7, - 0xF9C8, - 0xF9C9, - 0xF9CA, - 0xF9CB, - 0xF9CC, - 0xF9CD, - 0xF9CE, - 0xF9CF, - 0xF9D0, - 0xF9D1, - 0xF9D2, - 0xF9D3, - 0xF9D4, - 0xF9D5, - 0xF9D6, - 0xF9D7, - 0xF9D8, - 0xF9D9, - 0xF9DA, - 0xF9DB, - 0xF9DC, - 0xF9DD, - 0xF9DE, - 0xF9DF, - 0xF9E0, - 0xF9E1, - 0xF9E2, - 0xF9E3, - 0xF9E4, - 0xF9E5, - 0xF9E6, - 0xF9E7, - 0xF9E8, - 0xF9E9, - 0xF9EA, - 0xF9EB, - 0xF9EC, - 0xF9ED, - 0xF9EE, - 0xF9EF, - 0xF9F0, - 0xF9F1, - 0xF9F2, - 0xF9F3, - 0xF9F4, - 0xF9F5, - 0xF9F6, - 0xF9F7, - 0xF9F8, - 0xF9F9, - 0xF9FA, - 0xF9FB, - 0xF9FC, - 0xF9FD, - 0xF9FE, - 0xF9FF, - 0xFA00, - 0xFA01, - 0xFA02, - 0xFA03, - 0xFA04, - 0xFA05, - 0xFA06, - 0xFA07, - 0xFA08, - 0xFA09, - 0xFA0A, - 0xFA0B, - 0xFA0C, - 0xFA0D, - 0xFA0E, - 0xFA10, - 0xFA11, - 0xFA12, - 0xFA13, - 0xFA15, - 0xFA16, - 0xFA17, - 0xFA18, - 0xFA19, - 0xFA1A, - 0xFA1B, - 0xFA1C, - 0xFA1D, - 0xFA1E, - 0xFA1F, - 0xFA20, - 0xFA21, - 0xFA22, - 0xFA23, - 0xFA25, - 0xFA26, - 0xFA27, - 0xFA2A, - 0xFA2B, - 0xFA2C, - 0xFA2D, - 0xFA2E, - 0xFA2F, - 0xFA30, - 0xFA31, - 0xFA32, - 0xFA33, - 0xFA34, - 0xFA35, - 0xFA36, - 0xFA37, - 0xFA38, - 0xFA39, - 0xFA3A, - 0xFA3B, - 0xFA3C, - 0xFA3D, - 0xFA3E, - 0xFA3F, - 0xFA40, - 0xFA41, - 0xFA42, - 0xFA43, - 0xFA44, - 0xFA45, - 0xFA46, - 0xFA47, - 0xFA48, - 0xFA49, - 0xFA4A, - 0xFA4B, - 0xFA4C, - 0xFA4D, - 0xFA4E, - 0xFA4F, - 0xFA50, - 0xFA51, - 0xFA52, - 0xFA53, - 0xFA54, - 0xFA55, - 0xFA56, - 0xFA57, - 0xFA58, - 0xFA59, - 0xFA5A, - 0xFA5B, - 0xFA5C, - 0xFA5D, - 0xFA5F, - 0xFA60, - 0xFA61, - 0xFA62, - 0xFA63, - 0xFA64, - 0xFA65, - 0xFA66, - 0xFA67, - 0xFA68, - 0xFA69, - 0xFA6A, - 0xFA6B, - 0xFA6C, - 0xFA6D, - 0xFA6E, - 0xFA70, - 0xFA71, - 0xFA72, - 0xFA73, - 0xFA74, - 0xFA75, - 0xFA76, - 0xFA77, - 0xFA78, - 0xFA79, - 0xFA7A, - 0xFA7B, - 0xFA7C, - 0xFA7D, - 0xFA7E, - 0xFA7F, - 0xFA80, - 0xFA81, - 0xFA82, - 0xFA83, - 0xFA84, - 0xFA85, - 0xFA86, - 0xFA87, - 0xFA88, - 0xFA89, - 0xFA8A, - 0xFA8B, - 0xFA8C, - 0xFA8D, - 0xFA8E, - 0xFA8F, - 0xFA90, - 0xFA91, - 0xFA92, - 0xFA93, - 0xFA94, - 0xFA95, - 0xFA96, - 0xFA97, - 0xFA98, - 0xFA99, - 0xFA9A, - 0xFA9B, - 0xFA9C, - 0xFA9D, - 0xFA9E, - 0xFA9F, - 0xFAA0, - 0xFAA1, - 0xFAA2, - 0xFAA3, - 0xFAA4, - 0xFAA5, - 0xFAA6, - 0xFAA7, - 0xFAA8, - 0xFAA9, - 0xFAAA, - 0xFAAB, - 0xFAAC, - 0xFAAD, - 0xFAAE, - 0xFAAF, - 0xFAB0, - 0xFAB1, - 0xFAB2, - 0xFAB3, - 0xFAB4, - 0xFAB5, - 0xFAB6, - 0xFAB7, - 0xFAB8, - 0xFAB9, - 0xFABA, - 0xFABB, - 0xFABC, - 0xFABD, - 0xFABE, - 0xFABF, - 0xFAC0, - 0xFAC1, - 0xFAC2, - 0xFAC3, - 0xFAC4, - 0xFAC5, - 0xFAC6, - 0xFAC7, - 0xFAC8, - 0xFAC9, - 0xFACA, - 0xFACB, - 0xFACC, - 0xFACD, - 0xFACE, - 0xFACF, - 0xFAD0, - 0xFAD1, - 0xFAD2, - 0xFAD3, - 0xFAD4, - 0xFAD5, - 0xFAD6, - 0xFAD7, - 0xFAD8, - 0xFAD9, - 0xFADA, - 0xFB00, - 0xFB01, - 0xFB02, - 0xFB03, - 0xFB04, - 0xFB05, - 0xFB07, - 0xFB13, - 0xFB14, - 0xFB15, - 0xFB16, - 0xFB17, - 0xFB18, - 0xFB1D, - 0xFB1E, - 0xFB1F, - 0xFB20, - 0xFB21, - 0xFB22, - 0xFB23, - 0xFB24, - 0xFB25, - 0xFB26, - 0xFB27, - 0xFB28, - 0xFB29, - 0xFB2A, - 0xFB2B, - 0xFB2C, - 0xFB2D, - 0xFB2E, - 0xFB2F, - 0xFB30, - 0xFB31, - 0xFB32, - 0xFB33, - 0xFB34, - 0xFB35, - 0xFB36, - 0xFB37, - 0xFB38, - 0xFB39, - 0xFB3A, - 0xFB3B, - 0xFB3C, - 0xFB3D, - 0xFB3E, - 0xFB3F, - 0xFB40, - 0xFB41, - 0xFB42, - 0xFB43, - 0xFB44, - 0xFB45, - 0xFB46, - 0xFB47, - 0xFB48, - 0xFB49, - 0xFB4A, - 0xFB4B, - 0xFB4C, - 0xFB4D, - 0xFB4E, - 0xFB4F, - 0xFB50, - 0xFB52, - 0xFB56, - 0xFB5A, - 0xFB5E, - 0xFB62, - 0xFB66, - 0xFB6A, - 0xFB6E, - 0xFB72, - 0xFB76, - 0xFB7A, - 0xFB7E, - 0xFB82, - 0xFB84, - 0xFB86, - 0xFB88, - 0xFB8A, - 0xFB8C, - 0xFB8E, - 0xFB92, - 0xFB96, - 0xFB9A, - 0xFB9E, - 0xFBA0, - 0xFBA4, - 0xFBA6, - 0xFBAA, - 0xFBAE, - 0xFBB0, - 0xFBB2, - 0xFBD3, - 0xFBD7, - 0xFBD9, - 0xFBDB, - 0xFBDD, - 0xFBDE, - 0xFBE0, - 0xFBE2, - 0xFBE4, - 0xFBE8, - 0xFBEA, - 0xFBEC, - 0xFBEE, - 0xFBF0, - 0xFBF2, - 0xFBF4, - 0xFBF6, - 0xFBF9, - 0xFBFC, - 0xFC00, - 0xFC01, - 0xFC02, - 0xFC03, - 0xFC04, - 0xFC05, - 0xFC06, - 0xFC07, - 0xFC08, - 0xFC09, - 0xFC0A, - 0xFC0B, - 0xFC0C, - 0xFC0D, - 0xFC0E, - 0xFC0F, - 0xFC10, - 0xFC11, - 0xFC12, - 0xFC13, - 0xFC14, - 0xFC15, - 0xFC16, - 0xFC17, - 0xFC18, - 0xFC19, - 0xFC1A, - 0xFC1B, - 0xFC1C, - 0xFC1D, - 0xFC1E, - 0xFC1F, - 0xFC20, - 0xFC21, - 0xFC22, - 0xFC23, - 0xFC24, - 0xFC25, - 0xFC26, - 0xFC27, - 0xFC28, - 0xFC29, - 0xFC2A, - 0xFC2B, - 0xFC2C, - 0xFC2D, - 0xFC2E, - 0xFC2F, - 0xFC30, - 0xFC31, - 0xFC32, - 0xFC33, - 0xFC34, - 0xFC35, - 0xFC36, - 0xFC37, - 0xFC38, - 0xFC39, - 0xFC3A, - 0xFC3B, - 0xFC3C, - 0xFC3D, - 0xFC3E, - 0xFC3F, - 0xFC40, - 0xFC41, - 0xFC42, - 0xFC43, - 0xFC44, - 0xFC45, - 0xFC46, - 0xFC47, - 0xFC48, - 0xFC49, - 0xFC4A, - 0xFC4B, - 0xFC4C, - 0xFC4D, - 0xFC4E, - 0xFC4F, - 0xFC50, - 0xFC51, - 0xFC52, - 0xFC53, - 0xFC54, - 0xFC55, - 0xFC56, - 0xFC57, - 0xFC58, - 0xFC59, - 0xFC5A, - 0xFC5B, - 0xFC5C, - 0xFC5D, - 0xFC5E, - 0xFC5F, - 0xFC60, - 0xFC61, - 0xFC62, - 0xFC63, - 0xFC64, - 0xFC65, - 0xFC66, - 0xFC67, - 0xFC68, - 0xFC69, - 0xFC6A, - 0xFC6B, - 0xFC6C, - 0xFC6D, - 0xFC6E, - 0xFC6F, - 0xFC70, - 0xFC71, - 0xFC72, - 0xFC73, - 0xFC74, - 0xFC75, - 0xFC76, - 0xFC77, - 0xFC78, - 0xFC79, - 0xFC7A, - 0xFC7B, - 0xFC7C, - 0xFC7D, - 0xFC7E, - 0xFC7F, - 0xFC80, - 0xFC81, - 0xFC82, - 0xFC83, - 0xFC84, - 0xFC85, - 0xFC86, - 0xFC87, - 0xFC88, - 0xFC89, - 0xFC8A, - 0xFC8B, - 0xFC8C, - 0xFC8D, - 0xFC8E, - 0xFC8F, - 0xFC90, - 0xFC91, - 0xFC92, - 0xFC93, - 0xFC94, - 0xFC95, - 0xFC96, - 0xFC97, - 0xFC98, - 0xFC99, - 0xFC9A, - 0xFC9B, - 0xFC9C, - 0xFC9D, - 0xFC9E, - 0xFC9F, - 0xFCA0, - 0xFCA1, - 0xFCA2, - 0xFCA3, - 0xFCA4, - 0xFCA5, - 0xFCA6, - 0xFCA7, - 0xFCA8, - 0xFCA9, - 0xFCAA, - 0xFCAB, - 0xFCAC, - 0xFCAD, - 0xFCAE, - 0xFCAF, - 0xFCB0, - 0xFCB1, - 0xFCB2, - 0xFCB3, - 0xFCB4, - 0xFCB5, - 0xFCB6, - 0xFCB7, - 0xFCB8, - 0xFCB9, - 0xFCBA, - 0xFCBB, - 0xFCBC, - 0xFCBD, - 0xFCBE, - 0xFCBF, - 0xFCC0, - 0xFCC1, - 0xFCC2, - 0xFCC3, - 0xFCC4, - 0xFCC5, - 0xFCC6, - 0xFCC7, - 0xFCC8, - 0xFCC9, - 0xFCCA, - 0xFCCB, - 0xFCCC, - 0xFCCD, - 0xFCCE, - 0xFCCF, - 0xFCD0, - 0xFCD1, - 0xFCD2, - 0xFCD3, - 0xFCD4, - 0xFCD5, - 0xFCD6, - 0xFCD7, - 0xFCD8, - 0xFCD9, - 0xFCDA, - 0xFCDB, - 0xFCDC, - 0xFCDD, - 0xFCDE, - 0xFCDF, - 0xFCE0, - 0xFCE1, - 0xFCE2, - 0xFCE3, - 0xFCE4, - 0xFCE5, - 0xFCE6, - 0xFCE7, - 0xFCE8, - 0xFCE9, - 0xFCEA, - 0xFCEB, - 0xFCEC, - 0xFCED, - 0xFCEE, - 0xFCEF, - 0xFCF0, - 0xFCF1, - 0xFCF2, - 0xFCF3, - 0xFCF4, - 0xFCF5, - 0xFCF6, - 0xFCF7, - 0xFCF8, - 0xFCF9, - 0xFCFA, - 0xFCFB, - 0xFCFC, - 0xFCFD, - 0xFCFE, - 0xFCFF, - 0xFD00, - 0xFD01, - 0xFD02, - 0xFD03, - 0xFD04, - 0xFD05, - 0xFD06, - 0xFD07, - 0xFD08, - 0xFD09, - 0xFD0A, - 0xFD0B, - 0xFD0C, - 0xFD0D, - 0xFD0E, - 0xFD0F, - 0xFD10, - 0xFD11, - 0xFD12, - 0xFD13, - 0xFD14, - 0xFD15, - 0xFD16, - 0xFD17, - 0xFD18, - 0xFD19, - 0xFD1A, - 0xFD1B, - 0xFD1C, - 0xFD1D, - 0xFD1E, - 0xFD1F, - 0xFD20, - 0xFD21, - 0xFD22, - 0xFD23, - 0xFD24, - 0xFD25, - 0xFD26, - 0xFD27, - 0xFD28, - 0xFD29, - 0xFD2A, - 0xFD2B, - 0xFD2C, - 0xFD2D, - 0xFD2E, - 0xFD2F, - 0xFD30, - 0xFD31, - 0xFD32, - 0xFD33, - 0xFD34, - 0xFD35, - 0xFD36, - 0xFD37, - 0xFD38, - 0xFD39, - 0xFD3A, - 0xFD3B, - 0xFD3C, - 0xFD3E, - 0xFD50, - 0xFD51, - 0xFD53, - 0xFD54, - 0xFD55, - 0xFD56, - 0xFD57, - 0xFD58, - 0xFD5A, - 0xFD5B, - 0xFD5C, - 0xFD5D, - 0xFD5E, - 0xFD5F, - 0xFD61, - 0xFD62, - 0xFD64, - 0xFD66, - 0xFD67, - 0xFD69, - 0xFD6A, - 0xFD6C, - 0xFD6E, - 0xFD6F, - 0xFD71, - 0xFD73, - 0xFD74, - 0xFD75, - 0xFD76, - 0xFD78, - 0xFD79, - 0xFD7A, - 0xFD7B, - 0xFD7C, - 0xFD7E, - 0xFD7F, - 0xFD80, - 0xFD81, - 0xFD82, - 0xFD83, - 0xFD85, - 0xFD87, - 0xFD89, - 0xFD8A, - 0xFD8B, - 0xFD8C, - 0xFD8D, - 0xFD8E, - 0xFD8F, - 0xFD90, - 0xFD92, - 0xFD93, - 0xFD94, - 0xFD95, - 0xFD96, - 0xFD97, - 0xFD99, - 0xFD9A, - 0xFD9B, - 0xFD9C, - 0xFD9E, - 0xFD9F, - 0xFDA0, - 0xFDA1, - 0xFDA2, - 0xFDA3, - 0xFDA4, - 0xFDA5, - 0xFDA6, - 0xFDA7, - 0xFDA8, - 0xFDA9, - 0xFDAA, - 0xFDAB, - 0xFDAC, - 0xFDAD, - 0xFDAE, - 0xFDAF, - 0xFDB0, - 0xFDB1, - 0xFDB2, - 0xFDB3, - 0xFDB4, - 0xFDB5, - 0xFDB6, - 0xFDB7, - 0xFDB8, - 0xFDB9, - 0xFDBA, - 0xFDBB, - 0xFDBC, - 0xFDBD, - 0xFDBE, - 0xFDBF, - 0xFDC0, - 0xFDC1, - 0xFDC2, - 0xFDC3, - 0xFDC4, - 0xFDC5, - 0xFDC6, - 0xFDC7, - 0xFDC8, - 0xFDD0, - 0xFDF0, - 0xFDF1, - 0xFDF2, - 0xFDF3, - 0xFDF4, - 0xFDF5, - 0xFDF6, - 0xFDF7, - 0xFDF8, - 0xFDF9, - 0xFDFA, - 0xFDFB, - 0xFDFC, - 0xFDFD, - 0xFE00, - 0xFE10, - 0xFE11, - 0xFE12, - 0xFE13, - 0xFE14, - 0xFE15, - 0xFE16, - 0xFE17, - 0xFE18, - 0xFE19, - 0xFE20, - 0xFE30, - 0xFE31, - 0xFE32, - 0xFE33, - 0xFE35, - 0xFE36, - 0xFE37, - 0xFE38, - 0xFE39, - 0xFE3A, - 0xFE3B, - 0xFE3C, - 0xFE3D, - 0xFE3E, - 0xFE3F, - 0xFE40, - 0xFE41, - 0xFE42, - 0xFE43, - 0xFE44, - 0xFE45, - 0xFE47, - 0xFE48, - 0xFE49, - 0xFE4D, - 0xFE50, - 0xFE51, - 0xFE52, - 0xFE54, - 0xFE55, - 0xFE56, - 0xFE57, - 0xFE58, - 0xFE59, - 0xFE5A, - 0xFE5B, - 0xFE5C, - 0xFE5D, - 0xFE5E, - 0xFE5F, - 0xFE60, - 0xFE61, - 0xFE62, - 0xFE63, - 0xFE64, - 0xFE65, - 0xFE66, - 0xFE67, - 0xFE68, - 0xFE69, - 0xFE6A, - 0xFE6B, - 0xFE6C, - 0xFE70, - 0xFE71, - 0xFE72, - 0xFE73, - 0xFE74, - 0xFE75, - 0xFE76, - 0xFE77, - 0xFE78, - 0xFE79, - 0xFE7A, - 0xFE7B, - 0xFE7C, - 0xFE7D, - 0xFE7E, - 0xFE7F, - 0xFE80, - 0xFE81, - 0xFE83, - 0xFE85, - 0xFE87, - 0xFE89, - 0xFE8D, - 0xFE8F, - 0xFE93, - 0xFE95, - 0xFE99, - 0xFE9D, - 0xFEA1, - 0xFEA5, - 0xFEA9, - 0xFEAB, - 0xFEAD, - 0xFEAF, - 0xFEB1, - 0xFEB5, - 0xFEB9, - 0xFEBD, - 0xFEC1, - 0xFEC5, - 0xFEC9, - 0xFECD, - 0xFED1, - 0xFED5, - 0xFED9, - 0xFEDD, - 0xFEE1, - 0xFEE5, - 0xFEE9, - 0xFEED, - 0xFEEF, - 0xFEF1, - 0xFEF5, - 0xFEF7, - 0xFEF9, - 0xFEFB, - 0xFEFD, - 0xFEFF, - 0xFF00, - 0xFF01, - 0xFF02, - 0xFF03, - 0xFF04, - 0xFF05, - 0xFF06, - 0xFF07, - 0xFF08, - 0xFF09, - 0xFF0A, - 0xFF0B, - 0xFF0C, - 0xFF0D, - 0xFF0E, - 0xFF0F, - 0xFF10, - 0xFF11, - 0xFF12, - 0xFF13, - 0xFF14, - 0xFF15, - 0xFF16, - 0xFF17, - 0xFF18, - 0xFF19, - 0xFF1A, - 0xFF1B, - 0xFF1C, - 0xFF1D, - 0xFF1E, - 0xFF1F, - 0xFF20, - 0xFF21, - 0xFF22, - 0xFF23, - 0xFF24, - 0xFF25, - 0xFF26, - 0xFF27, - 0xFF28, - 0xFF29, - 0xFF2A, - 0xFF2B, - 0xFF2C, - 0xFF2D, - 0xFF2E, - 0xFF2F, - 0xFF30, - 0xFF31, - 0xFF32, - 0xFF33, - 0xFF34, - 0xFF35, - 0xFF36, - 0xFF37, - 0xFF38, - 0xFF39, - 0xFF3A, - 0xFF3B, - 0xFF3C, - 0xFF3D, - 0xFF3E, - 0xFF3F, - 0xFF40, - 0xFF41, - 0xFF42, - 0xFF43, - 0xFF44, - 0xFF45, - 0xFF46, - 0xFF47, - 0xFF48, - 0xFF49, - 0xFF4A, - 0xFF4B, - 0xFF4C, - 0xFF4D, - 0xFF4E, - 0xFF4F, - 0xFF50, - 0xFF51, - 0xFF52, - 0xFF53, - 0xFF54, - 0xFF55, - 0xFF56, - 0xFF57, - 0xFF58, - 0xFF59, - 0xFF5A, - 0xFF5B, - 0xFF5C, - 0xFF5D, - 0xFF5E, - 0xFF5F, - 0xFF60, - 0xFF61, - 0xFF62, - 0xFF63, - 0xFF64, - 0xFF65, - 0xFF66, - 0xFF67, - 0xFF68, - 0xFF69, - 0xFF6A, - 0xFF6B, - 0xFF6C, - 0xFF6D, - 0xFF6E, - 0xFF6F, - 0xFF70, - 0xFF71, - 0xFF72, - 0xFF73, - 0xFF74, - 0xFF75, - 0xFF76, - 0xFF77, - 0xFF78, - 0xFF79, - 0xFF7A, - 0xFF7B, - 0xFF7C, - 0xFF7D, - 0xFF7E, - 0xFF7F, - 0xFF80, - 0xFF81, - 0xFF82, - 0xFF83, - 0xFF84, - 0xFF85, - 0xFF86, - 0xFF87, - 0xFF88, - 0xFF89, - 0xFF8A, - 0xFF8B, - 0xFF8C, - 0xFF8D, - 0xFF8E, - 0xFF8F, - 0xFF90, - 0xFF91, - 0xFF92, - 0xFF93, - 0xFF94, - 0xFF95, - 0xFF96, - 0xFF97, - 0xFF98, - 0xFF99, - 0xFF9A, - 0xFF9B, - 0xFF9C, - 0xFF9D, - 0xFF9E, - 0xFF9F, - 0xFFA0, - 0xFFA1, - 0xFFA2, - 0xFFA3, - 0xFFA4, - 0xFFA5, - 0xFFA6, - 0xFFA7, - 0xFFA8, - 0xFFA9, - 0xFFAA, - 0xFFAB, - 0xFFAC, - 0xFFAD, - 0xFFAE, - 0xFFAF, - 0xFFB0, - 0xFFB1, - 0xFFB2, - 0xFFB3, - 0xFFB4, - 0xFFB5, - 0xFFB6, - 0xFFB7, - 0xFFB8, - 0xFFB9, - 0xFFBA, - 0xFFBB, - 0xFFBC, - 0xFFBD, - 0xFFBE, - 0xFFBF, - 0xFFC2, - 0xFFC3, - 0xFFC4, - 0xFFC5, - 0xFFC6, - 0xFFC7, - 0xFFC8, - 0xFFCA, - 0xFFCB, - 0xFFCC, - 0xFFCD, - 0xFFCE, - 0xFFCF, - 0xFFD0, - 0xFFD2, - 0xFFD3, - 0xFFD4, - 0xFFD5, - 0xFFD6, - 0xFFD7, - 0xFFD8, - 0xFFDA, - 0xFFDB, - 0xFFDC, - 0xFFDD, - 0xFFE0, - 0xFFE1, - 0xFFE2, - 0xFFE3, - 0xFFE4, - 0xFFE5, - 0xFFE6, - 0xFFE7, - 0xFFE8, - 0xFFE9, - 0xFFEA, - 0xFFEB, - 0xFFEC, - 0xFFED, - 0xFFEE, - 0xFFEF, - 0x10000, - 0x1000C, - 0x1000D, - 0x10027, - 0x10028, - 0x1003B, - 0x1003C, - 0x1003E, - 0x1003F, - 0x1004E, - 0x10050, - 0x1005E, - 0x10080, - 0x100FB, - 0x10100, - 0x10103, - 0x10107, - 0x10134, - 0x10137, - 0x1018F, - 0x10190, - 0x1019D, - 0x101A0, - 0x101A1, - 0x101D0, - 0x101FE, - 0x10280, - 0x1029D, - 0x102A0, - 0x102D1, - 0x102E0, - 0x102FC, - 0x10300, - 0x10324, - 0x1032D, - 0x1034B, - 0x10350, - 0x1037B, - 0x10380, - 0x1039E, - 0x1039F, - 0x103C4, - 0x103C8, - 0x103D6, - 0x10400, - 0x10401, - 0x10402, - 0x10403, - 0x10404, - 0x10405, - 0x10406, - 0x10407, - 0x10408, - 0x10409, - 0x1040A, - 0x1040B, - 0x1040C, - 0x1040D, - 0x1040E, - 0x1040F, - 0x10410, - 0x10411, - 0x10412, - 0x10413, - 0x10414, - 0x10415, - 0x10416, - 0x10417, - 0x10418, - 0x10419, - 0x1041A, - 0x1041B, - 0x1041C, - 0x1041D, - 0x1041E, - 0x1041F, - 0x10420, - 0x10421, - 0x10422, - 0x10423, - 0x10424, - 0x10425, - 0x10426, - 0x10427, - 0x10428, - 0x1049E, - 0x104A0, - 0x104AA, - 0x104B0, - 0x104B1, - 0x104B2, - 0x104B3, - 0x104B4, - 0x104B5, - 0x104B6, - 0x104B7, - 0x104B8, - 0x104B9, - 0x104BA, - 0x104BB, - 0x104BC, - 0x104BD, - 0x104BE, - 0x104BF, - 0x104C0, - 0x104C1, - 0x104C2, - 0x104C3, - 0x104C4, - 0x104C5, - 0x104C6, - 0x104C7, - 0x104C8, - 0x104C9, - 0x104CA, - 0x104CB, - 0x104CC, - 0x104CD, - 0x104CE, - 0x104CF, - 0x104D0, - 0x104D1, - 0x104D2, - 0x104D3, - 0x104D4, - 0x104D8, - 0x104FC, - 0x10500, - 0x10528, - 0x10530, - 0x10564, - 0x1056F, - 0x10570, - 0x10571, - 0x10572, - 0x10573, - 0x10574, - 0x10575, - 0x10576, - 0x10577, - 0x10578, - 0x10579, - 0x1057A, - 0x1057B, - 0x1057C, - 0x1057D, - 0x1057E, - 0x1057F, - 0x10580, - 0x10581, - 0x10582, - 0x10583, - 0x10584, - 0x10585, - 0x10586, - 0x10587, - 0x10588, - 0x10589, - 0x1058A, - 0x1058B, - 0x1058C, - 0x1058D, - 0x1058E, - 0x1058F, - 0x10590, - 0x10591, - 0x10592, - 0x10593, - 0x10594, - 0x10595, - 0x10596, - 0x10597, - 0x105A2, - 0x105A3, - 0x105B2, - 0x105B3, - 0x105BA, - 0x105BB, - 0x105BD, - 0x105C0, - 0x105F4, - 0x10600, - 0x10737, - 0x10740, - 0x10756, - 0x10760, - 0x10768, - 0x10780, - 0x10781, - 0x10782, - 0x10783, - 0x10784, - 0x10785, - 0x10786, - 0x10787, - 0x10788, - 0x10789, - 0x1078A, - 0x1078B, - 0x1078C, - 0x1078D, - 0x1078E, - 0x1078F, - 0x10790, - 0x10791, - 0x10792, - 0x10793, - 0x10794, - 0x10795, - 0x10796, - 0x10797, - 0x10798, - 0x10799, - 0x1079A, - 0x1079B, - 0x1079C, - 0x1079D, - 0x1079E, - 0x1079F, - 0x107A0, - 0x107A1, - 0x107A2, - 0x107A3, - 0x107A4, - 0x107A5, - 0x107A6, - 0x107A7, - 0x107A8, - 0x107A9, - 0x107AA, - 0x107AB, - 0x107AC, - 0x107AD, - 0x107AE, - 0x107AF, - 0x107B0, - 0x107B1, - 0x107B2, - 0x107B3, - 0x107B4, - 0x107B5, - 0x107B6, - 0x107B7, - 0x107B8, - 0x107B9, - 0x107BA, - 0x107BB, - 0x10800, - 0x10806, - 0x10808, - 0x10809, - 0x1080A, - 0x10836, - 0x10837, - 0x10839, - 0x1083C, - 0x1083D, - 0x1083F, - 0x10856, - 0x10857, - 0x1089F, - 0x108A7, - 0x108B0, - 0x108E0, - 0x108F3, - 0x108F4, - 0x108F6, - 0x108FB, - 0x1091C, - 0x1091F, - 0x1093A, - 0x1093F, - 0x1095A, - 0x10980, - 0x109B8, - 0x109BC, - 0x109D0, - 0x109D2, - 0x10A04, - 0x10A05, - 0x10A07, - 0x10A0C, - 0x10A14, - 0x10A15, - 0x10A18, - 0x10A19, - 0x10A36, - 0x10A38, - 0x10A3B, - 0x10A3F, - 0x10A49, - 0x10A50, - 0x10A59, - 0x10A60, - 0x10AA0, - 0x10AC0, - 0x10AE7, - 0x10AEB, - 0x10AF7, - 0x10B00, - 0x10B36, - 0x10B39, - 0x10B56, - 0x10B58, - 0x10B73, - 0x10B78, - 0x10B92, - 0x10B99, - 0x10B9D, - 0x10BA9, - 0x10BB0, - 0x10C00, - 0x10C49, - 0x10C80, - 0x10C81, - 0x10C82, - 0x10C83, - 0x10C84, - 0x10C85, - 0x10C86, - 0x10C87, - 0x10C88, - 0x10C89, - 0x10C8A, - 0x10C8B, - 0x10C8C, - 0x10C8D, - 0x10C8E, - 0x10C8F, - 0x10C90, - 0x10C91, - 0x10C92, - 0x10C93, - 0x10C94, - 0x10C95, - 0x10C96, - 0x10C97, - 0x10C98, - 0x10C99, - 0x10C9A, - 0x10C9B, - 0x10C9C, - 0x10C9D, - 0x10C9E, - 0x10C9F, - 0x10CA0, - 0x10CA1, - 0x10CA2, - 0x10CA3, - 0x10CA4, - 0x10CA5, - 0x10CA6, - 0x10CA7, - 0x10CA8, - 0x10CA9, - 0x10CAA, - 0x10CAB, - 0x10CAC, - 0x10CAD, - 0x10CAE, - 0x10CAF, - 0x10CB0, - 0x10CB1, - 0x10CB2, - 0x10CB3, - 0x10CC0, - 0x10CF3, - 0x10CFA, - 0x10D28, - 0x10D30, - 0x10D3A, - 0x10D40, - 0x10D50, - 0x10D51, - 0x10D52, - 0x10D53, - 0x10D54, - 0x10D55, - 0x10D56, - 0x10D57, - 0x10D58, - 0x10D59, - 0x10D5A, - 0x10D5B, - 0x10D5C, - 0x10D5D, - 0x10D5E, - 0x10D5F, - 0x10D60, - 0x10D61, - 0x10D62, - 0x10D63, - 0x10D64, - 0x10D65, - 0x10D66, - 0x10D69, - 0x10D86, - 0x10D8E, - 0x10D90, - 0x10E60, - 0x10E7F, - 0x10E80, - 0x10EAA, - 0x10EAB, - 0x10EAE, - 0x10EB0, - 0x10EB2, - 0x10EC2, - 0x10EC8, - 0x10ED0, - 0x10ED9, - 0x10EFA, - 0x10F28, - 0x10F30, - 0x10F5A, - 0x10F70, - 0x10F8A, - 0x10FB0, - 0x10FCC, - 0x10FE0, - 0x10FF7, - 0x11000, - 0x1104E, - 0x11052, - 0x11076, - 0x1107F, - 0x110BD, - 0x110BE, - 0x110C3, - 0x110D0, - 0x110E9, - 0x110F0, - 0x110FA, - 0x11100, - 0x11135, - 0x11136, - 0x11148, - 0x11150, - 0x11177, - 0x11180, - 0x111E0, - 0x111E1, - 0x111F5, - 0x11200, - 0x11212, - 0x11213, - 0x11242, - 0x11280, - 0x11287, - 0x11288, - 0x11289, - 0x1128A, - 0x1128E, - 0x1128F, - 0x1129E, - 0x1129F, - 0x112AA, - 0x112B0, - 0x112EB, - 0x112F0, - 0x112FA, - 0x11300, - 0x11304, - 0x11305, - 0x1130D, - 0x1130F, - 0x11311, - 0x11313, - 0x11329, - 0x1132A, - 0x11331, - 0x11332, - 0x11334, - 0x11335, - 0x1133A, - 0x1133B, - 0x11345, - 0x11347, - 0x11349, - 0x1134B, - 0x1134E, - 0x11350, - 0x11351, - 0x11357, - 0x11358, - 0x1135D, - 0x11364, - 0x11366, - 0x1136D, - 0x11370, - 0x11375, - 0x11380, - 0x1138A, - 0x1138B, - 0x1138C, - 0x1138E, - 0x1138F, - 0x11390, - 0x113B6, - 0x113B7, - 0x113C1, - 0x113C2, - 0x113C3, - 0x113C5, - 0x113C6, - 0x113C7, - 0x113CB, - 0x113CC, - 0x113D6, - 0x113D7, - 0x113D9, - 0x113E1, - 0x113E3, - 0x11400, - 0x1145C, - 0x1145D, - 0x11462, - 0x11480, - 0x114C8, - 0x114D0, - 0x114DA, - 0x11580, - 0x115B6, - 0x115B8, - 0x115DE, - 0x11600, - 0x11645, - 0x11650, - 0x1165A, - 0x11660, - 0x1166D, - 0x11680, - 0x116BA, - 0x116C0, - 0x116CA, - 0x116D0, - 0x116E4, - 0x11700, - 0x1171B, - 0x1171D, - 0x1172C, - 0x11730, - 0x11747, - 0x11800, - 0x1183C, - 0x118A0, - 0x118A1, - 0x118A2, - 0x118A3, - 0x118A4, - 0x118A5, - 0x118A6, - 0x118A7, - 0x118A8, - 0x118A9, - 0x118AA, - 0x118AB, - 0x118AC, - 0x118AD, - 0x118AE, - 0x118AF, - 0x118B0, - 0x118B1, - 0x118B2, - 0x118B3, - 0x118B4, - 0x118B5, - 0x118B6, - 0x118B7, - 0x118B8, - 0x118B9, - 0x118BA, - 0x118BB, - 0x118BC, - 0x118BD, - 0x118BE, - 0x118BF, - 0x118C0, - 0x118F3, - 0x118FF, - 0x11907, - 0x11909, - 0x1190A, - 0x1190C, - 0x11914, - 0x11915, - 0x11917, - 0x11918, - 0x11936, - 0x11937, - 0x11939, - 0x1193B, - 0x11947, - 0x11950, - 0x1195A, - 0x119A0, - 0x119A8, - 0x119AA, - 0x119D8, - 0x119DA, - 0x119E5, - 0x11A00, - 0x11A48, - 0x11A50, - 0x11AA3, - 0x11AB0, - 0x11AF9, - 0x11B00, - 0x11B0A, - 0x11B60, - 0x11B68, - 0x11BC0, - 0x11BE2, - 0x11BF0, - 0x11BFA, - 0x11C00, - 0x11C09, - 0x11C0A, - 0x11C37, - 0x11C38, - 0x11C46, - 0x11C50, - 0x11C6D, - 0x11C70, - 0x11C90, - 0x11C92, - 0x11CA8, - 0x11CA9, - 0x11CB7, - 0x11D00, - 0x11D07, - 0x11D08, - 0x11D0A, - 0x11D0B, - 0x11D37, - 0x11D3A, - 0x11D3B, - 0x11D3C, - 0x11D3E, - 0x11D3F, - 0x11D48, - 0x11D50, - 0x11D5A, - 0x11D60, - 0x11D66, - 0x11D67, - 0x11D69, - 0x11D6A, - 0x11D8F, - 0x11D90, - 0x11D92, - 0x11D93, - 0x11D99, - 0x11DA0, - 0x11DAA, - 0x11DB0, - 0x11DDC, - 0x11DE0, - 0x11DEA, - 0x11EE0, - 0x11EF9, - 0x11F00, - 0x11F11, - 0x11F12, - 0x11F3B, - 0x11F3E, - 0x11F5B, - 0x11FB0, - 0x11FB1, - 0x11FC0, - 0x11FF2, - 0x11FFF, - 0x1239A, - 0x12400, - 0x1246F, - 0x12470, - 0x12475, - 0x12480, - 0x12544, - 0x12F90, - 0x12FF3, - 0x13000, - 0x13430, - 0x13440, - 0x13456, - 0x13460, - 0x143FB, - 0x14400, - 0x14647, - 0x16100, - 0x1613A, - 0x16800, - 0x16A39, - 0x16A40, - 0x16A5F, - 0x16A60, - 0x16A6A, - 0x16A6E, - 0x16ABF, - 0x16AC0, - 0x16ACA, - 0x16AD0, - 0x16AEE, - 0x16AF0, - 0x16AF6, - 0x16B00, - 0x16B46, - 0x16B50, - 0x16B5A, - 0x16B5B, - 0x16B62, - 0x16B63, - 0x16B78, - 0x16B7D, - 0x16B90, - 0x16D40, - 0x16D7A, - 0x16E40, - 0x16E41, - 0x16E42, - 0x16E43, - 0x16E44, - 0x16E45, - 0x16E46, - 0x16E47, - 0x16E48, - 0x16E49, - 0x16E4A, - 0x16E4B, - 0x16E4C, - 0x16E4D, - 0x16E4E, - 0x16E4F, - 0x16E50, - 0x16E51, - 0x16E52, - 0x16E53, - 0x16E54, - 0x16E55, - 0x16E56, - 0x16E57, - 0x16E58, - 0x16E59, - 0x16E5A, - 0x16E5B, - 0x16E5C, - 0x16E5D, - 0x16E5E, - 0x16E5F, - 0x16E60, - 0x16E9B, - 0x16EA0, - 0x16EA1, - 0x16EA2, - 0x16EA3, - 0x16EA4, - 0x16EA5, - 0x16EA6, - 0x16EA7, - 0x16EA8, - 0x16EA9, - 0x16EAA, - 0x16EAB, - 0x16EAC, - 0x16EAD, - 0x16EAE, - 0x16EAF, - 0x16EB0, - 0x16EB1, - 0x16EB2, - 0x16EB3, - 0x16EB4, - 0x16EB5, - 0x16EB6, - 0x16EB7, - 0x16EB8, - 0x16EB9, - 0x16EBB, - 0x16ED4, - 0x16F00, - 0x16F4B, - 0x16F4F, - 0x16F88, - 0x16F8F, - 0x16FA0, - 0x16FE0, - 0x16FE5, - 0x16FF0, - 0x16FF7, - 0x17000, - 0x18CD6, - 0x18CFF, - 0x18D1F, - 0x18D80, - 0x18DF3, - 0x1AFF0, - 0x1AFF4, - 0x1AFF5, - 0x1AFFC, - 0x1AFFD, - 0x1AFFF, - 0x1B000, - 0x1B123, - 0x1B132, - 0x1B133, - 0x1B150, - 0x1B153, - 0x1B155, - 0x1B156, - 0x1B164, - 0x1B168, - 0x1B170, - 0x1B2FC, - 0x1BC00, - 0x1BC6B, - 0x1BC70, - 0x1BC7D, - 0x1BC80, - 0x1BC89, - 0x1BC90, - 0x1BC9A, - 0x1BC9C, - 0x1BCA0, - 0x1BCA4, - 0x1CC00, - 0x1CCD6, - 0x1CCD7, - 0x1CCD8, - 0x1CCD9, - 0x1CCDA, - 0x1CCDB, - 0x1CCDC, - 0x1CCDD, - 0x1CCDE, - 0x1CCDF, - 0x1CCE0, - 0x1CCE1, - 0x1CCE2, - 0x1CCE3, - 0x1CCE4, - 0x1CCE5, - 0x1CCE6, - 0x1CCE7, - 0x1CCE8, - 0x1CCE9, - 0x1CCEA, - 0x1CCEB, - 0x1CCEC, - 0x1CCED, - 0x1CCEE, - 0x1CCEF, - 0x1CCF0, - 0x1CCF1, - 0x1CCF2, - 0x1CCF3, - 0x1CCF4, - 0x1CCF5, - 0x1CCF6, - 0x1CCF7, - 0x1CCF8, - 0x1CCF9, - 0x1CCFA, - 0x1CCFD, - 0x1CD00, - 0x1CEB4, - 0x1CEBA, - 0x1CED1, - 0x1CEE0, - 0x1CEF1, - 0x1CF00, - 0x1CF2E, - 0x1CF30, - 0x1CF47, - 0x1CF50, - 0x1CFC4, - 0x1D000, - 0x1D0F6, - 0x1D100, - 0x1D127, - 0x1D129, - 0x1D15E, - 0x1D15F, - 0x1D160, - 0x1D161, - 0x1D162, - 0x1D163, - 0x1D164, - 0x1D165, - 0x1D173, - 0x1D17B, - 0x1D1BB, - 0x1D1BC, - 0x1D1BD, - 0x1D1BE, - 0x1D1BF, - 0x1D1C0, - 0x1D1C1, - 0x1D1EB, - 0x1D200, - 0x1D246, - 0x1D2C0, - 0x1D2D4, - 0x1D2E0, - 0x1D2F4, - 0x1D300, - 0x1D357, - 0x1D360, - 0x1D379, - 0x1D400, - 0x1D401, - 0x1D402, - 0x1D403, - 0x1D404, - 0x1D405, - 0x1D406, - 0x1D407, - 0x1D408, - 0x1D409, - 0x1D40A, - 0x1D40B, - 0x1D40C, - 0x1D40D, - 0x1D40E, - 0x1D40F, - 0x1D410, - 0x1D411, - 0x1D412, - 0x1D413, - 0x1D414, - 0x1D415, - 0x1D416, - 0x1D417, - 0x1D418, - 0x1D419, - 0x1D41A, - 0x1D41B, - 0x1D41C, - 0x1D41D, - 0x1D41E, - 0x1D41F, - 0x1D420, - 0x1D421, - 0x1D422, - 0x1D423, - 0x1D424, - 0x1D425, - 0x1D426, - 0x1D427, - 0x1D428, - 0x1D429, - 0x1D42A, - 0x1D42B, - 0x1D42C, - 0x1D42D, - 0x1D42E, - 0x1D42F, - 0x1D430, - 0x1D431, - 0x1D432, - 0x1D433, - 0x1D434, - 0x1D435, - 0x1D436, - 0x1D437, - 0x1D438, - 0x1D439, - 0x1D43A, - 0x1D43B, - 0x1D43C, - 0x1D43D, - 0x1D43E, - 0x1D43F, - 0x1D440, - 0x1D441, - 0x1D442, - 0x1D443, - 0x1D444, - 0x1D445, - 0x1D446, - 0x1D447, - 0x1D448, - 0x1D449, - 0x1D44A, - 0x1D44B, - 0x1D44C, - 0x1D44D, - 0x1D44E, - 0x1D44F, - 0x1D450, - 0x1D451, - 0x1D452, - 0x1D453, - 0x1D454, - 0x1D455, - 0x1D456, - 0x1D457, - 0x1D458, - 0x1D459, - 0x1D45A, - 0x1D45B, - 0x1D45C, - 0x1D45D, - 0x1D45E, - 0x1D45F, - 0x1D460, - 0x1D461, - 0x1D462, - 0x1D463, - 0x1D464, - 0x1D465, - 0x1D466, - 0x1D467, - 0x1D468, - 0x1D469, - 0x1D46A, - 0x1D46B, - 0x1D46C, - 0x1D46D, - 0x1D46E, - 0x1D46F, - 0x1D470, - 0x1D471, - 0x1D472, - 0x1D473, - 0x1D474, - 0x1D475, - 0x1D476, - 0x1D477, - 0x1D478, - 0x1D479, - 0x1D47A, - 0x1D47B, - 0x1D47C, - 0x1D47D, - 0x1D47E, - 0x1D47F, - 0x1D480, - 0x1D481, - 0x1D482, - 0x1D483, - 0x1D484, - 0x1D485, - 0x1D486, - 0x1D487, - 0x1D488, - 0x1D489, - 0x1D48A, - 0x1D48B, - 0x1D48C, - 0x1D48D, - 0x1D48E, - 0x1D48F, - 0x1D490, - 0x1D491, - 0x1D492, - 0x1D493, - 0x1D494, - 0x1D495, - 0x1D496, - 0x1D497, - 0x1D498, - 0x1D499, - 0x1D49A, - 0x1D49B, - 0x1D49C, - 0x1D49D, - 0x1D49E, - 0x1D49F, - 0x1D4A0, - 0x1D4A2, - 0x1D4A3, - 0x1D4A5, - 0x1D4A6, - 0x1D4A7, - 0x1D4A9, - 0x1D4AA, - 0x1D4AB, - 0x1D4AC, - 0x1D4AD, - 0x1D4AE, - 0x1D4AF, - 0x1D4B0, - 0x1D4B1, - 0x1D4B2, - 0x1D4B3, - 0x1D4B4, - 0x1D4B5, - 0x1D4B6, - 0x1D4B7, - 0x1D4B8, - 0x1D4B9, - 0x1D4BA, - 0x1D4BB, - 0x1D4BC, - 0x1D4BD, - 0x1D4BE, - 0x1D4BF, - 0x1D4C0, - 0x1D4C1, - 0x1D4C2, - 0x1D4C3, - 0x1D4C4, - 0x1D4C5, - 0x1D4C6, - 0x1D4C7, - 0x1D4C8, - 0x1D4C9, - 0x1D4CA, - 0x1D4CB, - 0x1D4CC, - 0x1D4CD, - 0x1D4CE, - 0x1D4CF, - 0x1D4D0, - 0x1D4D1, - 0x1D4D2, - 0x1D4D3, - 0x1D4D4, - 0x1D4D5, - 0x1D4D6, - 0x1D4D7, - 0x1D4D8, - 0x1D4D9, - 0x1D4DA, - 0x1D4DB, - 0x1D4DC, - 0x1D4DD, - 0x1D4DE, - 0x1D4DF, - 0x1D4E0, - 0x1D4E1, - 0x1D4E2, - 0x1D4E3, - 0x1D4E4, - 0x1D4E5, - 0x1D4E6, - 0x1D4E7, - 0x1D4E8, - 0x1D4E9, - 0x1D4EA, - 0x1D4EB, - 0x1D4EC, - 0x1D4ED, - 0x1D4EE, - 0x1D4EF, - 0x1D4F0, - 0x1D4F1, - 0x1D4F2, - 0x1D4F3, - 0x1D4F4, - 0x1D4F5, - 0x1D4F6, - 0x1D4F7, - 0x1D4F8, - 0x1D4F9, - 0x1D4FA, - 0x1D4FB, - 0x1D4FC, - 0x1D4FD, - 0x1D4FE, - 0x1D4FF, - 0x1D500, - 0x1D501, - 0x1D502, - 0x1D503, - 0x1D504, - 0x1D505, - 0x1D506, - 0x1D507, - 0x1D508, - 0x1D509, - 0x1D50A, - 0x1D50B, - 0x1D50D, - 0x1D50E, - 0x1D50F, - 0x1D510, - 0x1D511, - 0x1D512, - 0x1D513, - 0x1D514, - 0x1D515, - 0x1D516, - 0x1D517, - 0x1D518, - 0x1D519, - 0x1D51A, - 0x1D51B, - 0x1D51C, - 0x1D51D, - 0x1D51E, - 0x1D51F, - 0x1D520, - 0x1D521, - 0x1D522, - 0x1D523, - 0x1D524, - 0x1D525, - 0x1D526, - 0x1D527, - 0x1D528, - 0x1D529, - 0x1D52A, - 0x1D52B, - 0x1D52C, - 0x1D52D, - 0x1D52E, - 0x1D52F, - 0x1D530, - 0x1D531, - 0x1D532, - 0x1D533, - 0x1D534, - 0x1D535, - 0x1D536, - 0x1D537, - 0x1D538, - 0x1D539, - 0x1D53A, - 0x1D53B, - 0x1D53C, - 0x1D53D, - 0x1D53E, - 0x1D53F, - 0x1D540, - 0x1D541, - 0x1D542, - 0x1D543, - 0x1D544, - 0x1D545, - 0x1D546, - 0x1D547, - 0x1D54A, - 0x1D54B, - 0x1D54C, - 0x1D54D, - 0x1D54E, - 0x1D54F, - 0x1D550, - 0x1D551, - 0x1D552, - 0x1D553, - 0x1D554, - 0x1D555, - 0x1D556, - 0x1D557, - 0x1D558, - 0x1D559, - 0x1D55A, - 0x1D55B, - 0x1D55C, - 0x1D55D, - 0x1D55E, - 0x1D55F, - 0x1D560, - 0x1D561, - 0x1D562, - 0x1D563, - 0x1D564, - 0x1D565, - 0x1D566, - 0x1D567, - 0x1D568, - 0x1D569, - 0x1D56A, - 0x1D56B, - 0x1D56C, - 0x1D56D, - 0x1D56E, - 0x1D56F, - 0x1D570, - 0x1D571, - 0x1D572, - 0x1D573, - 0x1D574, - 0x1D575, - 0x1D576, - 0x1D577, - 0x1D578, - 0x1D579, - 0x1D57A, - 0x1D57B, - 0x1D57C, - 0x1D57D, - 0x1D57E, - 0x1D57F, - 0x1D580, - 0x1D581, - 0x1D582, - 0x1D583, - 0x1D584, - 0x1D585, - 0x1D586, - 0x1D587, - 0x1D588, - 0x1D589, - 0x1D58A, - 0x1D58B, - 0x1D58C, - 0x1D58D, - 0x1D58E, - 0x1D58F, - 0x1D590, - 0x1D591, - 0x1D592, - 0x1D593, - 0x1D594, - 0x1D595, - 0x1D596, - 0x1D597, - 0x1D598, - 0x1D599, - 0x1D59A, - 0x1D59B, - 0x1D59C, - 0x1D59D, - 0x1D59E, - 0x1D59F, - 0x1D5A0, - 0x1D5A1, - 0x1D5A2, - 0x1D5A3, - 0x1D5A4, - 0x1D5A5, - 0x1D5A6, - 0x1D5A7, - 0x1D5A8, - 0x1D5A9, - 0x1D5AA, - 0x1D5AB, - 0x1D5AC, - 0x1D5AD, - 0x1D5AE, - 0x1D5AF, - 0x1D5B0, - 0x1D5B1, - 0x1D5B2, - 0x1D5B3, - 0x1D5B4, - 0x1D5B5, - 0x1D5B6, - 0x1D5B7, - 0x1D5B8, - 0x1D5B9, - 0x1D5BA, - 0x1D5BB, - 0x1D5BC, - 0x1D5BD, - 0x1D5BE, - 0x1D5BF, - 0x1D5C0, - 0x1D5C1, - 0x1D5C2, - 0x1D5C3, - 0x1D5C4, - 0x1D5C5, - 0x1D5C6, - 0x1D5C7, - 0x1D5C8, - 0x1D5C9, - 0x1D5CA, - 0x1D5CB, - 0x1D5CC, - 0x1D5CD, - 0x1D5CE, - 0x1D5CF, - 0x1D5D0, - 0x1D5D1, - 0x1D5D2, - 0x1D5D3, - 0x1D5D4, - 0x1D5D5, - 0x1D5D6, - 0x1D5D7, - 0x1D5D8, - 0x1D5D9, - 0x1D5DA, - 0x1D5DB, - 0x1D5DC, - 0x1D5DD, - 0x1D5DE, - 0x1D5DF, - 0x1D5E0, - 0x1D5E1, - 0x1D5E2, - 0x1D5E3, - 0x1D5E4, - 0x1D5E5, - 0x1D5E6, - 0x1D5E7, - 0x1D5E8, - 0x1D5E9, - 0x1D5EA, - 0x1D5EB, - 0x1D5EC, - 0x1D5ED, - 0x1D5EE, - 0x1D5EF, - 0x1D5F0, - 0x1D5F1, - 0x1D5F2, - 0x1D5F3, - 0x1D5F4, - 0x1D5F5, - 0x1D5F6, - 0x1D5F7, - 0x1D5F8, - 0x1D5F9, - 0x1D5FA, - 0x1D5FB, - 0x1D5FC, - 0x1D5FD, - 0x1D5FE, - 0x1D5FF, - 0x1D600, - 0x1D601, - 0x1D602, - 0x1D603, - 0x1D604, - 0x1D605, - 0x1D606, - 0x1D607, - 0x1D608, - 0x1D609, - 0x1D60A, - 0x1D60B, - 0x1D60C, - 0x1D60D, - 0x1D60E, - 0x1D60F, - 0x1D610, - 0x1D611, - 0x1D612, - 0x1D613, - 0x1D614, - 0x1D615, - 0x1D616, - 0x1D617, - 0x1D618, - 0x1D619, - 0x1D61A, - 0x1D61B, - 0x1D61C, - 0x1D61D, - 0x1D61E, - 0x1D61F, - 0x1D620, - 0x1D621, - 0x1D622, - 0x1D623, - 0x1D624, - 0x1D625, - 0x1D626, - 0x1D627, - 0x1D628, - 0x1D629, - 0x1D62A, - 0x1D62B, - 0x1D62C, - 0x1D62D, - 0x1D62E, - 0x1D62F, - 0x1D630, - 0x1D631, - 0x1D632, - 0x1D633, - 0x1D634, - 0x1D635, - 0x1D636, - 0x1D637, - 0x1D638, - 0x1D639, - 0x1D63A, - 0x1D63B, - 0x1D63C, - 0x1D63D, - 0x1D63E, - 0x1D63F, - 0x1D640, - 0x1D641, - 0x1D642, - 0x1D643, - 0x1D644, - 0x1D645, - 0x1D646, - 0x1D647, - 0x1D648, - 0x1D649, - 0x1D64A, - 0x1D64B, - 0x1D64C, - 0x1D64D, - 0x1D64E, - 0x1D64F, - 0x1D650, - 0x1D651, - 0x1D652, - 0x1D653, - 0x1D654, - 0x1D655, - 0x1D656, - 0x1D657, - 0x1D658, - 0x1D659, - 0x1D65A, - 0x1D65B, - 0x1D65C, - 0x1D65D, - 0x1D65E, - 0x1D65F, - 0x1D660, - 0x1D661, - 0x1D662, - 0x1D663, - 0x1D664, - 0x1D665, - 0x1D666, - 0x1D667, - 0x1D668, - 0x1D669, - 0x1D66A, - 0x1D66B, - 0x1D66C, - 0x1D66D, - 0x1D66E, - 0x1D66F, - 0x1D670, - 0x1D671, - 0x1D672, - 0x1D673, - 0x1D674, - 0x1D675, - 0x1D676, - 0x1D677, - 0x1D678, - 0x1D679, - 0x1D67A, - 0x1D67B, - 0x1D67C, - 0x1D67D, - 0x1D67E, - 0x1D67F, - 0x1D680, - 0x1D681, - 0x1D682, - 0x1D683, - 0x1D684, - 0x1D685, - 0x1D686, - 0x1D687, - 0x1D688, - 0x1D689, - 0x1D68A, - 0x1D68B, - 0x1D68C, - 0x1D68D, - 0x1D68E, - 0x1D68F, - 0x1D690, - 0x1D691, - 0x1D692, - 0x1D693, - 0x1D694, - 0x1D695, - 0x1D696, - 0x1D697, - 0x1D698, - 0x1D699, - 0x1D69A, - 0x1D69B, - 0x1D69C, - 0x1D69D, - 0x1D69E, - 0x1D69F, - 0x1D6A0, - 0x1D6A1, - 0x1D6A2, - 0x1D6A3, - 0x1D6A4, - 0x1D6A5, - 0x1D6A6, - 0x1D6A8, - 0x1D6A9, - 0x1D6AA, - 0x1D6AB, - 0x1D6AC, - 0x1D6AD, - 0x1D6AE, - 0x1D6AF, - 0x1D6B0, - 0x1D6B1, - 0x1D6B2, - 0x1D6B3, - 0x1D6B4, - 0x1D6B5, - 0x1D6B6, - 0x1D6B7, - 0x1D6B8, - 0x1D6B9, - 0x1D6BA, - 0x1D6BB, - 0x1D6BC, - 0x1D6BD, - 0x1D6BE, - 0x1D6BF, - 0x1D6C0, - 0x1D6C1, - 0x1D6C2, - 0x1D6C3, - 0x1D6C4, - 0x1D6C5, - 0x1D6C6, - 0x1D6C7, - 0x1D6C8, - 0x1D6C9, - 0x1D6CA, - 0x1D6CB, - 0x1D6CC, - 0x1D6CD, - 0x1D6CE, - 0x1D6CF, - 0x1D6D0, - 0x1D6D1, - 0x1D6D2, - 0x1D6D3, - 0x1D6D5, - 0x1D6D6, - 0x1D6D7, - 0x1D6D8, - 0x1D6D9, - 0x1D6DA, - 0x1D6DB, - 0x1D6DC, - 0x1D6DD, - 0x1D6DE, - 0x1D6DF, - 0x1D6E0, - 0x1D6E1, - 0x1D6E2, - 0x1D6E3, - 0x1D6E4, - 0x1D6E5, - 0x1D6E6, - 0x1D6E7, - 0x1D6E8, - 0x1D6E9, - 0x1D6EA, - 0x1D6EB, - 0x1D6EC, - 0x1D6ED, - 0x1D6EE, - 0x1D6EF, - 0x1D6F0, - 0x1D6F1, - 0x1D6F2, - 0x1D6F3, - 0x1D6F4, - 0x1D6F5, - 0x1D6F6, - 0x1D6F7, - 0x1D6F8, - 0x1D6F9, - 0x1D6FA, - 0x1D6FB, - 0x1D6FC, - 0x1D6FD, - 0x1D6FE, - 0x1D6FF, - 0x1D700, - 0x1D701, - 0x1D702, - 0x1D703, - 0x1D704, - 0x1D705, - 0x1D706, - 0x1D707, - 0x1D708, - 0x1D709, - 0x1D70A, - 0x1D70B, - 0x1D70C, - 0x1D70D, - 0x1D70F, - 0x1D710, - 0x1D711, - 0x1D712, - 0x1D713, - 0x1D714, - 0x1D715, - 0x1D716, - 0x1D717, - 0x1D718, - 0x1D719, - 0x1D71A, - 0x1D71B, - 0x1D71C, - 0x1D71D, - 0x1D71E, - 0x1D71F, - 0x1D720, - 0x1D721, - 0x1D722, - 0x1D723, - 0x1D724, - 0x1D725, - 0x1D726, - 0x1D727, - 0x1D728, - 0x1D729, - 0x1D72A, - 0x1D72B, - 0x1D72C, - 0x1D72D, - 0x1D72E, - 0x1D72F, - 0x1D730, - 0x1D731, - 0x1D732, - 0x1D733, - 0x1D734, - 0x1D735, - 0x1D736, - 0x1D737, - 0x1D738, - 0x1D739, - 0x1D73A, - 0x1D73B, - 0x1D73C, - 0x1D73D, - 0x1D73E, - 0x1D73F, - 0x1D740, - 0x1D741, - 0x1D742, - 0x1D743, - 0x1D744, - 0x1D745, - 0x1D746, - 0x1D747, - 0x1D749, - 0x1D74A, - 0x1D74B, - 0x1D74C, - 0x1D74D, - 0x1D74E, - 0x1D74F, - 0x1D750, - 0x1D751, - 0x1D752, - 0x1D753, - 0x1D754, - 0x1D755, - 0x1D756, - 0x1D757, - 0x1D758, - 0x1D759, - 0x1D75A, - 0x1D75B, - 0x1D75C, - 0x1D75D, - 0x1D75E, - 0x1D75F, - 0x1D760, - 0x1D761, - 0x1D762, - 0x1D763, - 0x1D764, - 0x1D765, - 0x1D766, - 0x1D767, - 0x1D768, - 0x1D769, - 0x1D76A, - 0x1D76B, - 0x1D76C, - 0x1D76D, - 0x1D76E, - 0x1D76F, - 0x1D770, - 0x1D771, - 0x1D772, - 0x1D773, - 0x1D774, - 0x1D775, - 0x1D776, - 0x1D777, - 0x1D778, - 0x1D779, - 0x1D77A, - 0x1D77B, - 0x1D77C, - 0x1D77D, - 0x1D77E, - 0x1D77F, - 0x1D780, - 0x1D781, - 0x1D783, - 0x1D784, - 0x1D785, - 0x1D786, - 0x1D787, - 0x1D788, - 0x1D789, - 0x1D78A, - 0x1D78B, - 0x1D78C, - 0x1D78D, - 0x1D78E, - 0x1D78F, - 0x1D790, - 0x1D791, - 0x1D792, - 0x1D793, - 0x1D794, - 0x1D795, - 0x1D796, - 0x1D797, - 0x1D798, - 0x1D799, - 0x1D79A, - 0x1D79B, - 0x1D79C, - 0x1D79D, - 0x1D79E, - 0x1D79F, - 0x1D7A0, - 0x1D7A1, - 0x1D7A2, - 0x1D7A3, - 0x1D7A4, - 0x1D7A5, - 0x1D7A6, - 0x1D7A7, - 0x1D7A8, - 0x1D7A9, - 0x1D7AA, - 0x1D7AB, - 0x1D7AC, - 0x1D7AD, - 0x1D7AE, - 0x1D7AF, - 0x1D7B0, - 0x1D7B1, - 0x1D7B2, - 0x1D7B3, - 0x1D7B4, - 0x1D7B5, - 0x1D7B6, - 0x1D7B7, - 0x1D7B8, - 0x1D7B9, - 0x1D7BA, - 0x1D7BB, - 0x1D7BD, - 0x1D7BE, - 0x1D7BF, - 0x1D7C0, - 0x1D7C1, - 0x1D7C2, - 0x1D7C3, - 0x1D7C4, - 0x1D7C5, - 0x1D7C6, - 0x1D7C7, - 0x1D7C8, - 0x1D7C9, - 0x1D7CA, - 0x1D7CC, - 0x1D7CE, - 0x1D7CF, - 0x1D7D0, - 0x1D7D1, - 0x1D7D2, - 0x1D7D3, - 0x1D7D4, - 0x1D7D5, - 0x1D7D6, - 0x1D7D7, - 0x1D7D8, - 0x1D7D9, - 0x1D7DA, - 0x1D7DB, - 0x1D7DC, - 0x1D7DD, - 0x1D7DE, - 0x1D7DF, - 0x1D7E0, - 0x1D7E1, - 0x1D7E2, - 0x1D7E3, - 0x1D7E4, - 0x1D7E5, - 0x1D7E6, - 0x1D7E7, - 0x1D7E8, - 0x1D7E9, - 0x1D7EA, - 0x1D7EB, - 0x1D7EC, - 0x1D7ED, - 0x1D7EE, - 0x1D7EF, - 0x1D7F0, - 0x1D7F1, - 0x1D7F2, - 0x1D7F3, - 0x1D7F4, - 0x1D7F5, - 0x1D7F6, - 0x1D7F7, - 0x1D7F8, - 0x1D7F9, - 0x1D7FA, - 0x1D7FB, - 0x1D7FC, - 0x1D7FD, - 0x1D7FE, - 0x1D7FF, - 0x1D800, - 0x1DA8C, - 0x1DA9B, - 0x1DAA0, - 0x1DAA1, - 0x1DAB0, - 0x1DF00, - 0x1DF1F, - 0x1DF25, - 0x1DF2B, - 0x1E000, - 0x1E007, - 0x1E008, - 0x1E019, - 0x1E01B, - 0x1E022, - 0x1E023, - 0x1E025, - 0x1E026, - 0x1E02B, - 0x1E030, - 0x1E031, - 0x1E032, - 0x1E033, - 0x1E034, - 0x1E035, - 0x1E036, - 0x1E037, - 0x1E038, - 0x1E039, - 0x1E03A, - 0x1E03B, - 0x1E03C, - 0x1E03D, - 0x1E03E, - 0x1E03F, - 0x1E040, - 0x1E041, - 0x1E042, - 0x1E043, - 0x1E044, - 0x1E045, - 0x1E046, - 0x1E047, - 0x1E048, - 0x1E049, - 0x1E04A, - 0x1E04B, - 0x1E04C, - 0x1E04D, - 0x1E04E, - 0x1E04F, - 0x1E050, - 0x1E051, - 0x1E052, - 0x1E053, - 0x1E054, - 0x1E055, - 0x1E056, - 0x1E057, - 0x1E058, - 0x1E059, - 0x1E05A, - 0x1E05B, - 0x1E05C, - 0x1E05D, - 0x1E05E, - 0x1E05F, - 0x1E060, - 0x1E061, - 0x1E062, - 0x1E063, - 0x1E064, - 0x1E065, - 0x1E066, - 0x1E067, - 0x1E068, - 0x1E069, - 0x1E06A, - 0x1E06B, - 0x1E06C, - 0x1E06D, - 0x1E06E, - 0x1E08F, - 0x1E090, - 0x1E100, - 0x1E12D, - 0x1E130, - 0x1E13E, - 0x1E140, - 0x1E14A, - 0x1E14E, - 0x1E150, - 0x1E290, - 0x1E2AF, - 0x1E2C0, - 0x1E2FA, - 0x1E2FF, - 0x1E300, - 0x1E4D0, - 0x1E4FA, - 0x1E5D0, - 0x1E5FB, - 0x1E5FF, - 0x1E600, - 0x1E6C0, - 0x1E6DF, - 0x1E6E0, - 0x1E6F6, - 0x1E6FE, - 0x1E700, - 0x1E7E0, - 0x1E7E7, - 0x1E7E8, - 0x1E7EC, - 0x1E7ED, - 0x1E7EF, - 0x1E7F0, - 0x1E7FF, - 0x1E800, - 0x1E8C5, - 0x1E8C7, - 0x1E8D7, - 0x1E900, - 0x1E901, - 0x1E902, - 0x1E903, - 0x1E904, - 0x1E905, - 0x1E906, - 0x1E907, - 0x1E908, - 0x1E909, - 0x1E90A, - 0x1E90B, - 0x1E90C, - 0x1E90D, - 0x1E90E, - 0x1E90F, - 0x1E910, - 0x1E911, - 0x1E912, - 0x1E913, - 0x1E914, - 0x1E915, - 0x1E916, - 0x1E917, - 0x1E918, - 0x1E919, - 0x1E91A, - 0x1E91B, - 0x1E91C, - 0x1E91D, - 0x1E91E, - 0x1E91F, - 0x1E920, - 0x1E921, - 0x1E922, - 0x1E94C, - 0x1E950, - 0x1E95A, - 0x1E95E, - 0x1E960, - 0x1EC71, - 0x1ECB5, - 0x1ED01, - 0x1ED3E, - 0x1EE00, - 0x1EE01, - 0x1EE02, - 0x1EE03, - 0x1EE04, - 0x1EE05, - 0x1EE06, - 0x1EE07, - 0x1EE08, - 0x1EE09, - 0x1EE0A, - 0x1EE0B, - 0x1EE0C, - 0x1EE0D, - 0x1EE0E, - 0x1EE0F, - 0x1EE10, - 0x1EE11, - 0x1EE12, - 0x1EE13, - 0x1EE14, - 0x1EE15, - 0x1EE16, - 0x1EE17, - 0x1EE18, - 0x1EE19, - 0x1EE1A, - 0x1EE1B, - 0x1EE1C, - 0x1EE1D, - 0x1EE1E, - 0x1EE1F, - 0x1EE20, - 0x1EE21, - 0x1EE22, - 0x1EE23, - 0x1EE24, - 0x1EE25, - 0x1EE27, - 0x1EE28, - 0x1EE29, - 0x1EE2A, - 0x1EE2B, - 0x1EE2C, - 0x1EE2D, - 0x1EE2E, - 0x1EE2F, - 0x1EE30, - 0x1EE31, - 0x1EE32, - 0x1EE33, - 0x1EE34, - 0x1EE35, - 0x1EE36, - 0x1EE37, - 0x1EE38, - 0x1EE39, - 0x1EE3A, - 0x1EE3B, - 0x1EE3C, - 0x1EE42, - 0x1EE43, - 0x1EE47, - 0x1EE48, - 0x1EE49, - 0x1EE4A, - 0x1EE4B, - 0x1EE4C, - 0x1EE4D, - 0x1EE4E, - 0x1EE4F, - 0x1EE50, - 0x1EE51, - 0x1EE52, - 0x1EE53, - 0x1EE54, - 0x1EE55, - 0x1EE57, - 0x1EE58, - 0x1EE59, - 0x1EE5A, - 0x1EE5B, - 0x1EE5C, - 0x1EE5D, - 0x1EE5E, - 0x1EE5F, - 0x1EE60, - 0x1EE61, - 0x1EE62, - 0x1EE63, - 0x1EE64, - 0x1EE65, - 0x1EE67, - 0x1EE68, - 0x1EE69, - 0x1EE6A, - 0x1EE6B, - 0x1EE6C, - 0x1EE6D, - 0x1EE6E, - 0x1EE6F, - 0x1EE70, - 0x1EE71, - 0x1EE72, - 0x1EE73, - 0x1EE74, - 0x1EE75, - 0x1EE76, - 0x1EE77, - 0x1EE78, - 0x1EE79, - 0x1EE7A, - 0x1EE7B, - 0x1EE7C, - 0x1EE7D, - 0x1EE7E, - 0x1EE7F, - 0x1EE80, - 0x1EE81, - 0x1EE82, - 0x1EE83, - 0x1EE84, - 0x1EE85, - 0x1EE86, - 0x1EE87, - 0x1EE88, - 0x1EE89, - 0x1EE8A, - 0x1EE8B, - 0x1EE8C, - 0x1EE8D, - 0x1EE8E, - 0x1EE8F, - 0x1EE90, - 0x1EE91, - 0x1EE92, - 0x1EE93, - 0x1EE94, - 0x1EE95, - 0x1EE96, - 0x1EE97, - 0x1EE98, - 0x1EE99, - 0x1EE9A, - 0x1EE9B, - 0x1EE9C, - 0x1EEA1, - 0x1EEA2, - 0x1EEA3, - 0x1EEA4, - 0x1EEA5, - 0x1EEA6, - 0x1EEA7, - 0x1EEA8, - 0x1EEA9, - 0x1EEAA, - 0x1EEAB, - 0x1EEAC, - 0x1EEAD, - 0x1EEAE, - 0x1EEAF, - 0x1EEB0, - 0x1EEB1, - 0x1EEB2, - 0x1EEB3, - 0x1EEB4, - 0x1EEB5, - 0x1EEB6, - 0x1EEB7, - 0x1EEB8, - 0x1EEB9, - 0x1EEBA, - 0x1EEBB, - 0x1EEBC, - 0x1EEF0, - 0x1EEF2, - 0x1F000, - 0x1F02C, - 0x1F030, - 0x1F094, - 0x1F0A0, - 0x1F0AF, - 0x1F0B1, - 0x1F0C0, - 0x1F0C1, - 0x1F0D0, - 0x1F0D1, - 0x1F0F6, - 0x1F101, - 0x1F102, - 0x1F103, - 0x1F104, - 0x1F105, - 0x1F106, - 0x1F107, - 0x1F108, - 0x1F109, - 0x1F10A, - 0x1F10B, - 0x1F110, - 0x1F111, - 0x1F112, - 0x1F113, - 0x1F114, - 0x1F115, - 0x1F116, - 0x1F117, - 0x1F118, - 0x1F119, - 0x1F11A, - 0x1F11B, - 0x1F11C, - 0x1F11D, - 0x1F11E, - 0x1F11F, - 0x1F120, - 0x1F121, - 0x1F122, - 0x1F123, - 0x1F124, - 0x1F125, - 0x1F126, - 0x1F127, - 0x1F128, - 0x1F129, - 0x1F12A, - 0x1F12B, - 0x1F12C, - 0x1F12D, - 0x1F12E, - 0x1F12F, - 0x1F130, - 0x1F131, - 0x1F132, - 0x1F133, - 0x1F134, - 0x1F135, - 0x1F136, - 0x1F137, - 0x1F138, - 0x1F139, - 0x1F13A, - 0x1F13B, - 0x1F13C, - 0x1F13D, - 0x1F13E, - 0x1F13F, - 0x1F140, - 0x1F141, - 0x1F142, - 0x1F143, - 0x1F144, - 0x1F145, - 0x1F146, - 0x1F147, - 0x1F148, - 0x1F149, - 0x1F14A, - 0x1F14B, - 0x1F14C, - 0x1F14D, - 0x1F14E, - 0x1F14F, - 0x1F150, - 0x1F16A, - 0x1F16B, - 0x1F16C, - 0x1F16D, - 0x1F190, - 0x1F191, - 0x1F1AE, - 0x1F1E6, - 0x1F200, - 0x1F201, - 0x1F202, - 0x1F203, - 0x1F210, - 0x1F211, - 0x1F212, - 0x1F213, - 0x1F214, - 0x1F215, - 0x1F216, - 0x1F217, - 0x1F218, - 0x1F219, - 0x1F21A, - 0x1F21B, - 0x1F21C, - 0x1F21D, - 0x1F21E, - 0x1F21F, - 0x1F220, - 0x1F221, - 0x1F222, - 0x1F223, - 0x1F224, - 0x1F225, - 0x1F226, - 0x1F227, - 0x1F228, - 0x1F229, - 0x1F22A, - 0x1F22B, - 0x1F22C, - 0x1F22D, - 0x1F22E, - 0x1F22F, - 0x1F230, - 0x1F231, - 0x1F232, - 0x1F233, - 0x1F234, - 0x1F235, - 0x1F236, - 0x1F237, - 0x1F238, - 0x1F239, - 0x1F23A, - 0x1F23B, - 0x1F23C, - 0x1F240, - 0x1F241, - 0x1F242, - 0x1F243, - 0x1F244, - 0x1F245, - 0x1F246, - 0x1F247, - 0x1F248, - 0x1F249, - 0x1F250, - 0x1F251, - 0x1F252, - 0x1F260, - 0x1F266, - 0x1F300, - 0x1F6D9, - 0x1F6DC, - 0x1F6ED, - 0x1F6F0, - 0x1F6FD, - 0x1F700, - 0x1F7DA, - 0x1F7E0, - 0x1F7EC, - 0x1F7F0, - 0x1F7F1, - 0x1F800, - 0x1F80C, - 0x1F810, - 0x1F848, - 0x1F850, - 0x1F85A, - 0x1F860, - 0x1F888, - 0x1F890, - 0x1F8AE, - 0x1F8B0, - 0x1F8BC, - 0x1F8C0, - 0x1F8C2, - 0x1F8D0, - 0x1F8D9, - 0x1F900, - 0x1FA58, - 0x1FA60, - 0x1FA6E, - 0x1FA70, - 0x1FA7D, - 0x1FA80, - 0x1FA8B, - 0x1FA8E, - 0x1FAC7, - 0x1FAC8, - 0x1FAC9, - 0x1FACD, - 0x1FADD, - 0x1FADF, - 0x1FAEB, - 0x1FAEF, - 0x1FAF9, - 0x1FB00, - 0x1FB93, - 0x1FB94, - 0x1FBF0, - 0x1FBF1, - 0x1FBF2, - 0x1FBF3, - 0x1FBF4, - 0x1FBF5, - 0x1FBF6, - 0x1FBF7, - 0x1FBF8, - 0x1FBF9, - 0x1FBFA, - 0x1FBFB, - 0x20000, - 0x2A6E0, - 0x2A700, - 0x2B81E, - 0x2B820, - 0x2CEAE, - 0x2CEB0, - 0x2EBE1, - 0x2EBF0, - 0x2EE5E, - 0x2F800, - 0x2F801, - 0x2F802, - 0x2F803, - 0x2F804, - 0x2F805, - 0x2F806, - 0x2F807, - 0x2F808, - 0x2F809, - 0x2F80A, - 0x2F80B, - 0x2F80C, - 0x2F80D, - 0x2F80E, - 0x2F80F, - 0x2F810, - 0x2F811, - 0x2F812, - 0x2F813, - 0x2F814, - 0x2F815, - 0x2F816, - 0x2F817, - 0x2F818, - 0x2F819, - 0x2F81A, - 0x2F81B, - 0x2F81C, - 0x2F81D, - 0x2F81E, - 0x2F81F, - 0x2F820, - 0x2F821, - 0x2F822, - 0x2F823, - 0x2F824, - 0x2F825, - 0x2F826, - 0x2F827, - 0x2F828, - 0x2F829, - 0x2F82A, - 0x2F82B, - 0x2F82C, - 0x2F82D, - 0x2F82E, - 0x2F82F, - 0x2F830, - 0x2F831, - 0x2F834, - 0x2F835, - 0x2F836, - 0x2F837, - 0x2F838, - 0x2F839, - 0x2F83A, - 0x2F83B, - 0x2F83C, - 0x2F83D, - 0x2F83E, - 0x2F83F, - 0x2F840, - 0x2F841, - 0x2F842, - 0x2F843, - 0x2F844, - 0x2F845, - 0x2F847, - 0x2F848, - 0x2F849, - 0x2F84A, - 0x2F84B, - 0x2F84C, - 0x2F84D, - 0x2F84E, - 0x2F84F, - 0x2F850, - 0x2F851, - 0x2F852, - 0x2F853, - 0x2F854, - 0x2F855, - 0x2F856, - 0x2F857, - 0x2F858, - 0x2F859, - 0x2F85A, - 0x2F85B, - 0x2F85C, - 0x2F85D, - 0x2F85E, - 0x2F85F, - 0x2F860, - 0x2F861, - 0x2F862, - 0x2F863, - 0x2F864, - 0x2F865, - 0x2F866, - 0x2F867, - 0x2F868, - 0x2F869, - 0x2F86A, - 0x2F86C, - 0x2F86D, - 0x2F86E, - 0x2F86F, - 0x2F870, - 0x2F871, - 0x2F872, - 0x2F873, - 0x2F874, - 0x2F875, - 0x2F876, - 0x2F877, - 0x2F878, - 0x2F879, - 0x2F87A, - 0x2F87B, - 0x2F87C, - 0x2F87D, - 0x2F87E, - 0x2F87F, - 0x2F880, - 0x2F881, - 0x2F882, - 0x2F883, - 0x2F884, - 0x2F885, - 0x2F886, - 0x2F887, - 0x2F888, - 0x2F889, - 0x2F88A, - 0x2F88B, - 0x2F88C, - 0x2F88D, - 0x2F88E, - 0x2F88F, - 0x2F890, - 0x2F891, - 0x2F893, - 0x2F894, - 0x2F896, - 0x2F897, - 0x2F898, - 0x2F899, - 0x2F89A, - 0x2F89B, - 0x2F89C, - 0x2F89D, - 0x2F89E, - 0x2F89F, - 0x2F8A0, - 0x2F8A1, - 0x2F8A2, - 0x2F8A3, - 0x2F8A4, - 0x2F8A5, - 0x2F8A6, - 0x2F8A7, - 0x2F8A8, - 0x2F8A9, - 0x2F8AA, - 0x2F8AB, - 0x2F8AC, - 0x2F8AD, - 0x2F8AE, - 0x2F8AF, - 0x2F8B0, - 0x2F8B1, - 0x2F8B2, - 0x2F8B3, - 0x2F8B4, - 0x2F8B5, - 0x2F8B6, - 0x2F8B7, - 0x2F8B8, - 0x2F8B9, - 0x2F8BA, - 0x2F8BB, - 0x2F8BC, - 0x2F8BD, - 0x2F8BE, - 0x2F8BF, - 0x2F8C0, - 0x2F8C1, - 0x2F8C2, - 0x2F8C3, - 0x2F8C4, - 0x2F8C5, - 0x2F8C6, - 0x2F8C7, - 0x2F8C8, - 0x2F8C9, - 0x2F8CA, - 0x2F8CB, - 0x2F8CC, - 0x2F8CD, - 0x2F8CE, - 0x2F8CF, - 0x2F8D0, - 0x2F8D1, - 0x2F8D2, - 0x2F8D3, - 0x2F8D4, - 0x2F8D5, - 0x2F8D6, - 0x2F8D7, - 0x2F8D8, - 0x2F8D9, - 0x2F8DA, - 0x2F8DB, - 0x2F8DC, - 0x2F8DD, - 0x2F8DE, - 0x2F8DF, - 0x2F8E0, - 0x2F8E1, - 0x2F8E2, - 0x2F8E3, - 0x2F8E4, - 0x2F8E5, - 0x2F8E6, - 0x2F8E7, - 0x2F8E8, - 0x2F8E9, - 0x2F8EA, - 0x2F8EB, - 0x2F8EC, - 0x2F8ED, - 0x2F8EE, - 0x2F8EF, - 0x2F8F0, - 0x2F8F1, - 0x2F8F2, - 0x2F8F3, - 0x2F8F4, - 0x2F8F5, - 0x2F8F6, - 0x2F8F7, - 0x2F8F8, - 0x2F8F9, - 0x2F8FA, - 0x2F8FB, - 0x2F8FC, - 0x2F8FD, - 0x2F8FE, - 0x2F8FF, - 0x2F900, - 0x2F901, - 0x2F902, - 0x2F903, - 0x2F904, - 0x2F905, - 0x2F906, - 0x2F907, - 0x2F908, - 0x2F909, - 0x2F90A, - 0x2F90B, - 0x2F90C, - 0x2F90D, - 0x2F90E, - 0x2F90F, - 0x2F910, - 0x2F911, - 0x2F912, - 0x2F913, - 0x2F914, - 0x2F915, - 0x2F916, - 0x2F917, - 0x2F918, - 0x2F919, - 0x2F91A, - 0x2F91B, - 0x2F91C, - 0x2F91D, - 0x2F91E, - 0x2F91F, - 0x2F920, - 0x2F921, - 0x2F922, - 0x2F923, - 0x2F924, - 0x2F925, - 0x2F926, - 0x2F927, - 0x2F928, - 0x2F929, - 0x2F92A, - 0x2F92B, - 0x2F92C, - 0x2F92E, - 0x2F92F, - 0x2F930, - 0x2F931, - 0x2F932, - 0x2F933, - 0x2F934, - 0x2F935, - 0x2F936, - 0x2F937, - 0x2F938, - 0x2F939, - 0x2F93A, - 0x2F93B, - 0x2F93C, - 0x2F93D, - 0x2F93E, - 0x2F93F, - 0x2F940, - 0x2F941, - 0x2F942, - 0x2F943, - 0x2F944, - 0x2F945, - 0x2F946, - 0x2F948, - 0x2F949, - 0x2F94A, - 0x2F94B, - 0x2F94C, - 0x2F94D, - 0x2F94E, - 0x2F94F, - 0x2F950, - 0x2F951, - 0x2F952, - 0x2F953, - 0x2F954, - 0x2F955, - 0x2F956, - 0x2F957, - 0x2F958, - 0x2F959, - 0x2F95A, - 0x2F95B, - 0x2F95C, - 0x2F95D, - 0x2F95F, - 0x2F960, - 0x2F961, - 0x2F962, - 0x2F963, - 0x2F964, - 0x2F965, - 0x2F966, - 0x2F967, - 0x2F968, - 0x2F969, - 0x2F96A, - 0x2F96B, - 0x2F96C, - 0x2F96D, - 0x2F96E, - 0x2F96F, - 0x2F970, - 0x2F971, - 0x2F972, - 0x2F973, - 0x2F974, - 0x2F975, - 0x2F976, - 0x2F977, - 0x2F978, - 0x2F979, - 0x2F97A, - 0x2F97B, - 0x2F97C, - 0x2F97D, - 0x2F97E, - 0x2F97F, - 0x2F980, - 0x2F981, - 0x2F982, - 0x2F983, - 0x2F984, - 0x2F985, - 0x2F986, - 0x2F987, - 0x2F988, - 0x2F989, - 0x2F98A, - 0x2F98B, - 0x2F98C, - 0x2F98D, - 0x2F98E, - 0x2F98F, - 0x2F990, - 0x2F991, - 0x2F992, - 0x2F993, - 0x2F994, - 0x2F995, - 0x2F996, - 0x2F997, - 0x2F998, - 0x2F999, - 0x2F99A, - 0x2F99B, - 0x2F99C, - 0x2F99D, - 0x2F99E, - 0x2F99F, - 0x2F9A0, - 0x2F9A1, - 0x2F9A2, - 0x2F9A3, - 0x2F9A4, - 0x2F9A5, - 0x2F9A6, - 0x2F9A7, - 0x2F9A8, - 0x2F9A9, - 0x2F9AA, - 0x2F9AB, - 0x2F9AC, - 0x2F9AD, - 0x2F9AE, - 0x2F9AF, - 0x2F9B0, - 0x2F9B1, - 0x2F9B2, - 0x2F9B3, - 0x2F9B4, - 0x2F9B5, - 0x2F9B6, - 0x2F9B7, - 0x2F9B8, - 0x2F9B9, - 0x2F9BA, - 0x2F9BB, - 0x2F9BC, - 0x2F9BD, - 0x2F9BE, - 0x2F9BF, - 0x2F9C0, - 0x2F9C1, - 0x2F9C2, - 0x2F9C3, - 0x2F9C4, - 0x2F9C5, - 0x2F9C6, - 0x2F9C7, - 0x2F9C8, - 0x2F9C9, - 0x2F9CA, - 0x2F9CB, - 0x2F9CC, - 0x2F9CD, - 0x2F9CE, - 0x2F9CF, - 0x2F9D0, - 0x2F9D1, - 0x2F9D2, - 0x2F9D3, - 0x2F9D4, - 0x2F9D5, - 0x2F9D6, - 0x2F9D7, - 0x2F9D8, - 0x2F9D9, - 0x2F9DA, - 0x2F9DB, - 0x2F9DC, - 0x2F9DD, - 0x2F9DE, - 0x2F9DF, - 0x2F9E0, - 0x2F9E1, - 0x2F9E2, - 0x2F9E3, - 0x2F9E4, - 0x2F9E5, - 0x2F9E6, - 0x2F9E7, - 0x2F9E8, - 0x2F9E9, - 0x2F9EA, - 0x2F9EB, - 0x2F9EC, - 0x2F9ED, - 0x2F9EE, - 0x2F9EF, - 0x2F9F0, - 0x2F9F1, - 0x2F9F2, - 0x2F9F3, - 0x2F9F4, - 0x2F9F5, - 0x2F9F6, - 0x2F9F7, - 0x2F9F8, - 0x2F9F9, - 0x2F9FA, - 0x2F9FB, - 0x2F9FC, - 0x2F9FD, - 0x2F9FE, - 0x2FA00, - 0x2FA01, - 0x2FA02, - 0x2FA03, - 0x2FA04, - 0x2FA05, - 0x2FA06, - 0x2FA07, - 0x2FA08, - 0x2FA09, - 0x2FA0A, - 0x2FA0B, - 0x2FA0C, - 0x2FA0D, - 0x2FA0E, - 0x2FA0F, - 0x2FA10, - 0x2FA11, - 0x2FA12, - 0x2FA13, - 0x2FA14, - 0x2FA15, - 0x2FA16, - 0x2FA17, - 0x2FA18, - 0x2FA19, - 0x2FA1A, - 0x2FA1B, - 0x2FA1C, - 0x2FA1D, - 0x2FA1E, - 0x30000, - 0x3134B, - 0x31350, - 0x3347A, - 0xE0100, - 0xE01F0, - ), -) - -uts46_statuses: bytes = ( - b"VVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" - b"VMMMMMMMMMMMMMMMMMMMMMMMMMMVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" - b"XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXMVVVVVVVMVMVVIVMVVMMMMVVMMMVMMMV" - b"MMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMDVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVVV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMVMMV" - b"MVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVMVMVMM" - b"VMVMMVMMMVMMMMVMMVMMMVMMVMMVMVMVMMVMVMVMMVMMMVMVMMVMVMMMMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMMVMVMMMMVMVMVMVMVMMMMMMMMMVMMMMMM" - b"VMMMMMVMMVMMMVIVMVMVMVMVXMVMMXMMMMMMMXMXMMVMMMMMMMMMMMMMMMMMXMMM" - b"MMMMMMVDVMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMMMVMMVMVMMVMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMXVMVXVXVXVXVXVXVMMMMVXVXVXVXVXVXVXVXVXVX" - b"VXVXVMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXMMXMVXVXVXVXVXVXVXVMXVMXVXVXV" - b"XVXVXVXMMMVXMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" - b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" - b"XVMVXVXVXVXVXVXVXVMVXVXVXVXVXMMVXVMVMVXVMVMVMVMVMVXVMVMMMMMVMVMV" - b"XVMVMVMVMVMVXVXVXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMXMXVMV" - b"IVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMXVXVXVXVXVXVXVXVXVX" - b"VIVXVXVXVIVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMM" - b"MMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMVXVXVMMMV" - b"MMMMMMMMMMMVMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMMMMVXMMMMMMXVMMMMMMMMVMMMMMM" - b"MMVXMMMMMMXVXMXMXMXMVMMMMMMMMVMVMVMVMVMVMVMXMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMXVMMMMMMMMMMMMMMXVMMMMMMMMMVMXVM" - b"MMMXMMMVMVMMMMMMMMXMMMXVMMMMMMMMXMIDXVMVMVXVXMVMMVMMVMVMVMMMVMVM" - b"IXIMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMXVMVXVXMMMMVMMMVM" - b"MMMMMVMMVMMMVMMMVMVMVMVMMMMVMMMMMMMMMMVMMMMMVMMMMVMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMVXVMMVMMVMMVXVXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVXVMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMVMMMVMVMVMVMMMMVMVMVMM" - b"MMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVXVXVXVXVXMVXVXVXVXVX" - b"VXVXVXVXVXVXVXVMVMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMVMVMVMMMVXVXVMMVMVMXV" - b"XMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVMMMMMMMMMMMMMMVXVMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMMMMMM" - b"MMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXMVMVMVMVMVMVMVM" - b"VMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMVXV" - b"MVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMV" - b"MVMVMVMVMVMVMVMVMVMMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMVMMMMMVM" - b"MMMMVMVMVMVMVMVMVMVMMMMVMVMMVMVMVMVMVMVMVMVMXMMMMMVMMVXVXVXVXVXV" - b"XVXVXVXVXVXVXVXVXVXVXVXVXVXVXVMMMMVMVXMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVX" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMVMVMVMMMMMMMMMMVMVMVMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMXMMMMMMXMMMMMXMVMMMMMMMMMMMMMMMMMMMMMMMMXMMMMM" - b"XMXMMXMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXMMMMMMMMMMMMMVIMMXMMM" - b"MMMXVXMMMMMMMMMMMMMMMMMMMVMMMMMMXMMMMMMMMMMMMMMMMMMMXMMMMXMMMVMX" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXIXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMIMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMXMMMMM" - b"MXMMMMMMXMMMXMMMMMMMXMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXV" - b"XVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVMMMMMMMMMMMXMMMMMMMMMMMMMMMXMMM" - b"MMMMXMMXVXVXVXVXVXVXVXVXVMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMXMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMXVXVXVXVMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMVXMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVIXVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVX" - b"VXVXVXVXVXVXVMMMMMMMVIVMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMXMMXMXMMXMMMMXMMMMMMMMMMMMXMXMMMMMMMXMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMMMMXMMMMM" - b"MMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMXMMMMMXMXMMMMMMMXMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXVXVX" - b"VXVXVXVXVXVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVXVXVXVX" - b"VXMMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMXMMXMXMXMMMMMMMMMMXMMMMXMXMXMX" - b"MXMXMXMMMXMMXMXMXMXMXMXMXMMXMXMMMMXMMMMMMMXMMMMXMMMMXMXMMMMMMMMM" - b"MXMMMMMMMMMMMMMMMMMXMMMXMMMMMXMMMMMMMMMMMMMMMMMXVXVXVXVXVXVXVXMM" - b"MMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMVMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMVMMMVMVXVMMMXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMXMMMMMMMMMXMMXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVXVX" - b"VXVMMMMMMMMMMVXVXVXVXVXVXMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMM" - b"MMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMMXVXVXIX" -) - -uts46_replacements: tuple[Optional[str], ...] = ( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - " ", - None, - None, - None, - None, - None, - None, - None, - " ̈", - None, - "a", - None, - None, - None, - None, - " ̄", - None, - None, - "2", - "3", - " ́", - "μ", - None, - None, - " ̧", - "1", - "o", - None, - "1⁄4", - "1⁄2", - "3⁄4", - None, - "à", - "á", - "â", - "ã", - "ä", - "å", - "æ", - "ç", - "è", - "é", - "ê", - "ë", - "ì", - "í", - "î", - "ï", - "ð", - "ñ", - "ò", - "ó", - "ô", - "õ", - "ö", - None, - "ø", - "ù", - "ú", - "û", - "ü", - "ý", - "þ", - "ss", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ā", - None, - "ă", - None, - "ą", - None, - "ć", - None, - "ĉ", - None, - "ċ", - None, - "č", - None, - "ď", - None, - "đ", - None, - "ē", - None, - "ĕ", - None, - "ė", - None, - "ę", - None, - "ě", - None, - "ĝ", - None, - "ğ", - None, - "ġ", - None, - "ģ", - None, - "ĥ", - None, - "ħ", - None, - "ĩ", - None, - "ī", - None, - "ĭ", - None, - "į", - None, - "i̇", - None, - "ij", - "ĵ", - None, - "ķ", - None, - "ĺ", - None, - "ļ", - None, - "ľ", - None, - "l·", - "ł", - None, - "ń", - None, - "ņ", - None, - "ň", - None, - "ʼn", - "ŋ", - None, - "ō", - None, - "ŏ", - None, - "ő", - None, - "œ", - None, - "ŕ", - None, - "ŗ", - None, - "ř", - None, - "ś", - None, - "ŝ", - None, - "ş", - None, - "š", - None, - "ţ", - None, - "ť", - None, - "ŧ", - None, - "ũ", - None, - "ū", - None, - "ŭ", - None, - "ů", - None, - "ű", - None, - "ų", - None, - "ŵ", - None, - "ŷ", - None, - "ÿ", - "ź", - None, - "ż", - None, - "ž", - None, - "s", - None, - "ɓ", - "ƃ", - None, - "ƅ", - None, - "ɔ", - "ƈ", - None, - "ɖ", - "ɗ", - "ƌ", - None, - "ǝ", - "ə", - "ɛ", - "ƒ", - None, - "ɠ", - "ɣ", - None, - "ɩ", - "ɨ", - "ƙ", - None, - "ɯ", - "ɲ", - None, - "ɵ", - "ơ", - None, - "ƣ", - None, - "ƥ", - None, - "ʀ", - "ƨ", - None, - "ʃ", - None, - "ƭ", - None, - "ʈ", - "ư", - None, - "ʊ", - "ʋ", - "ƴ", - None, - "ƶ", - None, - "ʒ", - "ƹ", - None, - "ƽ", - None, - "dž", - "lj", - "nj", - "ǎ", - None, - "ǐ", - None, - "ǒ", - None, - "ǔ", - None, - "ǖ", - None, - "ǘ", - None, - "ǚ", - None, - "ǜ", - None, - "ǟ", - None, - "ǡ", - None, - "ǣ", - None, - "ǥ", - None, - "ǧ", - None, - "ǩ", - None, - "ǫ", - None, - "ǭ", - None, - "ǯ", - None, - "dz", - "ǵ", - None, - "ƕ", - "ƿ", - "ǹ", - None, - "ǻ", - None, - "ǽ", - None, - "ǿ", - None, - "ȁ", - None, - "ȃ", - None, - "ȅ", - None, - "ȇ", - None, - "ȉ", - None, - "ȋ", - None, - "ȍ", - None, - "ȏ", - None, - "ȑ", - None, - "ȓ", - None, - "ȕ", - None, - "ȗ", - None, - "ș", - None, - "ț", - None, - "ȝ", - None, - "ȟ", - None, - "ƞ", - None, - "ȣ", - None, - "ȥ", - None, - "ȧ", - None, - "ȩ", - None, - "ȫ", - None, - "ȭ", - None, - "ȯ", - None, - "ȱ", - None, - "ȳ", - None, - "ⱥ", - "ȼ", - None, - "ƚ", - "ⱦ", - None, - "ɂ", - None, - "ƀ", - "ʉ", - "ʌ", - "ɇ", - None, - "ɉ", - None, - "ɋ", - None, - "ɍ", - None, - "ɏ", - None, - "h", - "ɦ", - "j", - "r", - "ɹ", - "ɻ", - "ʁ", - "w", - "y", - None, - " ̆", - " ̇", - " ̊", - " ̨", - " ̃", - " ̋", - None, - "ɣ", - "l", - "s", - "x", - "ʕ", - None, - "̀", - "́", - None, - "̓", - "̈́", - "ι", - None, - None, - None, - "ͱ", - None, - "ͳ", - None, - "ʹ", - None, - "ͷ", - None, - None, - " ι", - None, - ";", - "ϳ", - None, - " ́", - " ̈́", - "ά", - "·", - "έ", - "ή", - "ί", - None, - "ό", - None, - "ύ", - "ώ", - None, - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - None, - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "ϊ", - "ϋ", - None, - "σ", - None, - "ϗ", - "β", - "θ", - "υ", - "ύ", - "ϋ", - "φ", - "π", - None, - "ϙ", - None, - "ϛ", - None, - "ϝ", - None, - "ϟ", - None, - "ϡ", - None, - "ϣ", - None, - "ϥ", - None, - "ϧ", - None, - "ϩ", - None, - "ϫ", - None, - "ϭ", - None, - "ϯ", - None, - "κ", - "ρ", - "σ", - None, - "θ", - "ε", - None, - "ϸ", - None, - "σ", - "ϻ", - None, - "ͻ", - "ͼ", - "ͽ", - "ѐ", - "ё", - "ђ", - "ѓ", - "є", - "ѕ", - "і", - "ї", - "ј", - "љ", - "њ", - "ћ", - "ќ", - "ѝ", - "ў", - "џ", - "а", - "б", - "в", - "г", - "д", - "е", - "ж", - "з", - "и", - "й", - "к", - "л", - "м", - "н", - "о", - "п", - "р", - "с", - "т", - "у", - "ф", - "х", - "ц", - "ч", - "ш", - "щ", - "ъ", - "ы", - "ь", - "э", - "ю", - "я", - None, - "ѡ", - None, - "ѣ", - None, - "ѥ", - None, - "ѧ", - None, - "ѩ", - None, - "ѫ", - None, - "ѭ", - None, - "ѯ", - None, - "ѱ", - None, - "ѳ", - None, - "ѵ", - None, - "ѷ", - None, - "ѹ", - None, - "ѻ", - None, - "ѽ", - None, - "ѿ", - None, - "ҁ", - None, - "ҋ", - None, - "ҍ", - None, - "ҏ", - None, - "ґ", - None, - "ғ", - None, - "ҕ", - None, - "җ", - None, - "ҙ", - None, - "қ", - None, - "ҝ", - None, - "ҟ", - None, - "ҡ", - None, - "ң", - None, - "ҥ", - None, - "ҧ", - None, - "ҩ", - None, - "ҫ", - None, - "ҭ", - None, - "ү", - None, - "ұ", - None, - "ҳ", - None, - "ҵ", - None, - "ҷ", - None, - "ҹ", - None, - "һ", - None, - "ҽ", - None, - "ҿ", - None, - "ӏ", - "ӂ", - None, - "ӄ", - None, - "ӆ", - None, - "ӈ", - None, - "ӊ", - None, - "ӌ", - None, - "ӎ", - None, - "ӑ", - None, - "ӓ", - None, - "ӕ", - None, - "ӗ", - None, - "ә", - None, - "ӛ", - None, - "ӝ", - None, - "ӟ", - None, - "ӡ", - None, - "ӣ", - None, - "ӥ", - None, - "ӧ", - None, - "ө", - None, - "ӫ", - None, - "ӭ", - None, - "ӯ", - None, - "ӱ", - None, - "ӳ", - None, - "ӵ", - None, - "ӷ", - None, - "ӹ", - None, - "ӻ", - None, - "ӽ", - None, - "ӿ", - None, - "ԁ", - None, - "ԃ", - None, - "ԅ", - None, - "ԇ", - None, - "ԉ", - None, - "ԋ", - None, - "ԍ", - None, - "ԏ", - None, - "ԑ", - None, - "ԓ", - None, - "ԕ", - None, - "ԗ", - None, - "ԙ", - None, - "ԛ", - None, - "ԝ", - None, - "ԟ", - None, - "ԡ", - None, - "ԣ", - None, - "ԥ", - None, - "ԧ", - None, - "ԩ", - None, - "ԫ", - None, - "ԭ", - None, - "ԯ", - None, - None, - "ա", - "բ", - "գ", - "դ", - "ե", - "զ", - "է", - "ը", - "թ", - "ժ", - "ի", - "լ", - "խ", - "ծ", - "կ", - "հ", - "ձ", - "ղ", - "ճ", - "մ", - "յ", - "ն", - "շ", - "ո", - "չ", - "պ", - "ջ", - "ռ", - "ս", - "վ", - "տ", - "ր", - "ց", - "ւ", - "փ", - "ք", - "օ", - "ֆ", - None, - None, - "եւ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "اٴ", - "وٴ", - "ۇٴ", - "يٴ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "क़", - "ख़", - "ग़", - "ज़", - "ड़", - "ढ़", - "फ़", - "य़", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ড়", - "ঢ়", - None, - "য়", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ਲ਼", - None, - None, - "ਸ਼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ਖ਼", - "ਗ਼", - "ਜ਼", - None, - None, - "ਫ਼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ଡ଼", - "ଢ଼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ํา", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ໍາ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ຫນ", - "ຫມ", - None, - None, - None, - "་", - None, - "གྷ", - None, - None, - None, - "ཌྷ", - None, - "དྷ", - None, - "བྷ", - None, - "ཛྷ", - None, - "ཀྵ", - None, - None, - None, - "ཱི", - None, - "ཱུ", - "ྲྀ", - "ྲཱྀ", - "ླྀ", - "ླཱྀ", - None, - "ཱྀ", - None, - "ྒྷ", - None, - None, - None, - "ྜྷ", - None, - "ྡྷ", - None, - "ྦྷ", - None, - "ྫྷ", - None, - "ྐྵ", - None, - None, - None, - None, - None, - None, - None, - "ⴀ", - "ⴁ", - "ⴂ", - "ⴃ", - "ⴄ", - "ⴅ", - "ⴆ", - "ⴇ", - "ⴈ", - "ⴉ", - "ⴊ", - "ⴋ", - "ⴌ", - "ⴍ", - "ⴎ", - "ⴏ", - "ⴐ", - "ⴑ", - "ⴒ", - "ⴓ", - "ⴔ", - "ⴕ", - "ⴖ", - "ⴗ", - "ⴘ", - "ⴙ", - "ⴚ", - "ⴛ", - "ⴜ", - "ⴝ", - "ⴞ", - "ⴟ", - "ⴠ", - "ⴡ", - "ⴢ", - "ⴣ", - "ⴤ", - "ⴥ", - None, - "ⴧ", - None, - "ⴭ", - None, - None, - "ნ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "Ᏸ", - "Ᏹ", - "Ᏺ", - "Ᏻ", - "Ᏼ", - "Ᏽ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "в", - "д", - "о", - "с", - "т", - "ъ", - "ѣ", - "ꙋ", - "\u1c8a", - None, - None, - "ა", - "ბ", - "გ", - "დ", - "ე", - "ვ", - "ზ", - "თ", - "ი", - "კ", - "ლ", - "მ", - "ნ", - "ო", - "პ", - "ჟ", - "რ", - "ს", - "ტ", - "უ", - "ფ", - "ქ", - "ღ", - "ყ", - "შ", - "ჩ", - "ც", - "ძ", - "წ", - "ჭ", - "ხ", - "ჯ", - "ჰ", - "ჱ", - "ჲ", - "ჳ", - "ჴ", - "ჵ", - "ჶ", - "ჷ", - "ჸ", - "ჹ", - "ჺ", - None, - "ჽ", - "ჾ", - "ჿ", - None, - None, - None, - None, - None, - "a", - "æ", - "b", - None, - "d", - "e", - "ǝ", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - None, - "o", - "ȣ", - "p", - "r", - "t", - "u", - "w", - "a", - "ɐ", - "ɑ", - "ᴂ", - "b", - "d", - "e", - "ə", - "ɛ", - "ɜ", - "g", - None, - "k", - "m", - "ŋ", - "o", - "ɔ", - "ᴖ", - "ᴗ", - "p", - "t", - "u", - "ᴝ", - "ɯ", - "v", - "ᴥ", - "β", - "γ", - "δ", - "φ", - "χ", - "i", - "r", - "u", - "v", - "β", - "γ", - "ρ", - "φ", - "χ", - None, - "н", - None, - "ɒ", - "c", - "ɕ", - "ð", - "ɜ", - "f", - "ɟ", - "ɡ", - "ɥ", - "ɨ", - "ɩ", - "ɪ", - "ᵻ", - "ʝ", - "ɭ", - "ᶅ", - "ʟ", - "ɱ", - "ɰ", - "ɲ", - "ɳ", - "ɴ", - "ɵ", - "ɸ", - "ʂ", - "ʃ", - "ƫ", - "ʉ", - "ʊ", - "ᴜ", - "ʋ", - "ʌ", - "z", - "ʐ", - "ʑ", - "ʒ", - "θ", - None, - "ḁ", - None, - "ḃ", - None, - "ḅ", - None, - "ḇ", - None, - "ḉ", - None, - "ḋ", - None, - "ḍ", - None, - "ḏ", - None, - "ḑ", - None, - "ḓ", - None, - "ḕ", - None, - "ḗ", - None, - "ḙ", - None, - "ḛ", - None, - "ḝ", - None, - "ḟ", - None, - "ḡ", - None, - "ḣ", - None, - "ḥ", - None, - "ḧ", - None, - "ḩ", - None, - "ḫ", - None, - "ḭ", - None, - "ḯ", - None, - "ḱ", - None, - "ḳ", - None, - "ḵ", - None, - "ḷ", - None, - "ḹ", - None, - "ḻ", - None, - "ḽ", - None, - "ḿ", - None, - "ṁ", - None, - "ṃ", - None, - "ṅ", - None, - "ṇ", - None, - "ṉ", - None, - "ṋ", - None, - "ṍ", - None, - "ṏ", - None, - "ṑ", - None, - "ṓ", - None, - "ṕ", - None, - "ṗ", - None, - "ṙ", - None, - "ṛ", - None, - "ṝ", - None, - "ṟ", - None, - "ṡ", - None, - "ṣ", - None, - "ṥ", - None, - "ṧ", - None, - "ṩ", - None, - "ṫ", - None, - "ṭ", - None, - "ṯ", - None, - "ṱ", - None, - "ṳ", - None, - "ṵ", - None, - "ṷ", - None, - "ṹ", - None, - "ṻ", - None, - "ṽ", - None, - "ṿ", - None, - "ẁ", - None, - "ẃ", - None, - "ẅ", - None, - "ẇ", - None, - "ẉ", - None, - "ẋ", - None, - "ẍ", - None, - "ẏ", - None, - "ẑ", - None, - "ẓ", - None, - "ẕ", - None, - "aʾ", - "ṡ", - None, - "ß", - None, - "ạ", - None, - "ả", - None, - "ấ", - None, - "ầ", - None, - "ẩ", - None, - "ẫ", - None, - "ậ", - None, - "ắ", - None, - "ằ", - None, - "ẳ", - None, - "ẵ", - None, - "ặ", - None, - "ẹ", - None, - "ẻ", - None, - "ẽ", - None, - "ế", - None, - "ề", - None, - "ể", - None, - "ễ", - None, - "ệ", - None, - "ỉ", - None, - "ị", - None, - "ọ", - None, - "ỏ", - None, - "ố", - None, - "ồ", - None, - "ổ", - None, - "ỗ", - None, - "ộ", - None, - "ớ", - None, - "ờ", - None, - "ở", - None, - "ỡ", - None, - "ợ", - None, - "ụ", - None, - "ủ", - None, - "ứ", - None, - "ừ", - None, - "ử", - None, - "ữ", - None, - "ự", - None, - "ỳ", - None, - "ỵ", - None, - "ỷ", - None, - "ỹ", - None, - "ỻ", - None, - "ỽ", - None, - "ỿ", - None, - "ἀ", - "ἁ", - "ἂ", - "ἃ", - "ἄ", - "ἅ", - "ἆ", - "ἇ", - None, - None, - "ἐ", - "ἑ", - "ἒ", - "ἓ", - "ἔ", - "ἕ", - None, - None, - "ἠ", - "ἡ", - "ἢ", - "ἣ", - "ἤ", - "ἥ", - "ἦ", - "ἧ", - None, - "ἰ", - "ἱ", - "ἲ", - "ἳ", - "ἴ", - "ἵ", - "ἶ", - "ἷ", - None, - None, - "ὀ", - "ὁ", - "ὂ", - "ὃ", - "ὄ", - "ὅ", - None, - None, - None, - "ὑ", - None, - "ὓ", - None, - "ὕ", - None, - "ὗ", - None, - "ὠ", - "ὡ", - "ὢ", - "ὣ", - "ὤ", - "ὥ", - "ὦ", - "ὧ", - None, - "ά", - None, - "έ", - None, - "ή", - None, - "ί", - None, - "ό", - None, - "ύ", - None, - "ώ", - None, - "ἀι", - "ἁι", - "ἂι", - "ἃι", - "ἄι", - "ἅι", - "ἆι", - "ἇι", - "ἀι", - "ἁι", - "ἂι", - "ἃι", - "ἄι", - "ἅι", - "ἆι", - "ἇι", - "ἠι", - "ἡι", - "ἢι", - "ἣι", - "ἤι", - "ἥι", - "ἦι", - "ἧι", - "ἠι", - "ἡι", - "ἢι", - "ἣι", - "ἤι", - "ἥι", - "ἦι", - "ἧι", - "ὠι", - "ὡι", - "ὢι", - "ὣι", - "ὤι", - "ὥι", - "ὦι", - "ὧι", - "ὠι", - "ὡι", - "ὢι", - "ὣι", - "ὤι", - "ὥι", - "ὦι", - "ὧι", - None, - "ὰι", - "αι", - "άι", - None, - None, - "ᾶι", - "ᾰ", - "ᾱ", - "ὰ", - "ά", - "αι", - " ̓", - "ι", - " ̓", - " ͂", - " ̈͂", - "ὴι", - "ηι", - "ήι", - None, - None, - "ῆι", - "ὲ", - "έ", - "ὴ", - "ή", - "ηι", - " ̓̀", - " ̓́", - " ̓͂", - None, - "ΐ", - None, - None, - "ῐ", - "ῑ", - "ὶ", - "ί", - None, - " ̔̀", - " ̔́", - " ̔͂", - None, - "ΰ", - None, - "ῠ", - "ῡ", - "ὺ", - "ύ", - "ῥ", - " ̈̀", - " ̈́", - "`", - None, - "ὼι", - "ωι", - "ώι", - None, - None, - "ῶι", - "ὸ", - "ό", - "ὼ", - "ώ", - "ωι", - " ́", - " ̔", - None, - " ", - None, - "", - None, - None, - "‐", - None, - " ̳", - None, - None, - None, - None, - " ", - None, - "′′", - "′′′", - None, - "‵‵", - "‵‵‵", - None, - "!!", - None, - " ̅", - None, - "??", - "?!", - "!?", - None, - "′′′′", - None, - " ", - None, - None, - None, - "0", - "i", - None, - "4", - "5", - "6", - "7", - "8", - "9", - "+", - "−", - "=", - "(", - ")", - "n", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "+", - "−", - "=", - "(", - ")", - None, - "a", - "e", - "o", - "x", - "ə", - "h", - "k", - "l", - "m", - "n", - "p", - "s", - "t", - None, - None, - "rs", - None, - None, - None, - None, - "a/c", - "a/s", - "c", - "°c", - None, - "c/o", - "c/u", - "ɛ", - None, - "°f", - "g", - "h", - "ħ", - "i", - "l", - None, - "n", - "no", - None, - "p", - "q", - "r", - None, - "sm", - "tel", - "tm", - None, - "z", - None, - "ω", - None, - "z", - None, - "k", - "å", - "b", - "c", - None, - "e", - "f", - "ⅎ", - "m", - "o", - "א", - "ב", - "ג", - "ד", - "i", - None, - "fax", - "π", - "γ", - "π", - "∑", - None, - "d", - "e", - "i", - "j", - None, - "1⁄7", - "1⁄9", - "1⁄10", - "1⁄3", - "2⁄3", - "1⁄5", - "2⁄5", - "3⁄5", - "4⁄5", - "1⁄6", - "5⁄6", - "1⁄8", - "3⁄8", - "5⁄8", - "7⁄8", - "1⁄", - "i", - "ii", - "iii", - "iv", - "v", - "vi", - "vii", - "viii", - "ix", - "x", - "xi", - "xii", - "l", - "c", - "d", - "m", - "i", - "ii", - "iii", - "iv", - "v", - "vi", - "vii", - "viii", - "ix", - "x", - "xi", - "xii", - "l", - "c", - "d", - "m", - None, - "ↄ", - None, - "0⁄3", - None, - None, - None, - "∫∫", - "∫∫∫", - None, - "∮∮", - "∮∮∮", - None, - "〈", - "〉", - None, - None, - None, - None, - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "10", - "11", - "12", - "13", - "14", - "15", - "16", - "17", - "18", - "19", - "20", - "(1)", - "(2)", - "(3)", - "(4)", - "(5)", - "(6)", - "(7)", - "(8)", - "(9)", - "(10)", - "(11)", - "(12)", - "(13)", - "(14)", - "(15)", - "(16)", - "(17)", - "(18)", - "(19)", - "(20)", - None, - "(a)", - "(b)", - "(c)", - "(d)", - "(e)", - "(f)", - "(g)", - "(h)", - "(i)", - "(j)", - "(k)", - "(l)", - "(m)", - "(n)", - "(o)", - "(p)", - "(q)", - "(r)", - "(s)", - "(t)", - "(u)", - "(v)", - "(w)", - "(x)", - "(y)", - "(z)", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "0", - None, - "∫∫∫∫", - None, - "::=", - "==", - "===", - None, - "⫝̸", - None, - None, - None, - "ⰰ", - "ⰱ", - "ⰲ", - "ⰳ", - "ⰴ", - "ⰵ", - "ⰶ", - "ⰷ", - "ⰸ", - "ⰹ", - "ⰺ", - "ⰻ", - "ⰼ", - "ⰽ", - "ⰾ", - "ⰿ", - "ⱀ", - "ⱁ", - "ⱂ", - "ⱃ", - "ⱄ", - "ⱅ", - "ⱆ", - "ⱇ", - "ⱈ", - "ⱉ", - "ⱊ", - "ⱋ", - "ⱌ", - "ⱍ", - "ⱎ", - "ⱏ", - "ⱐ", - "ⱑ", - "ⱒ", - "ⱓ", - "ⱔ", - "ⱕ", - "ⱖ", - "ⱗ", - "ⱘ", - "ⱙ", - "ⱚ", - "ⱛ", - "ⱜ", - "ⱝ", - "ⱞ", - "ⱟ", - None, - "ⱡ", - None, - "ɫ", - "ᵽ", - "ɽ", - None, - "ⱨ", - None, - "ⱪ", - None, - "ⱬ", - None, - "ɑ", - "ɱ", - "ɐ", - "ɒ", - None, - "ⱳ", - None, - "ⱶ", - None, - "j", - "v", - "ȿ", - "ɀ", - "ⲁ", - None, - "ⲃ", - None, - "ⲅ", - None, - "ⲇ", - None, - "ⲉ", - None, - "ⲋ", - None, - "ⲍ", - None, - "ⲏ", - None, - "ⲑ", - None, - "ⲓ", - None, - "ⲕ", - None, - "ⲗ", - None, - "ⲙ", - None, - "ⲛ", - None, - "ⲝ", - None, - "ⲟ", - None, - "ⲡ", - None, - "ⲣ", - None, - "ⲥ", - None, - "ⲧ", - None, - "ⲩ", - None, - "ⲫ", - None, - "ⲭ", - None, - "ⲯ", - None, - "ⲱ", - None, - "ⲳ", - None, - "ⲵ", - None, - "ⲷ", - None, - "ⲹ", - None, - "ⲻ", - None, - "ⲽ", - None, - "ⲿ", - None, - "ⳁ", - None, - "ⳃ", - None, - "ⳅ", - None, - "ⳇ", - None, - "ⳉ", - None, - "ⳋ", - None, - "ⳍ", - None, - "ⳏ", - None, - "ⳑ", - None, - "ⳓ", - None, - "ⳕ", - None, - "ⳗ", - None, - "ⳙ", - None, - "ⳛ", - None, - "ⳝ", - None, - "ⳟ", - None, - "ⳡ", - None, - "ⳣ", - None, - "ⳬ", - None, - "ⳮ", - None, - "ⳳ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ⵡ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "母", - None, - "龟", - None, - "一", - "丨", - "丶", - "丿", - "乙", - "亅", - "二", - "亠", - "人", - "儿", - "入", - "八", - "冂", - "冖", - "冫", - "几", - "凵", - "刀", - "力", - "勹", - "匕", - "匚", - "匸", - "十", - "卜", - "卩", - "厂", - "厶", - "又", - "口", - "囗", - "土", - "士", - "夂", - "夊", - "夕", - "大", - "女", - "子", - "宀", - "寸", - "小", - "尢", - "尸", - "屮", - "山", - "巛", - "工", - "己", - "巾", - "干", - "幺", - "广", - "廴", - "廾", - "弋", - "弓", - "彐", - "彡", - "彳", - "心", - "戈", - "戶", - "手", - "支", - "攴", - "文", - "斗", - "斤", - "方", - "无", - "日", - "曰", - "月", - "木", - "欠", - "止", - "歹", - "殳", - "毋", - "比", - "毛", - "氏", - "气", - "水", - "火", - "爪", - "父", - "爻", - "爿", - "片", - "牙", - "牛", - "犬", - "玄", - "玉", - "瓜", - "瓦", - "甘", - "生", - "用", - "田", - "疋", - "疒", - "癶", - "白", - "皮", - "皿", - "目", - "矛", - "矢", - "石", - "示", - "禸", - "禾", - "穴", - "立", - "竹", - "米", - "糸", - "缶", - "网", - "羊", - "羽", - "老", - "而", - "耒", - "耳", - "聿", - "肉", - "臣", - "自", - "至", - "臼", - "舌", - "舛", - "舟", - "艮", - "色", - "艸", - "虍", - "虫", - "血", - "行", - "衣", - "襾", - "見", - "角", - "言", - "谷", - "豆", - "豕", - "豸", - "貝", - "赤", - "走", - "足", - "身", - "車", - "辛", - "辰", - "辵", - "邑", - "酉", - "釆", - "里", - "金", - "長", - "門", - "阜", - "隶", - "隹", - "雨", - "靑", - "非", - "面", - "革", - "韋", - "韭", - "音", - "頁", - "風", - "飛", - "食", - "首", - "香", - "馬", - "骨", - "高", - "髟", - "鬥", - "鬯", - "鬲", - "鬼", - "魚", - "鳥", - "鹵", - "鹿", - "麥", - "麻", - "黃", - "黍", - "黑", - "黹", - "黽", - "鼎", - "鼓", - "鼠", - "鼻", - "齊", - "齒", - "龍", - "龜", - "龠", - None, - " ", - None, - ".", - None, - "〒", - None, - "十", - "卄", - "卅", - None, - None, - None, - None, - None, - " ゙", - " ゚", - None, - "より", - None, - "コト", - None, - None, - None, - "ᄀ", - "ᄁ", - "ᆪ", - "ᄂ", - "ᆬ", - "ᆭ", - "ᄃ", - "ᄄ", - "ᄅ", - "ᆰ", - "ᆱ", - "ᆲ", - "ᆳ", - "ᆴ", - "ᆵ", - "ᄚ", - "ᄆ", - "ᄇ", - "ᄈ", - "ᄡ", - "ᄉ", - "ᄊ", - "ᄋ", - "ᄌ", - "ᄍ", - "ᄎ", - "ᄏ", - "ᄐ", - "ᄑ", - "ᄒ", - "ᅡ", - "ᅢ", - "ᅣ", - "ᅤ", - "ᅥ", - "ᅦ", - "ᅧ", - "ᅨ", - "ᅩ", - "ᅪ", - "ᅫ", - "ᅬ", - "ᅭ", - "ᅮ", - "ᅯ", - "ᅰ", - "ᅱ", - "ᅲ", - "ᅳ", - "ᅴ", - "ᅵ", - None, - "ᄔ", - "ᄕ", - "ᇇ", - "ᇈ", - "ᇌ", - "ᇎ", - "ᇓ", - "ᇗ", - "ᇙ", - "ᄜ", - "ᇝ", - "ᇟ", - "ᄝ", - "ᄞ", - "ᄠ", - "ᄢ", - "ᄣ", - "ᄧ", - "ᄩ", - "ᄫ", - "ᄬ", - "ᄭ", - "ᄮ", - "ᄯ", - "ᄲ", - "ᄶ", - "ᅀ", - "ᅇ", - "ᅌ", - "ᇱ", - "ᇲ", - "ᅗ", - "ᅘ", - "ᅙ", - "ᆄ", - "ᆅ", - "ᆈ", - "ᆑ", - "ᆒ", - "ᆔ", - "ᆞ", - "ᆡ", - None, - None, - "一", - "二", - "三", - "四", - "上", - "中", - "下", - "甲", - "乙", - "丙", - "丁", - "天", - "地", - "人", - None, - None, - None, - "(ᄀ)", - "(ᄂ)", - "(ᄃ)", - "(ᄅ)", - "(ᄆ)", - "(ᄇ)", - "(ᄉ)", - "(ᄋ)", - "(ᄌ)", - "(ᄎ)", - "(ᄏ)", - "(ᄐ)", - "(ᄑ)", - "(ᄒ)", - "(가)", - "(나)", - "(다)", - "(라)", - "(마)", - "(바)", - "(사)", - "(아)", - "(자)", - "(차)", - "(카)", - "(타)", - "(파)", - "(하)", - "(주)", - "(오전)", - "(오후)", - None, - "(一)", - "(二)", - "(三)", - "(四)", - "(五)", - "(六)", - "(七)", - "(八)", - "(九)", - "(十)", - "(月)", - "(火)", - "(水)", - "(木)", - "(金)", - "(土)", - "(日)", - "(株)", - "(有)", - "(社)", - "(名)", - "(特)", - "(財)", - "(祝)", - "(労)", - "(代)", - "(呼)", - "(学)", - "(監)", - "(企)", - "(資)", - "(協)", - "(祭)", - "(休)", - "(自)", - "(至)", - "問", - "幼", - "文", - "箏", - None, - "pte", - "21", - "22", - "23", - "24", - "25", - "26", - "27", - "28", - "29", - "30", - "31", - "32", - "33", - "34", - "35", - "ᄀ", - "ᄂ", - "ᄃ", - "ᄅ", - "ᄆ", - "ᄇ", - "ᄉ", - "ᄋ", - "ᄌ", - "ᄎ", - "ᄏ", - "ᄐ", - "ᄑ", - "ᄒ", - "가", - "나", - "다", - "라", - "마", - "바", - "사", - "아", - "자", - "차", - "카", - "타", - "파", - "하", - "참고", - "주의", - "우", - None, - "一", - "二", - "三", - "四", - "五", - "六", - "七", - "八", - "九", - "十", - "月", - "火", - "水", - "木", - "金", - "土", - "日", - "株", - "有", - "社", - "名", - "特", - "財", - "祝", - "労", - "秘", - "男", - "女", - "適", - "優", - "印", - "注", - "項", - "休", - "写", - "正", - "上", - "中", - "下", - "左", - "右", - "医", - "宗", - "学", - "監", - "企", - "資", - "協", - "夜", - "36", - "37", - "38", - "39", - "40", - "41", - "42", - "43", - "44", - "45", - "46", - "47", - "48", - "49", - "50", - "1月", - "2月", - "3月", - "4月", - "5月", - "6月", - "7月", - "8月", - "9月", - "10月", - "11月", - "12月", - "hg", - "erg", - "ev", - "ltd", - "ア", - "イ", - "ウ", - "エ", - "オ", - "カ", - "キ", - "ク", - "ケ", - "コ", - "サ", - "シ", - "ス", - "セ", - "ソ", - "タ", - "チ", - "ツ", - "テ", - "ト", - "ナ", - "ニ", - "ヌ", - "ネ", - "ノ", - "ハ", - "ヒ", - "フ", - "ヘ", - "ホ", - "マ", - "ミ", - "ム", - "メ", - "モ", - "ヤ", - "ユ", - "ヨ", - "ラ", - "リ", - "ル", - "レ", - "ロ", - "ワ", - "ヰ", - "ヱ", - "ヲ", - "令和", - "アパート", - "アルファ", - "アンペア", - "アール", - "イニング", - "インチ", - "ウォン", - "エスクード", - "エーカー", - "オンス", - "オーム", - "カイリ", - "カラット", - "カロリー", - "ガロン", - "ガンマ", - "ギガ", - "ギニー", - "キュリー", - "ギルダー", - "キロ", - "キログラム", - "キロメートル", - "キロワット", - "グラム", - "グラムトン", - "クルゼイロ", - "クローネ", - "ケース", - "コルナ", - "コーポ", - "サイクル", - "サンチーム", - "シリング", - "センチ", - "セント", - "ダース", - "デシ", - "ドル", - "トン", - "ナノ", - "ノット", - "ハイツ", - "パーセント", - "パーツ", - "バーレル", - "ピアストル", - "ピクル", - "ピコ", - "ビル", - "ファラッド", - "フィート", - "ブッシェル", - "フラン", - "ヘクタール", - "ペソ", - "ペニヒ", - "ヘルツ", - "ペンス", - "ページ", - "ベータ", - "ポイント", - "ボルト", - "ホン", - "ポンド", - "ホール", - "ホーン", - "マイクロ", - "マイル", - "マッハ", - "マルク", - "マンション", - "ミクロン", - "ミリ", - "ミリバール", - "メガ", - "メガトン", - "メートル", - "ヤード", - "ヤール", - "ユアン", - "リットル", - "リラ", - "ルピー", - "ルーブル", - "レム", - "レントゲン", - "ワット", - "0点", - "1点", - "2点", - "3点", - "4点", - "5点", - "6点", - "7点", - "8点", - "9点", - "10点", - "11点", - "12点", - "13点", - "14点", - "15点", - "16点", - "17点", - "18点", - "19点", - "20点", - "21点", - "22点", - "23点", - "24点", - "hpa", - "da", - "au", - "bar", - "ov", - "pc", - "dm", - "dm2", - "dm3", - "iu", - "平成", - "昭和", - "大正", - "明治", - "株式会社", - "pa", - "na", - "μa", - "ma", - "ka", - "kb", - "mb", - "gb", - "cal", - "kcal", - "pf", - "nf", - "μf", - "μg", - "mg", - "kg", - "hz", - "khz", - "mhz", - "ghz", - "thz", - "μl", - "ml", - "dl", - "kl", - "fm", - "nm", - "μm", - "mm", - "cm", - "km", - "mm2", - "cm2", - "m2", - "km2", - "mm3", - "cm3", - "m3", - "km3", - "m∕s", - "m∕s2", - "pa", - "kpa", - "mpa", - "gpa", - "rad", - "rad∕s", - "rad∕s2", - "ps", - "ns", - "μs", - "ms", - "pv", - "nv", - "μv", - "mv", - "kv", - "mv", - "pw", - "nw", - "μw", - "mw", - "kw", - "mw", - "kω", - "mω", - None, - "bq", - "cc", - "cd", - "c∕kg", - None, - "db", - "gy", - "ha", - "hp", - "in", - "kk", - "km", - "kt", - "lm", - "ln", - "log", - "lx", - "mb", - "mil", - "mol", - "ph", - None, - "ppm", - "pr", - "sr", - "sv", - "wb", - "v∕m", - "a∕m", - "1日", - "2日", - "3日", - "4日", - "5日", - "6日", - "7日", - "8日", - "9日", - "10日", - "11日", - "12日", - "13日", - "14日", - "15日", - "16日", - "17日", - "18日", - "19日", - "20日", - "21日", - "22日", - "23日", - "24日", - "25日", - "26日", - "27日", - "28日", - "29日", - "30日", - "31日", - "gal", - None, - None, - None, - None, - None, - None, - "ꙁ", - None, - "ꙃ", - None, - "ꙅ", - None, - "ꙇ", - None, - "ꙉ", - None, - "ꙋ", - None, - "ꙍ", - None, - "ꙏ", - None, - "ꙑ", - None, - "ꙓ", - None, - "ꙕ", - None, - "ꙗ", - None, - "ꙙ", - None, - "ꙛ", - None, - "ꙝ", - None, - "ꙟ", - None, - "ꙡ", - None, - "ꙣ", - None, - "ꙥ", - None, - "ꙧ", - None, - "ꙩ", - None, - "ꙫ", - None, - "ꙭ", - None, - "ꚁ", - None, - "ꚃ", - None, - "ꚅ", - None, - "ꚇ", - None, - "ꚉ", - None, - "ꚋ", - None, - "ꚍ", - None, - "ꚏ", - None, - "ꚑ", - None, - "ꚓ", - None, - "ꚕ", - None, - "ꚗ", - None, - "ꚙ", - None, - "ꚛ", - None, - "ъ", - "ь", - None, - None, - None, - "ꜣ", - None, - "ꜥ", - None, - "ꜧ", - None, - "ꜩ", - None, - "ꜫ", - None, - "ꜭ", - None, - "ꜯ", - None, - "ꜳ", - None, - "ꜵ", - None, - "ꜷ", - None, - "ꜹ", - None, - "ꜻ", - None, - "ꜽ", - None, - "ꜿ", - None, - "ꝁ", - None, - "ꝃ", - None, - "ꝅ", - None, - "ꝇ", - None, - "ꝉ", - None, - "ꝋ", - None, - "ꝍ", - None, - "ꝏ", - None, - "ꝑ", - None, - "ꝓ", - None, - "ꝕ", - None, - "ꝗ", - None, - "ꝙ", - None, - "ꝛ", - None, - "ꝝ", - None, - "ꝟ", - None, - "ꝡ", - None, - "ꝣ", - None, - "ꝥ", - None, - "ꝧ", - None, - "ꝩ", - None, - "ꝫ", - None, - "ꝭ", - None, - "ꝯ", - None, - "ꝯ", - None, - "ꝺ", - None, - "ꝼ", - None, - "ᵹ", - "ꝿ", - None, - "ꞁ", - None, - "ꞃ", - None, - "ꞅ", - None, - "ꞇ", - None, - "ꞌ", - None, - "ɥ", - None, - "ꞑ", - None, - "ꞓ", - None, - "ꞗ", - None, - "ꞙ", - None, - "ꞛ", - None, - "ꞝ", - None, - "ꞟ", - None, - "ꞡ", - None, - "ꞣ", - None, - "ꞥ", - None, - "ꞧ", - None, - "ꞩ", - None, - "ɦ", - "ɜ", - "ɡ", - "ɬ", - "ɪ", - None, - "ʞ", - "ʇ", - "ʝ", - "ꭓ", - "ꞵ", - None, - "ꞷ", - None, - "ꞹ", - None, - "ꞻ", - None, - "ꞽ", - None, - "ꞿ", - None, - "ꟁ", - None, - "ꟃ", - None, - "ꞔ", - "ʂ", - "ᶎ", - "ꟈ", - None, - "ꟊ", - None, - "ɤ", - "\ua7cd", - None, - "\ua7cf", - None, - "ꟑ", - None, - "ꟓ", - None, - "ꟕ", - None, - "ꟗ", - None, - "ꟙ", - None, - "\ua7db", - None, - "ƛ", - None, - "s", - "c", - "f", - "q", - "ꟶ", - None, - "ħ", - "œ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ꜧ", - "ꬷ", - "ɫ", - "ꭒ", - None, - "ʍ", - None, - None, - "Ꭰ", - "Ꭱ", - "Ꭲ", - "Ꭳ", - "Ꭴ", - "Ꭵ", - "Ꭶ", - "Ꭷ", - "Ꭸ", - "Ꭹ", - "Ꭺ", - "Ꭻ", - "Ꭼ", - "Ꭽ", - "Ꭾ", - "Ꭿ", - "Ꮀ", - "Ꮁ", - "Ꮂ", - "Ꮃ", - "Ꮄ", - "Ꮅ", - "Ꮆ", - "Ꮇ", - "Ꮈ", - "Ꮉ", - "Ꮊ", - "Ꮋ", - "Ꮌ", - "Ꮍ", - "Ꮎ", - "Ꮏ", - "Ꮐ", - "Ꮑ", - "Ꮒ", - "Ꮓ", - "Ꮔ", - "Ꮕ", - "Ꮖ", - "Ꮗ", - "Ꮘ", - "Ꮙ", - "Ꮚ", - "Ꮛ", - "Ꮜ", - "Ꮝ", - "Ꮞ", - "Ꮟ", - "Ꮠ", - "Ꮡ", - "Ꮢ", - "Ꮣ", - "Ꮤ", - "Ꮥ", - "Ꮦ", - "Ꮧ", - "Ꮨ", - "Ꮩ", - "Ꮪ", - "Ꮫ", - "Ꮬ", - "Ꮭ", - "Ꮮ", - "Ꮯ", - "Ꮰ", - "Ꮱ", - "Ꮲ", - "Ꮳ", - "Ꮴ", - "Ꮵ", - "Ꮶ", - "Ꮷ", - "Ꮸ", - "Ꮹ", - "Ꮺ", - "Ꮻ", - "Ꮼ", - "Ꮽ", - "Ꮾ", - "Ꮿ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "豈", - "更", - "車", - "賈", - "滑", - "串", - "句", - "龜", - "契", - "金", - "喇", - "奈", - "懶", - "癩", - "羅", - "蘿", - "螺", - "裸", - "邏", - "樂", - "洛", - "烙", - "珞", - "落", - "酪", - "駱", - "亂", - "卵", - "欄", - "爛", - "蘭", - "鸞", - "嵐", - "濫", - "藍", - "襤", - "拉", - "臘", - "蠟", - "廊", - "朗", - "浪", - "狼", - "郎", - "來", - "冷", - "勞", - "擄", - "櫓", - "爐", - "盧", - "老", - "蘆", - "虜", - "路", - "露", - "魯", - "鷺", - "碌", - "祿", - "綠", - "菉", - "錄", - "鹿", - "論", - "壟", - "弄", - "籠", - "聾", - "牢", - "磊", - "賂", - "雷", - "壘", - "屢", - "樓", - "淚", - "漏", - "累", - "縷", - "陋", - "勒", - "肋", - "凜", - "凌", - "稜", - "綾", - "菱", - "陵", - "讀", - "拏", - "樂", - "諾", - "丹", - "寧", - "怒", - "率", - "異", - "北", - "磻", - "便", - "復", - "不", - "泌", - "數", - "索", - "參", - "塞", - "省", - "葉", - "說", - "殺", - "辰", - "沈", - "拾", - "若", - "掠", - "略", - "亮", - "兩", - "凉", - "梁", - "糧", - "良", - "諒", - "量", - "勵", - "呂", - "女", - "廬", - "旅", - "濾", - "礪", - "閭", - "驪", - "麗", - "黎", - "力", - "曆", - "歷", - "轢", - "年", - "憐", - "戀", - "撚", - "漣", - "煉", - "璉", - "秊", - "練", - "聯", - "輦", - "蓮", - "連", - "鍊", - "列", - "劣", - "咽", - "烈", - "裂", - "說", - "廉", - "念", - "捻", - "殮", - "簾", - "獵", - "令", - "囹", - "寧", - "嶺", - "怜", - "玲", - "瑩", - "羚", - "聆", - "鈴", - "零", - "靈", - "領", - "例", - "禮", - "醴", - "隸", - "惡", - "了", - "僚", - "寮", - "尿", - "料", - "樂", - "燎", - "療", - "蓼", - "遼", - "龍", - "暈", - "阮", - "劉", - "杻", - "柳", - "流", - "溜", - "琉", - "留", - "硫", - "紐", - "類", - "六", - "戮", - "陸", - "倫", - "崙", - "淪", - "輪", - "律", - "慄", - "栗", - "率", - "隆", - "利", - "吏", - "履", - "易", - "李", - "梨", - "泥", - "理", - "痢", - "罹", - "裏", - "裡", - "里", - "離", - "匿", - "溺", - "吝", - "燐", - "璘", - "藺", - "隣", - "鱗", - "麟", - "林", - "淋", - "臨", - "立", - "笠", - "粒", - "狀", - "炙", - "識", - "什", - "茶", - "刺", - "切", - "度", - "拓", - "糖", - "宅", - "洞", - "暴", - "輻", - "行", - "降", - "見", - "廓", - "兀", - "嗀", - None, - "塚", - None, - "晴", - None, - "凞", - "猪", - "益", - "礼", - "神", - "祥", - "福", - "靖", - "精", - "羽", - None, - "蘒", - None, - "諸", - None, - "逸", - "都", - None, - "飯", - "飼", - "館", - "鶴", - "郞", - "隷", - "侮", - "僧", - "免", - "勉", - "勤", - "卑", - "喝", - "嘆", - "器", - "塀", - "墨", - "層", - "屮", - "悔", - "慨", - "憎", - "懲", - "敏", - "既", - "暑", - "梅", - "海", - "渚", - "漢", - "煮", - "爫", - "琢", - "碑", - "社", - "祉", - "祈", - "祐", - "祖", - "祝", - "禍", - "禎", - "穀", - "突", - "節", - "練", - "縉", - "繁", - "署", - "者", - "臭", - "艹", - "著", - "褐", - "視", - "謁", - "謹", - "賓", - "贈", - "辶", - "逸", - "難", - "響", - "頻", - "恵", - "𤋮", - "舘", - None, - "並", - "况", - "全", - "侀", - "充", - "冀", - "勇", - "勺", - "喝", - "啕", - "喙", - "嗢", - "塚", - "墳", - "奄", - "奔", - "婢", - "嬨", - "廒", - "廙", - "彩", - "徭", - "惘", - "慎", - "愈", - "憎", - "慠", - "懲", - "戴", - "揄", - "搜", - "摒", - "敖", - "晴", - "朗", - "望", - "杖", - "歹", - "殺", - "流", - "滛", - "滋", - "漢", - "瀞", - "煮", - "瞧", - "爵", - "犯", - "猪", - "瑱", - "甆", - "画", - "瘝", - "瘟", - "益", - "盛", - "直", - "睊", - "着", - "磌", - "窱", - "節", - "类", - "絛", - "練", - "缾", - "者", - "荒", - "華", - "蝹", - "襁", - "覆", - "視", - "調", - "諸", - "請", - "謁", - "諾", - "諭", - "謹", - "變", - "贈", - "輸", - "遲", - "醙", - "鉶", - "陼", - "難", - "靖", - "韛", - "響", - "頋", - "頻", - "鬒", - "龜", - "𢡊", - "𢡄", - "𣏕", - "㮝", - "䀘", - "䀹", - "𥉉", - "𥳐", - "𧻓", - "齃", - "龎", - None, - "ff", - "fi", - "fl", - "ffi", - "ffl", - "st", - None, - "մն", - "մե", - "մի", - "վն", - "մխ", - None, - "יִ", - None, - "ײַ", - "ע", - "א", - "ד", - "ה", - "כ", - "ל", - "ם", - "ר", - "ת", - "+", - "שׁ", - "שׂ", - "שּׁ", - "שּׂ", - "אַ", - "אָ", - "אּ", - "בּ", - "גּ", - "דּ", - "הּ", - "וּ", - "זּ", - None, - "טּ", - "יּ", - "ךּ", - "כּ", - "לּ", - None, - "מּ", - None, - "נּ", - "סּ", - None, - "ףּ", - "פּ", - None, - "צּ", - "קּ", - "רּ", - "שּ", - "תּ", - "וֹ", - "בֿ", - "כֿ", - "פֿ", - "אל", - "ٱ", - "ٻ", - "پ", - "ڀ", - "ٺ", - "ٿ", - "ٹ", - "ڤ", - "ڦ", - "ڄ", - "ڃ", - "چ", - "ڇ", - "ڍ", - "ڌ", - "ڎ", - "ڈ", - "ژ", - "ڑ", - "ک", - "گ", - "ڳ", - "ڱ", - "ں", - "ڻ", - "ۀ", - "ہ", - "ھ", - "ے", - "ۓ", - None, - "ڭ", - "ۇ", - "ۆ", - "ۈ", - "ۇٴ", - "ۋ", - "ۅ", - "ۉ", - "ې", - "ى", - "ئا", - "ئە", - "ئو", - "ئۇ", - "ئۆ", - "ئۈ", - "ئې", - "ئى", - "ی", - "ئج", - "ئح", - "ئم", - "ئى", - "ئي", - "بج", - "بح", - "بخ", - "بم", - "بى", - "بي", - "تج", - "تح", - "تخ", - "تم", - "تى", - "تي", - "ثج", - "ثم", - "ثى", - "ثي", - "جح", - "جم", - "حج", - "حم", - "خج", - "خح", - "خم", - "سج", - "سح", - "سخ", - "سم", - "صح", - "صم", - "ضج", - "ضح", - "ضخ", - "ضم", - "طح", - "طم", - "ظم", - "عج", - "عم", - "غج", - "غم", - "فج", - "فح", - "فخ", - "فم", - "فى", - "في", - "قح", - "قم", - "قى", - "قي", - "كا", - "كج", - "كح", - "كخ", - "كل", - "كم", - "كى", - "كي", - "لج", - "لح", - "لخ", - "لم", - "لى", - "لي", - "مج", - "مح", - "مخ", - "مم", - "مى", - "مي", - "نج", - "نح", - "نخ", - "نم", - "نى", - "ني", - "هج", - "هم", - "هى", - "هي", - "يج", - "يح", - "يخ", - "يم", - "يى", - "يي", - "ذٰ", - "رٰ", - "ىٰ", - " ٌّ", - " ٍّ", - " َّ", - " ُّ", - " ِّ", - " ّٰ", - "ئر", - "ئز", - "ئم", - "ئن", - "ئى", - "ئي", - "بر", - "بز", - "بم", - "بن", - "بى", - "بي", - "تر", - "تز", - "تم", - "تن", - "تى", - "تي", - "ثر", - "ثز", - "ثم", - "ثن", - "ثى", - "ثي", - "فى", - "في", - "قى", - "قي", - "كا", - "كل", - "كم", - "كى", - "كي", - "لم", - "لى", - "لي", - "ما", - "مم", - "نر", - "نز", - "نم", - "نن", - "نى", - "ني", - "ىٰ", - "ير", - "يز", - "يم", - "ين", - "يى", - "يي", - "ئج", - "ئح", - "ئخ", - "ئم", - "ئه", - "بج", - "بح", - "بخ", - "بم", - "به", - "تج", - "تح", - "تخ", - "تم", - "ته", - "ثم", - "جح", - "جم", - "حج", - "حم", - "خج", - "خم", - "سج", - "سح", - "سخ", - "سم", - "صح", - "صخ", - "صم", - "ضج", - "ضح", - "ضخ", - "ضم", - "طح", - "ظم", - "عج", - "عم", - "غج", - "غم", - "فج", - "فح", - "فخ", - "فم", - "قح", - "قم", - "كج", - "كح", - "كخ", - "كل", - "كم", - "لج", - "لح", - "لخ", - "لم", - "له", - "مج", - "مح", - "مخ", - "مم", - "نج", - "نح", - "نخ", - "نم", - "نه", - "هج", - "هم", - "هٰ", - "يج", - "يح", - "يخ", - "يم", - "يه", - "ئم", - "ئه", - "بم", - "به", - "تم", - "ته", - "ثم", - "ثه", - "سم", - "سه", - "شم", - "شه", - "كل", - "كم", - "لم", - "نم", - "نه", - "يم", - "يه", - "ـَّ", - "ـُّ", - "ـِّ", - "طى", - "طي", - "عى", - "عي", - "غى", - "غي", - "سى", - "سي", - "شى", - "شي", - "حى", - "حي", - "جى", - "جي", - "خى", - "خي", - "صى", - "صي", - "ضى", - "ضي", - "شج", - "شح", - "شخ", - "شم", - "شر", - "سر", - "صر", - "ضر", - "طى", - "طي", - "عى", - "عي", - "غى", - "غي", - "سى", - "سي", - "شى", - "شي", - "حى", - "حي", - "جى", - "جي", - "خى", - "خي", - "صى", - "صي", - "ضى", - "ضي", - "شج", - "شح", - "شخ", - "شم", - "شر", - "سر", - "صر", - "ضر", - "شج", - "شح", - "شخ", - "شم", - "سه", - "شه", - "طم", - "سج", - "سح", - "سخ", - "شج", - "شح", - "شخ", - "طم", - "ظم", - "اً", - None, - "تجم", - "تحج", - "تحم", - "تخم", - "تمج", - "تمح", - "تمخ", - "جمح", - "حمي", - "حمى", - "سحج", - "سجح", - "سجى", - "سمح", - "سمج", - "سمم", - "صحح", - "صمم", - "شحم", - "شجي", - "شمخ", - "شمم", - "ضحى", - "ضخم", - "طمح", - "طمم", - "طمي", - "عجم", - "عمم", - "عمى", - "غمم", - "غمي", - "غمى", - "فخم", - "قمح", - "قمم", - "لحم", - "لحي", - "لحى", - "لجج", - "لخم", - "لمح", - "محج", - "محم", - "محي", - "مجح", - "مجم", - "مخج", - "مخم", - None, - "مجخ", - "همج", - "همم", - "نحم", - "نحى", - "نجم", - "نجى", - "نمي", - "نمى", - "يمم", - "بخي", - "تجي", - "تجى", - "تخي", - "تخى", - "تمي", - "تمى", - "جمي", - "جحى", - "جمى", - "سخى", - "صحي", - "شحي", - "ضحي", - "لجي", - "لمي", - "يحي", - "يجي", - "يمي", - "ممي", - "قمي", - "نحي", - "قمح", - "لحم", - "عمي", - "كمي", - "نجح", - "مخي", - "لجم", - "كمم", - "لجم", - "نجح", - "جحي", - "حجي", - "مجي", - "فمي", - "بحي", - "كمم", - "عجم", - "صمم", - "سخي", - "نجي", - None, - None, - "صلے", - "قلے", - "الله", - "اكبر", - "محمد", - "صلعم", - "رسول", - "عليه", - "وسلم", - "صلى", - "صلى الله عليه وسلم", - "جل جلاله", - "ریال", - None, - None, - ",", - "、", - None, - ":", - ";", - "!", - "?", - "〖", - "〗", - None, - None, - None, - "—", - "–", - "_", - "(", - ")", - "{", - "}", - "〔", - "〕", - "【", - "】", - "《", - "》", - "〈", - "〉", - "「", - "」", - "『", - "』", - None, - "[", - "]", - " ̅", - "_", - ",", - "、", - None, - ";", - ":", - "?", - "!", - "—", - "(", - ")", - "{", - "}", - "〔", - "〕", - "#", - "&", - "*", - "+", - "-", - "<", - ">", - "=", - None, - "\\", - "$", - "%", - "@", - None, - " ً", - "ـً", - " ٌ", - None, - " ٍ", - None, - " َ", - "ـَ", - " ُ", - "ـُ", - " ِ", - "ـِ", - " ّ", - "ـّ", - " ْ", - "ـْ", - "ء", - "آ", - "أ", - "ؤ", - "إ", - "ئ", - "ا", - "ب", - "ة", - "ت", - "ث", - "ج", - "ح", - "خ", - "د", - "ذ", - "ر", - "ز", - "س", - "ش", - "ص", - "ض", - "ط", - "ظ", - "ع", - "غ", - "ف", - "ق", - "ك", - "ل", - "م", - "ن", - "ه", - "و", - "ى", - "ي", - "لآ", - "لأ", - "لإ", - "لا", - None, - None, - None, - "!", - '"', - "#", - "$", - "%", - "&", - "'", - "(", - ")", - "*", - "+", - ",", - "-", - ".", - "/", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - ":", - ";", - "<", - "=", - ">", - "?", - "@", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "[", - "\\", - "]", - "^", - "_", - "`", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "{", - "|", - "}", - "~", - "⦅", - "⦆", - ".", - "「", - "」", - "、", - "・", - "ヲ", - "ァ", - "ィ", - "ゥ", - "ェ", - "ォ", - "ャ", - "ュ", - "ョ", - "ッ", - "ー", - "ア", - "イ", - "ウ", - "エ", - "オ", - "カ", - "キ", - "ク", - "ケ", - "コ", - "サ", - "シ", - "ス", - "セ", - "ソ", - "タ", - "チ", - "ツ", - "テ", - "ト", - "ナ", - "ニ", - "ヌ", - "ネ", - "ノ", - "ハ", - "ヒ", - "フ", - "ヘ", - "ホ", - "マ", - "ミ", - "ム", - "メ", - "モ", - "ヤ", - "ユ", - "ヨ", - "ラ", - "リ", - "ル", - "レ", - "ロ", - "ワ", - "ン", - "゙", - "゚", - None, - "ᄀ", - "ᄁ", - "ᆪ", - "ᄂ", - "ᆬ", - "ᆭ", - "ᄃ", - "ᄄ", - "ᄅ", - "ᆰ", - "ᆱ", - "ᆲ", - "ᆳ", - "ᆴ", - "ᆵ", - "ᄚ", - "ᄆ", - "ᄇ", - "ᄈ", - "ᄡ", - "ᄉ", - "ᄊ", - "ᄋ", - "ᄌ", - "ᄍ", - "ᄎ", - "ᄏ", - "ᄐ", - "ᄑ", - "ᄒ", - None, - "ᅡ", - "ᅢ", - "ᅣ", - "ᅤ", - "ᅥ", - "ᅦ", - None, - "ᅧ", - "ᅨ", - "ᅩ", - "ᅪ", - "ᅫ", - "ᅬ", - None, - "ᅭ", - "ᅮ", - "ᅯ", - "ᅰ", - "ᅱ", - "ᅲ", - None, - "ᅳ", - "ᅴ", - "ᅵ", - None, - "¢", - "£", - "¬", - " ̄", - "¦", - "¥", - "₩", - None, - "│", - "←", - "↑", - "→", - "↓", - "■", - "○", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𐐨", - "𐐩", - "𐐪", - "𐐫", - "𐐬", - "𐐭", - "𐐮", - "𐐯", - "𐐰", - "𐐱", - "𐐲", - "𐐳", - "𐐴", - "𐐵", - "𐐶", - "𐐷", - "𐐸", - "𐐹", - "𐐺", - "𐐻", - "𐐼", - "𐐽", - "𐐾", - "𐐿", - "𐑀", - "𐑁", - "𐑂", - "𐑃", - "𐑄", - "𐑅", - "𐑆", - "𐑇", - "𐑈", - "𐑉", - "𐑊", - "𐑋", - "𐑌", - "𐑍", - "𐑎", - "𐑏", - None, - None, - None, - None, - "𐓘", - "𐓙", - "𐓚", - "𐓛", - "𐓜", - "𐓝", - "𐓞", - "𐓟", - "𐓠", - "𐓡", - "𐓢", - "𐓣", - "𐓤", - "𐓥", - "𐓦", - "𐓧", - "𐓨", - "𐓩", - "𐓪", - "𐓫", - "𐓬", - "𐓭", - "𐓮", - "𐓯", - "𐓰", - "𐓱", - "𐓲", - "𐓳", - "𐓴", - "𐓵", - "𐓶", - "𐓷", - "𐓸", - "𐓹", - "𐓺", - "𐓻", - None, - None, - None, - None, - None, - None, - None, - None, - "𐖗", - "𐖘", - "𐖙", - "𐖚", - "𐖛", - "𐖜", - "𐖝", - "𐖞", - "𐖟", - "𐖠", - "𐖡", - None, - "𐖣", - "𐖤", - "𐖥", - "𐖦", - "𐖧", - "𐖨", - "𐖩", - "𐖪", - "𐖫", - "𐖬", - "𐖭", - "𐖮", - "𐖯", - "𐖰", - "𐖱", - None, - "𐖳", - "𐖴", - "𐖵", - "𐖶", - "𐖷", - "𐖸", - "𐖹", - None, - "𐖻", - "𐖼", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ː", - "ˑ", - "æ", - "ʙ", - "ɓ", - None, - "ʣ", - "ꭦ", - "ʥ", - "ʤ", - "ɖ", - "ɗ", - "ᶑ", - "ɘ", - "ɞ", - "ʩ", - "ɤ", - "ɢ", - "ɠ", - "ʛ", - "ħ", - "ʜ", - "ɧ", - "ʄ", - "ʪ", - "ʫ", - "ɬ", - "𝼄", - "ꞎ", - "ɮ", - "𝼅", - "ʎ", - "𝼆", - "ø", - "ɶ", - "ɷ", - "q", - "ɺ", - "𝼈", - "ɽ", - "ɾ", - "ʀ", - "ʨ", - "ʦ", - "ꭧ", - "ʧ", - "ʈ", - "ⱱ", - None, - "ʏ", - "ʡ", - "ʢ", - "ʘ", - "ǀ", - "ǁ", - "ǂ", - "𝼊", - "𝼞", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𐳀", - "𐳁", - "𐳂", - "𐳃", - "𐳄", - "𐳅", - "𐳆", - "𐳇", - "𐳈", - "𐳉", - "𐳊", - "𐳋", - "𐳌", - "𐳍", - "𐳎", - "𐳏", - "𐳐", - "𐳑", - "𐳒", - "𐳓", - "𐳔", - "𐳕", - "𐳖", - "𐳗", - "𐳘", - "𐳙", - "𐳚", - "𐳛", - "𐳜", - "𐳝", - "𐳞", - "𐳟", - "𐳠", - "𐳡", - "𐳢", - "𐳣", - "𐳤", - "𐳥", - "𐳦", - "𐳧", - "𐳨", - "𐳩", - "𐳪", - "𐳫", - "𐳬", - "𐳭", - "𐳮", - "𐳯", - "𐳰", - "𐳱", - "𐳲", - None, - None, - None, - None, - None, - None, - None, - None, - "\U00010d70", - "\U00010d71", - "\U00010d72", - "\U00010d73", - "\U00010d74", - "\U00010d75", - "\U00010d76", - "\U00010d77", - "\U00010d78", - "\U00010d79", - "\U00010d7a", - "\U00010d7b", - "\U00010d7c", - "\U00010d7d", - "\U00010d7e", - "\U00010d7f", - "\U00010d80", - "\U00010d81", - "\U00010d82", - "\U00010d83", - "\U00010d84", - "\U00010d85", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𑣀", - "𑣁", - "𑣂", - "𑣃", - "𑣄", - "𑣅", - "𑣆", - "𑣇", - "𑣈", - "𑣉", - "𑣊", - "𑣋", - "𑣌", - "𑣍", - "𑣎", - "𑣏", - "𑣐", - "𑣑", - "𑣒", - "𑣓", - "𑣔", - "𑣕", - "𑣖", - "𑣗", - "𑣘", - "𑣙", - "𑣚", - "𑣛", - "𑣜", - "𑣝", - "𑣞", - "𑣟", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𖹠", - "𖹡", - "𖹢", - "𖹣", - "𖹤", - "𖹥", - "𖹦", - "𖹧", - "𖹨", - "𖹩", - "𖹪", - "𖹫", - "𖹬", - "𖹭", - "𖹮", - "𖹯", - "𖹰", - "𖹱", - "𖹲", - "𖹳", - "𖹴", - "𖹵", - "𖹶", - "𖹷", - "𖹸", - "𖹹", - "𖹺", - "𖹻", - "𖹼", - "𖹽", - "𖹾", - "𖹿", - None, - None, - "\U00016ebb", - "\U00016ebc", - "\U00016ebd", - "\U00016ebe", - "\U00016ebf", - "\U00016ec0", - "\U00016ec1", - "\U00016ec2", - "\U00016ec3", - "\U00016ec4", - "\U00016ec5", - "\U00016ec6", - "\U00016ec7", - "\U00016ec8", - "\U00016ec9", - "\U00016eca", - "\U00016ecb", - "\U00016ecc", - "\U00016ecd", - "\U00016ece", - "\U00016ecf", - "\U00016ed0", - "\U00016ed1", - "\U00016ed2", - "\U00016ed3", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𝅗𝅥", - "𝅘𝅥", - "𝅘𝅥𝅮", - "𝅘𝅥𝅯", - "𝅘𝅥𝅰", - "𝅘𝅥𝅱", - "𝅘𝅥𝅲", - None, - None, - None, - "𝆹𝅥", - "𝆺𝅥", - "𝆹𝅥𝅮", - "𝆺𝅥𝅮", - "𝆹𝅥𝅯", - "𝆺𝅥𝅯", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - None, - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - None, - "c", - "d", - None, - "g", - None, - "j", - "k", - None, - "n", - "o", - "p", - "q", - None, - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - None, - "f", - None, - "h", - "i", - "j", - "k", - "l", - "m", - "n", - None, - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - None, - "d", - "e", - "f", - "g", - None, - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - None, - "s", - "t", - "u", - "v", - "w", - "x", - "y", - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - None, - "d", - "e", - "f", - "g", - None, - "i", - "j", - "k", - "l", - "m", - None, - "o", - None, - "s", - "t", - "u", - "v", - "w", - "x", - "y", - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "ı", - "ȷ", - None, - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "θ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∇", - "α", - "β", - "γ", - "δ", - "ε", - "ζ", - "η", - "θ", - "ι", - "κ", - "λ", - "μ", - "ν", - "ξ", - "ο", - "π", - "ρ", - "σ", - "τ", - "υ", - "φ", - "χ", - "ψ", - "ω", - "∂", - "ε", - "θ", - "κ", - "φ", - "ρ", - "π", - "ϝ", - None, - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "а", - "б", - "в", - "г", - "д", - "е", - "ж", - "з", - "и", - "к", - "л", - "м", - "о", - "п", - "р", - "с", - "т", - "у", - "ф", - "х", - "ц", - "ч", - "ш", - "ы", - "э", - "ю", - "ꚉ", - "ә", - "і", - "ј", - "ө", - "ү", - "ӏ", - "а", - "б", - "в", - "г", - "д", - "е", - "ж", - "з", - "и", - "к", - "л", - "о", - "п", - "с", - "у", - "ф", - "х", - "ц", - "ч", - "ш", - "ъ", - "ы", - "ґ", - "і", - "ѕ", - "џ", - "ҫ", - "ꙑ", - "ұ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "𞤢", - "𞤣", - "𞤤", - "𞤥", - "𞤦", - "𞤧", - "𞤨", - "𞤩", - "𞤪", - "𞤫", - "𞤬", - "𞤭", - "𞤮", - "𞤯", - "𞤰", - "𞤱", - "𞤲", - "𞤳", - "𞤴", - "𞤵", - "𞤶", - "𞤷", - "𞤸", - "𞤹", - "𞤺", - "𞤻", - "𞤼", - "𞤽", - "𞤾", - "𞤿", - "𞥀", - "𞥁", - "𞥂", - "𞥃", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "ا", - "ب", - "ج", - "د", - None, - "و", - "ز", - "ح", - "ط", - "ي", - "ك", - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - "ر", - "ش", - "ت", - "ث", - "خ", - "ذ", - "ض", - "ظ", - "غ", - "ٮ", - "ں", - "ڡ", - "ٯ", - None, - "ب", - "ج", - None, - "ه", - None, - "ح", - None, - "ي", - "ك", - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - None, - "ش", - "ت", - "ث", - "خ", - None, - "ض", - None, - "غ", - None, - "ج", - None, - "ح", - None, - "ي", - None, - "ل", - None, - "ن", - "س", - "ع", - None, - "ص", - "ق", - None, - "ش", - None, - "خ", - None, - "ض", - None, - "غ", - None, - "ں", - None, - "ٯ", - None, - "ب", - "ج", - None, - "ه", - None, - "ح", - "ط", - "ي", - "ك", - None, - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - None, - "ش", - "ت", - "ث", - "خ", - None, - "ض", - "ظ", - "غ", - "ٮ", - None, - "ڡ", - None, - "ا", - "ب", - "ج", - "د", - "ه", - "و", - "ز", - "ح", - "ط", - "ي", - None, - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - "ر", - "ش", - "ت", - "ث", - "خ", - "ذ", - "ض", - "ظ", - "غ", - None, - "ب", - "ج", - "د", - None, - "و", - "ز", - "ح", - "ط", - "ي", - None, - "ل", - "م", - "ن", - "س", - "ع", - "ف", - "ص", - "ق", - "ر", - "ش", - "ت", - "ث", - "خ", - "ذ", - "ض", - "ظ", - "غ", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "0,", - "1,", - "2,", - "3,", - "4,", - "5,", - "6,", - "7,", - "8,", - "9,", - None, - "(a)", - "(b)", - "(c)", - "(d)", - "(e)", - "(f)", - "(g)", - "(h)", - "(i)", - "(j)", - "(k)", - "(l)", - "(m)", - "(n)", - "(o)", - "(p)", - "(q)", - "(r)", - "(s)", - "(t)", - "(u)", - "(v)", - "(w)", - "(x)", - "(y)", - "(z)", - "〔s〕", - "c", - "r", - "cd", - "wz", - None, - "a", - "b", - "c", - "d", - "e", - "f", - "g", - "h", - "i", - "j", - "k", - "l", - "m", - "n", - "o", - "p", - "q", - "r", - "s", - "t", - "u", - "v", - "w", - "x", - "y", - "z", - "hv", - "mv", - "sd", - "ss", - "ppv", - "wc", - None, - "mc", - "md", - "mr", - None, - "dj", - None, - None, - None, - "ほか", - "ココ", - "サ", - None, - "手", - "字", - "双", - "デ", - "二", - "多", - "解", - "天", - "交", - "映", - "無", - "料", - "前", - "後", - "再", - "新", - "初", - "終", - "生", - "販", - "声", - "吹", - "演", - "投", - "捕", - "一", - "三", - "遊", - "左", - "中", - "右", - "指", - "走", - "打", - "禁", - "空", - "合", - "満", - "有", - "月", - "申", - "割", - "営", - "配", - None, - "〔本〕", - "〔三〕", - "〔二〕", - "〔安〕", - "〔点〕", - "〔打〕", - "〔盗〕", - "〔勝〕", - "〔敗〕", - None, - "得", - "可", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "0", - "1", - "2", - "3", - "4", - "5", - "6", - "7", - "8", - "9", - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - "丽", - "丸", - "乁", - "𠄢", - "你", - "侮", - "侻", - "倂", - "偺", - "備", - "僧", - "像", - "㒞", - "𠘺", - "免", - "兔", - "兤", - "具", - "𠔜", - "㒹", - "內", - "再", - "𠕋", - "冗", - "冤", - "仌", - "冬", - "况", - "𩇟", - "凵", - "刃", - "㓟", - "刻", - "剆", - "割", - "剷", - "㔕", - "勇", - "勉", - "勤", - "勺", - "包", - "匆", - "北", - "卉", - "卑", - "博", - "即", - "卽", - "卿", - "𠨬", - "灰", - "及", - "叟", - "𠭣", - "叫", - "叱", - "吆", - "咞", - "吸", - "呈", - "周", - "咢", - "哶", - "唐", - "啓", - "啣", - "善", - "喙", - "喫", - "喳", - "嗂", - "圖", - "嘆", - "圗", - "噑", - "噴", - "切", - "壮", - "城", - "埴", - "堍", - "型", - "堲", - "報", - "墬", - "𡓤", - "売", - "壷", - "夆", - "多", - "夢", - "奢", - "𡚨", - "𡛪", - "姬", - "娛", - "娧", - "姘", - "婦", - "㛮", - "㛼", - "嬈", - "嬾", - "𡧈", - "寃", - "寘", - "寧", - "寳", - "𡬘", - "寿", - "将", - "当", - "尢", - "㞁", - "屠", - "屮", - "峀", - "岍", - "𡷤", - "嵃", - "𡷦", - "嵮", - "嵫", - "嵼", - "巡", - "巢", - "㠯", - "巽", - "帨", - "帽", - "幩", - "㡢", - "𢆃", - "㡼", - "庰", - "庳", - "庶", - "廊", - "𪎒", - "廾", - "𢌱", - "舁", - "弢", - "㣇", - "𣊸", - "𦇚", - "形", - "彫", - "㣣", - "徚", - "忍", - "志", - "忹", - "悁", - "㤺", - "㤜", - "悔", - "𢛔", - "惇", - "慈", - "慌", - "慎", - "慌", - "慺", - "憎", - "憲", - "憤", - "憯", - "懞", - "懲", - "懶", - "成", - "戛", - "扝", - "抱", - "拔", - "捐", - "𢬌", - "挽", - "拼", - "捨", - "掃", - "揤", - "𢯱", - "搢", - "揅", - "掩", - "㨮", - "摩", - "摾", - "撝", - "摷", - "㩬", - "敏", - "敬", - "𣀊", - "旣", - "書", - "晉", - "㬙", - "暑", - "㬈", - "㫤", - "冒", - "冕", - "最", - "暜", - "肭", - "䏙", - "朗", - "望", - "朡", - "杞", - "杓", - "𣏃", - "㭉", - "柺", - "枅", - "桒", - "梅", - "𣑭", - "梎", - "栟", - "椔", - "㮝", - "楂", - "榣", - "槪", - "檨", - "𣚣", - "櫛", - "㰘", - "次", - "𣢧", - "歔", - "㱎", - "歲", - "殟", - "殺", - "殻", - "𣪍", - "𡴋", - "𣫺", - "汎", - "𣲼", - "沿", - "泍", - "汧", - "洖", - "派", - "海", - "流", - "浩", - "浸", - "涅", - "𣴞", - "洴", - "港", - "湮", - "㴳", - "滋", - "滇", - "𣻑", - "淹", - "潮", - "𣽞", - "𣾎", - "濆", - "瀹", - "瀞", - "瀛", - "㶖", - "灊", - "災", - "灷", - "炭", - "𠔥", - "煅", - "𤉣", - "熜", - "𤎫", - "爨", - "爵", - "牐", - "𤘈", - "犀", - "犕", - "𤜵", - "𤠔", - "獺", - "王", - "㺬", - "玥", - "㺸", - "瑇", - "瑜", - "瑱", - "璅", - "瓊", - "㼛", - "甤", - "𤰶", - "甾", - "𤲒", - "異", - "𢆟", - "瘐", - "𤾡", - "𤾸", - "𥁄", - "㿼", - "䀈", - "直", - "𥃳", - "𥃲", - "𥄙", - "𥄳", - "眞", - "真", - "睊", - "䀹", - "瞋", - "䁆", - "䂖", - "𥐝", - "硎", - "碌", - "磌", - "䃣", - "𥘦", - "祖", - "𥚚", - "𥛅", - "福", - "秫", - "䄯", - "穀", - "穊", - "穏", - "𥥼", - "𥪧", - "竮", - "䈂", - "𥮫", - "篆", - "築", - "䈧", - "𥲀", - "糒", - "䊠", - "糨", - "糣", - "紀", - "𥾆", - "絣", - "䌁", - "緇", - "縂", - "繅", - "䌴", - "𦈨", - "𦉇", - "䍙", - "𦋙", - "罺", - "𦌾", - "羕", - "翺", - "者", - "𦓚", - "𦔣", - "聠", - "𦖨", - "聰", - "𣍟", - "䏕", - "育", - "脃", - "䐋", - "脾", - "媵", - "𦞧", - "𦞵", - "𣎓", - "𣎜", - "舁", - "舄", - "辞", - "䑫", - "芑", - "芋", - "芝", - "劳", - "花", - "芳", - "芽", - "苦", - "𦬼", - "若", - "茝", - "荣", - "莭", - "茣", - "莽", - "菧", - "著", - "荓", - "菊", - "菌", - "菜", - "𦰶", - "𦵫", - "𦳕", - "䔫", - "蓱", - "蓳", - "蔖", - "𧏊", - "蕤", - "𦼬", - "䕝", - "䕡", - "𦾱", - "𧃒", - "䕫", - "虐", - "虜", - "虧", - "虩", - "蚩", - "蚈", - "蜎", - "蛢", - "蝹", - "蜨", - "蝫", - "螆", - "䗗", - "蟡", - "蠁", - "䗹", - "衠", - "衣", - "𧙧", - "裗", - "裞", - "䘵", - "裺", - "㒻", - "𧢮", - "𧥦", - "䚾", - "䛇", - "誠", - "諭", - "變", - "豕", - "𧲨", - "貫", - "賁", - "贛", - "起", - "𧼯", - "𠠄", - "跋", - "趼", - "跰", - "𠣞", - "軔", - "輸", - "𨗒", - "𨗭", - "邔", - "郱", - "鄑", - "𨜮", - "鄛", - "鈸", - "鋗", - "鋘", - "鉼", - "鏹", - "鐕", - "𨯺", - "開", - "䦕", - "閷", - "𨵷", - "䧦", - "雃", - "嶲", - "霣", - "𩅅", - "𩈚", - "䩮", - "䩶", - "韠", - "𩐊", - "䪲", - "𩒖", - "頋", - "頩", - "𩖶", - "飢", - "䬳", - "餩", - "馧", - "駂", - "駾", - "䯎", - "𩬰", - "鬒", - "鱀", - "鳽", - "䳎", - "䳭", - "鵧", - "𪃎", - "䳸", - "𪄅", - "𪈎", - "𪊑", - "麻", - "䵖", - "黹", - "黾", - "鼅", - "鼏", - "鼖", - "鼻", - "𪘀", - None, - None, - None, - None, - None, - None, - None, -) diff --git a/functions/python/data-processor/mangum-0.21.0.dist-info/INSTALLER b/functions/python/data-processor/mangum-0.21.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/mangum-0.21.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/mangum-0.21.0.dist-info/METADATA b/functions/python/data-processor/mangum-0.21.0.dist-info/METADATA deleted file mode 100644 index 8457edfcd..000000000 --- a/functions/python/data-processor/mangum-0.21.0.dist-info/METADATA +++ /dev/null @@ -1,93 +0,0 @@ -Metadata-Version: 2.4 -Name: mangum -Version: 0.21.0 -Summary: AWS Lambda support for ASGI applications -Project-URL: Homepage, https://github.com/Kludex/mangum -Project-URL: Documentation, https://mangum.fastapiexpert.com -Project-URL: Changelog, https://github.com/Kludex/mangum/blob/main/CHANGELOG.md -Project-URL: Funding, https://github.com/sponsors/Kludex -Project-URL: Source, https://github.com/Kludex/mangum -Author-email: Jordan Eremieff , Marcelo Trylesinski -License-File: LICENSE -Classifier: License :: OSI Approved :: MIT License -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Internet :: WWW/HTTP -Requires-Python: >=3.9 -Requires-Dist: typing-extensions -Description-Content-Type: text/markdown - -# Mangum - - - Package version - -PyPI - Python Version - -Mangum is an adapter for running [ASGI](https://asgi.readthedocs.io/en/latest/) applications in AWS Lambda to handle Function URL, API Gateway, ALB, and Lambda@Edge events. - -***Documentation***: https://mangum.fastapiexpert.com/ - -## Features - -- Event handlers for API Gateway [HTTP](https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api.html) and [REST](https://docs.aws.amazon.com/apigateway/latest/developerguide/apigateway-rest-api.html) APIs, [Application Load Balancer](https://docs.aws.amazon.com/elasticloadbalancing/latest/application/lambda-functions.html), [Function URLs](https://docs.aws.amazon.com/lambda/latest/dg/lambda-urls.html), and [CloudFront Lambda@Edge](https://docs.aws.amazon.com/lambda/latest/dg/lambda-edge.html). - -- Compatibility with ASGI application frameworks, such as [Starlette](https://www.starlette.io/), [FastAPI](https://fastapi.tiangolo.com/), [Quart](https://pgjones.gitlab.io/quart/) and [Django](https://www.djangoproject.com/). - -- Support for binary media types and payload compression in API Gateway using GZip or Brotli. - -- Works with existing deployment and configuration tools, including [Serverless Framework](https://www.serverless.com/) and [AWS SAM](https://docs.aws.amazon.com/serverless-application-model/latest/developerguide/what-is-sam.html). - -- Startup and shutdown [lifespan](https://asgi.readthedocs.io/en/latest/specs/lifespan.html) events. - -## Installation - -```shell -pip install mangum -``` - -## Example - -```python -from mangum import Mangum - -async def app(scope, receive, send): - await send( - { - "type": "http.response.start", - "status": 200, - "headers": [[b"content-type", b"text/plain; charset=utf-8"]], - } - ) - await send({"type": "http.response.body", "body": b"Hello, world!"}) - - -handler = Mangum(app, lifespan="off") -``` - -Or using a framework: - -```python -from fastapi import FastAPI -from mangum import Mangum - -app = FastAPI() - - -@app.get("/") -def read_root(): - return {"Hello": "World"} - - -@app.get("/items/{item_id}") -def read_item(item_id: int, q: str = None): - return {"item_id": item_id, "q": q} - -handler = Mangum(app, lifespan="off") -``` diff --git a/functions/python/data-processor/mangum-0.21.0.dist-info/RECORD b/functions/python/data-processor/mangum-0.21.0.dist-info/RECORD deleted file mode 100644 index 28aa7df54..000000000 --- a/functions/python/data-processor/mangum-0.21.0.dist-info/RECORD +++ /dev/null @@ -1,31 +0,0 @@ -mangum-0.21.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -mangum-0.21.0.dist-info/METADATA,sha256=xNX0hnVvY1EwK2fKMGhKLcmd4VcJ0RBOi4Efydinsgw,3537 -mangum-0.21.0.dist-info/RECORD,, -mangum-0.21.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -mangum-0.21.0.dist-info/WHEEL,sha256=WLgqFyCfm_KASv4WHyYy0P3pM_m7J5L9k2skdKLirC8,87 -mangum-0.21.0.dist-info/licenses/LICENSE,sha256=DZ61C2cPKRfwLIbk4ZbQKL7XD1zaqYTND1adRFPl68s,1072 -mangum/__init__.py,sha256=MDyQsaU5u4zAp0-0izpQCquOyFJ_Ahx6kcaStQKCVhg,56 -mangum/__pycache__/__init__.cpython-314.pyc,, -mangum/__pycache__/adapter.cpython-314.pyc,, -mangum/__pycache__/exceptions.cpython-314.pyc,, -mangum/__pycache__/types.cpython-314.pyc,, -mangum/adapter.py,sha256=579z5ESpx860FWIgFoECiNoDxEW7Wuh9v4yI4dAcNV4,3157 -mangum/exceptions.py,sha256=AuQ6W2yb_vhpiZPHC6rVhHckeOTdrqNdJDWjTuRr8bo,440 -mangum/handlers/__init__.py,sha256=U9ppMGhzyL9X3V3QiYjuM4A3gKtJknc5DQ3wQTan0CM,220 -mangum/handlers/__pycache__/__init__.cpython-314.pyc,, -mangum/handlers/__pycache__/alb.cpython-314.pyc,, -mangum/handlers/__pycache__/api_gateway.cpython-314.pyc,, -mangum/handlers/__pycache__/lambda_at_edge.cpython-314.pyc,, -mangum/handlers/__pycache__/utils.cpython-314.pyc,, -mangum/handlers/alb.py,sha256=XNmmx_whLFBkWIOW-hwpWnjOjhUn0lvxkIlox_uSGyg,5952 -mangum/handlers/api_gateway.py,sha256=rMP_nHim2PnfVBpf9MfUNy9lb2S3_GmRiX9D_yX7JBo,7921 -mangum/handlers/lambda_at_edge.py,sha256=SQR76CTa1Va4axVKCnLa9zUB78lIvWeMZsxNPzEJOAw,3368 -mangum/handlers/utils.py,sha256=IvNeDsmO7mYBBcTwcmOfJRznPQ7Au53QI7SQiD_Jy94,2988 -mangum/protocols/__init__.py,sha256=iyggz3hePt363IMEIcySQ99xy7BxOOh0s0Ntl5Y4Pgc,148 -mangum/protocols/__pycache__/__init__.cpython-314.pyc,, -mangum/protocols/__pycache__/http.cpython-314.pyc,, -mangum/protocols/__pycache__/lifespan.cpython-314.pyc,, -mangum/protocols/http.py,sha256=4qRa8AJhkbDBu6lyvgtc4eC_395l7iEzU9eSnwYeqj8,4029 -mangum/protocols/lifespan.py,sha256=LheYkftfnsyKDW5No8uovJ37lf2lZPJJL5Y4ZpQMdL0,7735 -mangum/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -mangum/types.py,sha256=YGevlosLLzQsoM8CT6oM9QJvGKi7W7pdPyvVkDVxOBE,4230 diff --git a/functions/python/data-processor/mangum-0.21.0.dist-info/REQUESTED b/functions/python/data-processor/mangum-0.21.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/mangum-0.21.0.dist-info/WHEEL b/functions/python/data-processor/mangum-0.21.0.dist-info/WHEEL deleted file mode 100644 index ae8ec1bda..000000000 --- a/functions/python/data-processor/mangum-0.21.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.28.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/mangum-0.21.0.dist-info/licenses/LICENSE b/functions/python/data-processor/mangum-0.21.0.dist-info/licenses/LICENSE deleted file mode 100644 index 6168adbcb..000000000 --- a/functions/python/data-processor/mangum-0.21.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Jordan Eremieff - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/functions/python/data-processor/mangum/__init__.py b/functions/python/data-processor/mangum/__init__.py deleted file mode 100644 index fa5058b87..000000000 --- a/functions/python/data-processor/mangum/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from mangum.adapter import Mangum - -__all__ = ["Mangum"] diff --git a/functions/python/data-processor/mangum/adapter.py b/functions/python/data-processor/mangum/adapter.py deleted file mode 100644 index 4bef0d815..000000000 --- a/functions/python/data-processor/mangum/adapter.py +++ /dev/null @@ -1,84 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -from contextlib import ExitStack -from itertools import chain -from typing import Any - -from mangum.exceptions import ConfigurationError -from mangum.handlers import ALB, APIGateway, HTTPGateway, LambdaAtEdge -from mangum.protocols import HTTPCycle, LifespanCycle -from mangum.types import ASGI, LambdaConfig, LambdaContext, LambdaEvent, LambdaHandler, LifespanMode - -logger = logging.getLogger("mangum") - -HANDLERS: list[type[LambdaHandler]] = [ALB, HTTPGateway, APIGateway, LambdaAtEdge] - -DEFAULT_TEXT_MIME_TYPES: list[str] = [ - "text/", - "application/json", - "application/javascript", - "application/xml", - "application/vnd.api+json", - "application/vnd.oai.openapi", -] - - -class Mangum: - def __init__( - self, - app: ASGI, - lifespan: LifespanMode = "auto", - api_gateway_base_path: str = "/", - custom_handlers: list[type[LambdaHandler]] | None = None, - text_mime_types: list[str] | None = None, - exclude_headers: list[str] | None = None, - ) -> None: - if lifespan not in ("auto", "on", "off"): - raise ConfigurationError("Invalid argument supplied for `lifespan`. Choices are: auto|on|off") - - self.app = app - self.lifespan = lifespan - self.custom_handlers = custom_handlers or [] - exclude_headers = exclude_headers or [] - self.config = LambdaConfig( - api_gateway_base_path=api_gateway_base_path or "/", - text_mime_types=text_mime_types or [*DEFAULT_TEXT_MIME_TYPES], - exclude_headers=[header.lower() for header in exclude_headers], - ) - self._setup_event_loop() - - def infer(self, event: LambdaEvent, context: LambdaContext) -> LambdaHandler: - for handler_cls in chain(self.custom_handlers, HANDLERS): - if handler_cls.infer(event, context, self.config): - return handler_cls(event, context, self.config) - raise RuntimeError( # pragma: no cover - "The adapter was unable to infer a handler to use for the event. This " - "is likely related to how the Lambda function was invoked. (Are you " - "testing locally? Make sure the request payload is valid for a " - "supported handler.)" - ) - - def _setup_event_loop(self) -> None: - try: - asyncio.get_event_loop() - except RuntimeError: - loop = asyncio.new_event_loop() - asyncio.set_event_loop(loop) - - def __call__(self, event: LambdaEvent, context: LambdaContext) -> dict[str, Any]: - handler = self.infer(event, context) - scope = handler.scope - with ExitStack() as stack: - if self.lifespan in ("auto", "on"): - lifespan_cycle = LifespanCycle(self.app, self.lifespan) - stack.enter_context(lifespan_cycle) - scope.update({"state": lifespan_cycle.lifespan_state.copy()}) - - http_cycle = HTTPCycle(scope, handler.body) - http_response = http_cycle(self.app) - - return handler(http_response) - - assert False, "unreachable" # pragma: no cover diff --git a/functions/python/data-processor/mangum/exceptions.py b/functions/python/data-processor/mangum/exceptions.py deleted file mode 100644 index d14234c9d..000000000 --- a/functions/python/data-processor/mangum/exceptions.py +++ /dev/null @@ -1,14 +0,0 @@ -class LifespanFailure(Exception): - """Raise when a lifespan failure event is sent by an application.""" - - -class LifespanUnsupported(Exception): - """Raise when lifespan events are not supported by an application.""" - - -class UnexpectedMessage(Exception): - """Raise when an unexpected message type is received during an ASGI cycle.""" - - -class ConfigurationError(Exception): - """Raise when an error occurs parsing configuration.""" diff --git a/functions/python/data-processor/mangum/handlers/__init__.py b/functions/python/data-processor/mangum/handlers/__init__.py deleted file mode 100644 index 5afca3c9b..000000000 --- a/functions/python/data-processor/mangum/handlers/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from mangum.handlers.alb import ALB -from mangum.handlers.api_gateway import APIGateway, HTTPGateway -from mangum.handlers.lambda_at_edge import LambdaAtEdge - -__all__ = ["APIGateway", "HTTPGateway", "ALB", "LambdaAtEdge"] diff --git a/functions/python/data-processor/mangum/handlers/alb.py b/functions/python/data-processor/mangum/handlers/alb.py deleted file mode 100644 index 728b4349a..000000000 --- a/functions/python/data-processor/mangum/handlers/alb.py +++ /dev/null @@ -1,172 +0,0 @@ -from __future__ import annotations - -from collections.abc import Generator -from itertools import islice -from typing import Any -from urllib.parse import unquote, unquote_plus, urlencode - -from mangum.handlers.utils import ( - get_server_and_port, - handle_base64_response_body, - handle_exclude_headers, - maybe_encode_body, -) -from mangum.types import ( - LambdaConfig, - LambdaContext, - LambdaEvent, - QueryParams, - Response, - Scope, -) - - -def all_casings(input_string: str) -> Generator[str, None, None]: - """ - Permute all casings of a given string. - A pretty algoritm, via @Amber - http://stackoverflow.com/questions/6792803/finding-all-possible-case-permutations-in-python - """ - if not input_string: - yield "" - else: - first = input_string[:1] - if first.lower() == first.upper(): - for sub_casing in all_casings(input_string[1:]): - yield first + sub_casing - else: - for sub_casing in all_casings(input_string[1:]): - yield first.lower() + sub_casing - yield first.upper() + sub_casing - - -def case_mutated_headers(multi_value_headers: dict[str, list[str]]) -> dict[str, str]: - """Create str/str key/value headers, with duplicate keys case mutated.""" - headers: dict[str, str] = {} - for key, values in multi_value_headers.items(): - if len(values) > 0: - casings = list(islice(all_casings(key), len(values))) - for value, cased_key in zip(values, casings): - headers[cased_key] = value - return headers - - -def encode_query_string_for_alb(params: QueryParams) -> bytes: - """Encode the query string parameters for the ALB event. The parameters must be - decoded and then encoded again to prevent double encoding. - - According to the docs: - - "If the query parameters are URL-encoded, the load balancer does not decode - "them. You must decode them in your Lambda function." - """ - params = { - unquote_plus(key): ( - unquote_plus(value) if isinstance(value, str) else tuple(unquote_plus(element) for element in value) - ) - for key, value in params.items() - } - query_string = urlencode(params, doseq=True).encode() - - return query_string - - -def transform_headers(event: LambdaEvent) -> list[tuple[bytes, bytes]]: - headers: list[tuple[bytes, bytes]] = [] - if "multiValueHeaders" in event: - for k, v in event["multiValueHeaders"].items(): - for inner_v in v: - headers.append((k.lower().encode(), inner_v.encode())) - else: - for k, v in event["headers"].items(): - headers.append((k.lower().encode(), v.encode())) - - return headers - - -class ALB: - @classmethod - def infer(cls, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> bool: - return "requestContext" in event and "elb" in event["requestContext"] - - def __init__(self, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> None: - self.event = event - self.context = context - self.config = config - - @property - def body(self) -> bytes: - return maybe_encode_body( - self.event.get("body", b""), - is_base64=self.event.get("isBase64Encoded", False), - ) - - @property - def scope(self) -> Scope: - headers = transform_headers(self.event) - list_headers = [list(x) for x in headers] - # Unique headers. If there are duplicates, it will use the last defined. - uq_headers = {k.decode(): v.decode() for k, v in headers} - source_ip = uq_headers.get("x-forwarded-for", "") - path = unquote(self.event["path"]) if self.event["path"] else "/" - http_method = self.event["httpMethod"] - - params = self.event.get( - "multiValueQueryStringParameters", - self.event.get("queryStringParameters", {}), - ) - if not params: - query_string = b"" - else: - query_string = encode_query_string_for_alb(params) - - server = get_server_and_port(uq_headers) - client = (source_ip, 0) - - scope: Scope = { - "type": "http", - "method": http_method, - "http_version": "1.1", - "headers": list_headers, - "path": path, - "raw_path": None, - "root_path": "", - "scheme": uq_headers.get("x-forwarded-proto", "https"), - "query_string": query_string, - "server": server, - "client": client, - "asgi": {"version": "3.0", "spec_version": "2.0"}, - "aws.event": self.event, - "aws.context": self.context, - } - - return scope - - def __call__(self, response: Response) -> dict[str, Any]: - multi_value_headers: dict[str, list[str]] = {} - for key, value in response["headers"]: - lower_key = key.decode().lower() - if lower_key not in multi_value_headers: - multi_value_headers[lower_key] = [] - multi_value_headers[lower_key].append(value.decode()) - - finalized_headers = case_mutated_headers(multi_value_headers) - finalized_body, is_base64_encoded = handle_base64_response_body( - response["body"], finalized_headers, self.config["text_mime_types"] - ) - - out = { - "statusCode": response["status"], - "body": finalized_body, - "isBase64Encoded": is_base64_encoded, - } - - # You must use multiValueHeaders if you have enabled multi-value headers and - # headers otherwise. - multi_value_headers_enabled = "multiValueHeaders" in self.scope["aws.event"] - if multi_value_headers_enabled: - out["multiValueHeaders"] = handle_exclude_headers(multi_value_headers, self.config) - else: - out["headers"] = handle_exclude_headers(finalized_headers, self.config) - - return out diff --git a/functions/python/data-processor/mangum/handlers/api_gateway.py b/functions/python/data-processor/mangum/handlers/api_gateway.py deleted file mode 100644 index 0ecd3bd01..000000000 --- a/functions/python/data-processor/mangum/handlers/api_gateway.py +++ /dev/null @@ -1,219 +0,0 @@ -from __future__ import annotations - -from typing import Any -from urllib.parse import urlencode - -from mangum.handlers.utils import ( - get_server_and_port, - handle_base64_response_body, - handle_exclude_headers, - handle_multi_value_headers, - maybe_encode_body, - strip_api_gateway_path, -) -from mangum.types import ( - Headers, - LambdaConfig, - LambdaContext, - LambdaEvent, - QueryParams, - Response, - Scope, -) - - -def _encode_query_string_for_apigw(event: LambdaEvent) -> bytes: - params: QueryParams = event.get("multiValueQueryStringParameters", {}) - if not params: - params = event.get("queryStringParameters", {}) - if not params: - return b"" - - return urlencode(params, doseq=True).encode() - - -def _handle_multi_value_headers_for_request(event: LambdaEvent) -> dict[str, str]: - headers = event.get("headers", {}) or {} - headers = {k.lower(): v for k, v in headers.items()} - if event.get("multiValueHeaders"): - headers.update( - { - k.lower(): ", ".join(v) if isinstance(v, list) else "" - for k, v in event.get("multiValueHeaders", {}).items() - } - ) - - return headers - - -def _combine_headers_v2( - input_headers: Headers, -) -> tuple[dict[str, str], list[str]]: - output_headers: dict[str, str] = {} - cookies: list[str] = [] - for key, value in input_headers: - normalized_key: str = key.decode().lower() - normalized_value: str = value.decode() - if normalized_key == "set-cookie": - cookies.append(normalized_value) - else: - if normalized_key in output_headers: - normalized_value = f"{output_headers[normalized_key]},{normalized_value}" - output_headers[normalized_key] = normalized_value - - return output_headers, cookies - - -class APIGateway: - @classmethod - def infer(cls, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> bool: - return "resource" in event and "requestContext" in event - - def __init__(self, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> None: - self.event = event - self.context = context - self.config = config - - @property - def body(self) -> bytes: - return maybe_encode_body( - self.event.get("body", b""), - is_base64=self.event.get("isBase64Encoded", False), - ) - - @property - def scope(self) -> Scope: - headers = _handle_multi_value_headers_for_request(self.event) - return { - "type": "http", - "http_version": "1.1", - "method": self.event["httpMethod"], - "headers": [[k.encode(), v.encode()] for k, v in headers.items()], - "path": strip_api_gateway_path( - self.event["path"], - api_gateway_base_path=self.config["api_gateway_base_path"], - ), - "raw_path": None, - "root_path": "", - "scheme": headers.get("x-forwarded-proto", "https"), - "query_string": _encode_query_string_for_apigw(self.event), - "server": get_server_and_port(headers), - "client": ( - self.event["requestContext"].get("identity", {}).get("sourceIp"), - 0, - ), - "asgi": {"version": "3.0", "spec_version": "2.0"}, - "aws.event": self.event, - "aws.context": self.context, - } - - def __call__(self, response: Response) -> dict[str, Any]: - finalized_headers, multi_value_headers = handle_multi_value_headers(response["headers"]) - finalized_body, is_base64_encoded = handle_base64_response_body( - response["body"], finalized_headers, self.config["text_mime_types"] - ) - - return { - "statusCode": response["status"], - "headers": handle_exclude_headers(finalized_headers, self.config), - "multiValueHeaders": handle_exclude_headers(multi_value_headers, self.config), - "body": finalized_body, - "isBase64Encoded": is_base64_encoded, - } - - -class HTTPGateway: - @classmethod - def infer(cls, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> bool: - return "version" in event and "requestContext" in event - - def __init__(self, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> None: - self.event = event - self.context = context - self.config = config - - @property - def body(self) -> bytes: - return maybe_encode_body( - self.event.get("body", b""), - is_base64=self.event.get("isBase64Encoded", False), - ) - - @property - def scope(self) -> Scope: - request_context = self.event["requestContext"] - event_version = self.event["version"] - - # API Gateway v2 - if event_version == "2.0": - headers = {k.lower(): v for k, v in self.event.get("headers", {}).items()} - source_ip = request_context["http"]["sourceIp"] - path = request_context["http"]["path"] - http_method = request_context["http"]["method"] - query_string = self.event.get("rawQueryString", "").encode() - - if self.event.get("cookies"): - headers["cookie"] = "; ".join(self.event.get("cookies", [])) - - # API Gateway v1 - else: - headers = _handle_multi_value_headers_for_request(self.event) - source_ip = request_context.get("identity", {}).get("sourceIp") - path = self.event["path"] - http_method = self.event["httpMethod"] - query_string = _encode_query_string_for_apigw(self.event) - - path = strip_api_gateway_path( - path, - api_gateway_base_path=self.config["api_gateway_base_path"], - ) - server = get_server_and_port(headers) - client = (source_ip, 0) - - return { - "type": "http", - "method": http_method, - "http_version": "1.1", - "headers": [[k.encode(), v.encode()] for k, v in headers.items()], - "path": path, - "raw_path": None, - "root_path": "", - "scheme": headers.get("x-forwarded-proto", "https"), - "query_string": query_string, - "server": server, - "client": client, - "asgi": {"version": "3.0", "spec_version": "2.0"}, - "aws.event": self.event, - "aws.context": self.context, - } - - def __call__(self, response: Response) -> dict[str, Any]: - if self.scope["aws.event"]["version"] == "2.0": - finalized_headers, cookies = _combine_headers_v2(response["headers"]) - - if "content-type" not in finalized_headers and response["body"] is not None: - finalized_headers["content-type"] = "application/json" - - finalized_body, is_base64_encoded = handle_base64_response_body( - response["body"], finalized_headers, self.config["text_mime_types"] - ) - response_out = { - "statusCode": response["status"], - "body": finalized_body, - "headers": finalized_headers or None, - "cookies": cookies or None, - "isBase64Encoded": is_base64_encoded, - } - return {key: value for key, value in response_out.items() if value is not None} - - finalized_headers, multi_value_headers = handle_multi_value_headers(response["headers"]) - finalized_body, is_base64_encoded = handle_base64_response_body( - response["body"], finalized_headers, self.config["text_mime_types"] - ) - return { - "statusCode": response["status"], - "headers": finalized_headers, - "multiValueHeaders": multi_value_headers, - "body": finalized_body, - "isBase64Encoded": is_base64_encoded, - } diff --git a/functions/python/data-processor/mangum/handlers/lambda_at_edge.py b/functions/python/data-processor/mangum/handlers/lambda_at_edge.py deleted file mode 100644 index f4e9e68b4..000000000 --- a/functions/python/data-processor/mangum/handlers/lambda_at_edge.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from mangum.handlers.utils import ( - handle_base64_response_body, - handle_exclude_headers, - handle_multi_value_headers, - maybe_encode_body, -) -from mangum.types import LambdaConfig, LambdaContext, LambdaEvent, Response, Scope - - -class LambdaAtEdge: - @classmethod - def infer(cls, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> bool: - return "Records" in event and len(event["Records"]) > 0 and "cf" in event["Records"][0] - - # FIXME: Since this is the last in the chain it doesn't get coverage by default, - # # just ignoring it for now. - # return None # pragma: nocover - - def __init__(self, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> None: - self.event = event - self.context = context - self.config = config - - @property - def body(self) -> bytes: - cf_request_body = self.event["Records"][0]["cf"]["request"].get("body", {}) - return maybe_encode_body( - cf_request_body.get("data"), - is_base64=cf_request_body.get("encoding", "") == "base64", - ) - - @property - def scope(self) -> Scope: - cf_request = self.event["Records"][0]["cf"]["request"] - scheme_header = cf_request["headers"].get("cloudfront-forwarded-proto", [{}]) - scheme = scheme_header[0].get("value", "https") - host_header = cf_request["headers"].get("host", [{}]) - server_name = host_header[0].get("value", "mangum") - if ":" not in server_name: - forwarded_port_header = cf_request["headers"].get("x-forwarded-port", [{}]) - server_port = forwarded_port_header[0].get("value", 80) - else: - server_name, server_port = server_name.split(":") # pragma: no cover - - server = (server_name, int(server_port)) - source_ip = cf_request["clientIp"] - client = (source_ip, 0) - http_method = cf_request["method"] - - return { - "type": "http", - "method": http_method, - "http_version": "1.1", - "headers": [[k.encode(), v[0]["value"].encode()] for k, v in cf_request["headers"].items()], - "path": cf_request["uri"], - "raw_path": None, - "root_path": "", - "scheme": scheme, - "query_string": cf_request["querystring"].encode(), - "server": server, - "client": client, - "asgi": {"version": "3.0", "spec_version": "2.0"}, - "aws.event": self.event, - "aws.context": self.context, - } - - def __call__(self, response: Response) -> dict[str, Any]: - multi_value_headers, _ = handle_multi_value_headers(response["headers"]) - response_body, is_base64_encoded = handle_base64_response_body( - response["body"], multi_value_headers, self.config["text_mime_types"] - ) - finalized_headers: dict[str, list[dict[str, str]]] = { - key.decode().lower(): [{"key": key.decode().lower(), "value": val.decode()}] - for key, val in response["headers"] - } - - return { - "status": response["status"], - "headers": handle_exclude_headers(finalized_headers, self.config), - "body": response_body, - "isBase64Encoded": is_base64_encoded, - } diff --git a/functions/python/data-processor/mangum/handlers/utils.py b/functions/python/data-processor/mangum/handlers/utils.py deleted file mode 100644 index d2bdaf05b..000000000 --- a/functions/python/data-processor/mangum/handlers/utils.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import annotations - -import base64 -from typing import Any -from urllib.parse import unquote - -from mangum.types import Headers, LambdaConfig - - -def maybe_encode_body(body: str | bytes, *, is_base64: bool) -> bytes: - body = body or b"" - if is_base64: - body = base64.b64decode(body) - elif not isinstance(body, bytes): - body = body.encode() - - return body - - -def get_server_and_port(headers: dict[str, Any]) -> tuple[str, int]: - server_name = headers.get("host", "mangum") - if ":" not in server_name: - server_port = headers.get("x-forwarded-port", 80) - else: - server_name, server_port = server_name.split(":") # pragma: no cover - server = (server_name, int(server_port)) - - return server - - -def strip_api_gateway_path(path: str, *, api_gateway_base_path: str) -> str: - if not path: - return "/" - - if api_gateway_base_path and api_gateway_base_path != "/": - if not api_gateway_base_path.startswith("/"): - api_gateway_base_path = f"/{api_gateway_base_path}" - if path.startswith(api_gateway_base_path): - path = path[len(api_gateway_base_path) :] - - return unquote(path) - - -def handle_multi_value_headers( - response_headers: Headers, -) -> tuple[dict[str, str], dict[str, list[str]]]: - headers: dict[str, str] = {} - multi_value_headers: dict[str, list[str]] = {} - for key, value in response_headers: - lower_key = key.decode().lower() - if lower_key in multi_value_headers: - multi_value_headers[lower_key].append(value.decode()) - elif lower_key in headers: - # Move existing to multi_value_headers and append current - multi_value_headers[lower_key] = [ - headers[lower_key], - value.decode(), - ] - del headers[lower_key] - else: - headers[lower_key] = value.decode() - return headers, multi_value_headers - - -def handle_base64_response_body( - body: bytes, - headers: dict[str, str], - text_mime_types: list[str], -) -> tuple[str, bool]: - is_base64_encoded = False - output_body = "" - if body != b"": - for text_mime_type in text_mime_types: - if text_mime_type in headers.get("content-type", ""): - try: - output_body = body.decode() - except UnicodeDecodeError: - output_body = base64.b64encode(body).decode() - is_base64_encoded = True - break - else: - output_body = base64.b64encode(body).decode() - is_base64_encoded = True - - return output_body, is_base64_encoded - - -def handle_exclude_headers(headers: dict[str, Any], config: LambdaConfig) -> dict[str, Any]: - finalized_headers = {} - for header_key, header_value in headers.items(): - if header_key in config["exclude_headers"]: - continue - finalized_headers[header_key] = header_value - - return finalized_headers diff --git a/functions/python/data-processor/mangum/protocols/__init__.py b/functions/python/data-processor/mangum/protocols/__init__.py deleted file mode 100644 index 12571c3ea..000000000 --- a/functions/python/data-processor/mangum/protocols/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .http import HTTPCycle -from .lifespan import LifespanCycle, LifespanCycleState - -__all__ = ["HTTPCycle", "LifespanCycleState", "LifespanCycle"] diff --git a/functions/python/data-processor/mangum/protocols/http.py b/functions/python/data-processor/mangum/protocols/http.py deleted file mode 100644 index f0b50e5e4..000000000 --- a/functions/python/data-processor/mangum/protocols/http.py +++ /dev/null @@ -1,107 +0,0 @@ -import asyncio -import enum -import logging -from io import BytesIO - -from mangum.exceptions import UnexpectedMessage -from mangum.types import ASGI, Message, Response, Scope - - -class HTTPCycleState(enum.Enum): - """ - The state of the ASGI `http` connection. - * **REQUEST** - Initial state. The ASGI application instance will be run with the - connection scope containing the `http` type. - * **RESPONSE** - The `http.response.start` event has been sent by the application. - The next expected message is the `http.response.body` event, containing the body - content. An application may pass the `more_body` argument to send content in chunks, - however content will always be returned in a single response, never streamed. - * **COMPLETE** - The body content from the ASGI application has been completely - read. A disconnect event will be sent to the application, and the response will - be returned. - """ - - REQUEST = enum.auto() - RESPONSE = enum.auto() - COMPLETE = enum.auto() - - -class HTTPCycle: - def __init__(self, scope: Scope, body: bytes) -> None: - self.scope = scope - self.buffer = BytesIO() - self.state = HTTPCycleState.REQUEST - self.logger = logging.getLogger("mangum.http") - self.app_queue: asyncio.Queue[Message] = asyncio.Queue() - self.app_queue.put_nowait( - { - "type": "http.request", - "body": body, - "more_body": False, - } - ) - - def __call__(self, app: ASGI) -> Response: - asgi_instance = self.run(app) - loop = asyncio.get_event_loop() - asgi_task = loop.create_task(asgi_instance) - loop.run_until_complete(asgi_task) - - return { - "status": self.status, - "headers": self.headers, - "body": self.body, - } - - async def run(self, app: ASGI) -> None: - try: - await app(self.scope, self.receive, self.send) - except BaseException: - self.logger.exception("An error occurred running the application.") - if self.state is HTTPCycleState.REQUEST: - await self.send( - { - "type": "http.response.start", - "status": 500, - "headers": [[b"content-type", b"text/plain; charset=utf-8"]], - } - ) - await self.send( - { - "type": "http.response.body", - "body": b"Internal Server Error", - "more_body": False, - } - ) - elif self.state is not HTTPCycleState.COMPLETE: - self.status = 500 - self.body = b"Internal Server Error" - self.headers = [[b"content-type", b"text/plain; charset=utf-8"]] - - async def receive(self) -> Message: - return await self.app_queue.get() # pragma: no cover - - async def send(self, message: Message) -> None: - if self.state is HTTPCycleState.REQUEST and message["type"] == "http.response.start": - self.status = message["status"] - self.headers = message.get("headers", []) - self.state = HTTPCycleState.RESPONSE - elif self.state is HTTPCycleState.RESPONSE and message["type"] == "http.response.body": - body = message.get("body", b"") - more_body = message.get("more_body", False) - self.buffer.write(body) - if not more_body: - self.body = self.buffer.getvalue() - self.buffer.close() - - self.state = HTTPCycleState.COMPLETE - await self.app_queue.put({"type": "http.disconnect"}) - - self.logger.info( - "%s %s %s", - self.scope["method"], - self.scope["path"], - self.status, - ) - else: - raise UnexpectedMessage(f"Unexpected {message['type']}") diff --git a/functions/python/data-processor/mangum/protocols/lifespan.py b/functions/python/data-processor/mangum/protocols/lifespan.py deleted file mode 100644 index 1e495c822..000000000 --- a/functions/python/data-processor/mangum/protocols/lifespan.py +++ /dev/null @@ -1,177 +0,0 @@ -from __future__ import annotations - -import asyncio -import enum -import logging -from types import TracebackType -from typing import Any - -from mangum.exceptions import LifespanFailure, LifespanUnsupported, UnexpectedMessage -from mangum.types import ASGI, LifespanMode, Message - - -class LifespanCycleState(enum.Enum): - """ - The state of the ASGI `lifespan` connection. - - * **CONNECTING** - Initial state. The ASGI application instance will be run with - the connection scope containing the `lifespan` type. - * **STARTUP** - The lifespan startup event has been pushed to the queue to be - received by the application. - * **SHUTDOWN** - The lifespan shutdown event has been pushed to the queue to be - received by the application. - * **FAILED** - A lifespan failure has been detected, and the connection will be - closed with an error. - * **UNSUPPORTED** - An application attempted to send a message before receiving - the lifespan startup event. If the lifespan argument is "on", then the connection - will be closed with an error. - """ - - CONNECTING = enum.auto() - STARTUP = enum.auto() - SHUTDOWN = enum.auto() - FAILED = enum.auto() - UNSUPPORTED = enum.auto() - - -class LifespanCycle: - """ - Manages the application cycle for an ASGI `lifespan` connection. - - * **app** - An asynchronous callable that conforms to version 3.0 of the ASGI - specification. This will usually be an ASGI framework application instance. - * **lifespan** - A string to configure lifespan support. Choices are `auto`, `on`, - and `off`. Default is `auto`. - * **state** - An enumerated `LifespanCycleState` type that indicates the state of - the ASGI connection. - * **exception** - An exception raised while handling the ASGI event. This may or - may not be raised depending on the state. - * **app_queue** - An asyncio queue (FIFO) containing messages to be received by the - application. - * **startup_event** - An asyncio event object used to control the application - startup flow. - * **shutdown_event** - An asyncio event object used to control the application - shutdown flow. - """ - - def __init__(self, app: ASGI, lifespan: LifespanMode) -> None: - self.app = app - self.lifespan = lifespan - self.state: LifespanCycleState = LifespanCycleState.CONNECTING - self.exception: BaseException | None = None - self.loop = asyncio.get_event_loop() - self.app_queue: asyncio.Queue[Message] = asyncio.Queue() - self.startup_event: asyncio.Event = asyncio.Event() - self.shutdown_event: asyncio.Event = asyncio.Event() - self.logger = logging.getLogger("mangum.lifespan") - self.lifespan_state: dict[str, Any] = {} - - def __enter__(self) -> None: - """Runs the event loop for application startup.""" - self.loop.create_task(self.run()) - self.loop.run_until_complete(self.startup()) - - def __exit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: TracebackType | None, - ) -> None: - """Runs the event loop for application shutdown.""" - self.loop.run_until_complete(self.shutdown()) - - async def run(self) -> None: - """Calls the application with the `lifespan` connection scope.""" - try: - await self.app( - {"type": "lifespan", "asgi": {"spec_version": "2.0", "version": "3.0"}, "state": self.lifespan_state}, - self.receive, - self.send, - ) - except LifespanUnsupported: - self.logger.info("ASGI 'lifespan' protocol appears unsupported.") - except (LifespanFailure, UnexpectedMessage) as exc: - self.exception = exc - except BaseException as exc: - self.logger.error("Exception in 'lifespan' protocol.", exc_info=exc) - finally: - self.startup_event.set() - self.shutdown_event.set() - - async def receive(self) -> Message: - """Awaited by the application to receive ASGI `lifespan` events.""" - if self.state is LifespanCycleState.CONNECTING: - # Connection established. The next event returned by the queue will be - # `lifespan.startup` to inform the application that the connection is - # ready to receive lifespan messages. - self.state = LifespanCycleState.STARTUP - - elif self.state is LifespanCycleState.STARTUP: - # Connection shutting down. The next event returned by the queue will be - # `lifespan.shutdown` to inform the application that the connection is now - # closing so that it may perform cleanup. - self.state = LifespanCycleState.SHUTDOWN - - return await self.app_queue.get() - - async def send(self, message: Message) -> None: - """Awaited by the application to send ASGI `lifespan` events.""" - message_type = message["type"] - self.logger.info("%s: '%s' event received from application.", self.state, message_type) - - if self.state is LifespanCycleState.CONNECTING: - if self.lifespan == "on": - raise LifespanFailure("Lifespan connection failed during startup and lifespan is 'on'.") - - # If a message is sent before the startup event is received by the - # application, then assume that lifespan is unsupported. - self.state = LifespanCycleState.UNSUPPORTED - raise LifespanUnsupported("Lifespan protocol appears unsupported.") - - if message_type not in ( - "lifespan.startup.complete", - "lifespan.shutdown.complete", - "lifespan.startup.failed", - "lifespan.shutdown.failed", - ): - self.state = LifespanCycleState.FAILED - raise UnexpectedMessage(f"Unexpected '{message_type}' event received.") - - if self.state is LifespanCycleState.STARTUP: - if message_type == "lifespan.startup.complete": - self.startup_event.set() - elif message_type == "lifespan.startup.failed": - self.state = LifespanCycleState.FAILED - self.startup_event.set() - message_value = message.get("message", "") - raise LifespanFailure(f"Lifespan startup failure. {message_value}") - - elif self.state is LifespanCycleState.SHUTDOWN: - if message_type == "lifespan.shutdown.complete": - self.shutdown_event.set() - elif message_type == "lifespan.shutdown.failed": - self.state = LifespanCycleState.FAILED - self.shutdown_event.set() - message_value = message.get("message", "") - raise LifespanFailure(f"Lifespan shutdown failure. {message_value}") - - async def startup(self) -> None: - """Pushes the `lifespan` startup event to the queue and handles errors.""" - self.logger.info("Waiting for application startup.") - await self.app_queue.put({"type": "lifespan.startup"}) - await self.startup_event.wait() - if self.state is LifespanCycleState.FAILED: - raise LifespanFailure(self.exception) - - if not self.exception: - self.logger.info("Application startup complete.") - else: - self.logger.info("Application startup failed.") - - async def shutdown(self) -> None: - """Pushes the `lifespan` shutdown event to the queue and handles errors.""" - self.logger.info("Waiting for application shutdown.") - await self.app_queue.put({"type": "lifespan.shutdown"}) - await self.shutdown_event.wait() - if self.state is LifespanCycleState.FAILED: - raise LifespanFailure(self.exception) diff --git a/functions/python/data-processor/mangum/py.typed b/functions/python/data-processor/mangum/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/mangum/types.py b/functions/python/data-processor/mangum/types.py deleted file mode 100644 index cd5183f49..000000000 --- a/functions/python/data-processor/mangum/types.py +++ /dev/null @@ -1,125 +0,0 @@ -from __future__ import annotations - -from collections.abc import Awaitable, MutableMapping, Sequence -from typing import Any, Callable, Union - -from typing_extensions import Literal, Protocol, TypeAlias, TypedDict - -LambdaEvent = dict[str, Any] -QueryParams: TypeAlias = MutableMapping[str, Union[str, Sequence[str]]] - - -class LambdaCognitoIdentity(Protocol): - """Information about the Amazon Cognito identity that authorized the request. - - **cognito_identity_id** - The authenticated Amazon Cognito identity. - **cognito_identity_pool_id** - The Amazon Cognito identity pool that authorized the - invocation. - """ - - cognito_identity_id: str - cognito_identity_pool_id: str - - -class LambdaMobileClient(Protocol): - """Mobile client information for the application and the device. - - **installation_id** - A unique identifier for an installation instance of an - application. - **app_title** - The title of the application. For example, "My App". - **app_version_code** - The version of the application. For example, "V2.0". - **app_version_name** - The version code for the application. For example, 3. - **app_package_name** - The name of the package. For example, "com.example.my_app". - """ - - installation_id: str - app_title: str - app_version_name: str - app_version_code: str - app_package_name: str - - -class LambdaMobileClientContext(Protocol): - """Information about client application and device when invoked via AWS Mobile SDK. - - **client** - A dict of name-value pairs that describe the mobile client application. - **custom** - A dict of custom values set by the mobile client application. - **env** - A dict of environment information provided by the AWS SDK. - """ - - client: LambdaMobileClient - custom: dict[str, Any] - env: dict[str, Any] - - -class LambdaContext(Protocol): - """The context object passed to the handler function. - - **function_name** - The name of the Lambda function. - **function_version** - The version of the function. - **invoked_function_arn** - The Amazon Resource Name (ARN) that's used to invoke the - function. Indicates if the invoker specified a version number or alias. - **memory_limit_in_mb** - The amount of memory that's allocated for the function. - **aws_request_id** - The identifier of the invocation request. - **log_group_name** - The log group for the function. - **log_stream_name** - The log stream for the function instance. - **identity** - (mobile apps) Information about the Amazon Cognito identity that - authorized the request. - **client_context** - (mobile apps) Client context that's provided to Lambda by the - client application. - """ - - function_name: str - function_version: str - invoked_function_arn: str - memory_limit_in_mb: int - aws_request_id: str - log_group_name: str - log_stream_name: str - identity: LambdaCognitoIdentity | None - client_context: LambdaMobileClientContext | None - - def get_remaining_time_in_millis(self) -> int: - """Returns the number of milliseconds left before the execution times out.""" - ... # pragma: no cover - - -Headers: TypeAlias = list[list[bytes]] -Message: TypeAlias = MutableMapping[str, Any] -Scope: TypeAlias = MutableMapping[str, Any] -Receive: TypeAlias = Callable[[], Awaitable[Message]] -Send: TypeAlias = Callable[[Message], Awaitable[None]] - - -class ASGI(Protocol): - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: ... # pragma: no cover - - -LifespanMode: TypeAlias = Literal["auto", "on", "off"] - - -class Response(TypedDict): - status: int - headers: Headers - body: bytes - - -class LambdaConfig(TypedDict): - api_gateway_base_path: str - text_mime_types: list[str] - exclude_headers: list[str] - - -class LambdaHandler(Protocol): - def __init__(self, *args: Any) -> None: ... # pragma: no cover - - @classmethod - def infer(cls, event: LambdaEvent, context: LambdaContext, config: LambdaConfig) -> bool: ... # pragma: no cover - - @property - def body(self) -> bytes: ... # pragma: no cover - - @property - def scope(self) -> Scope: ... # pragma: no cover - - def __call__(self, response: Response) -> dict[str, Any]: ... # pragma: no cover diff --git a/functions/python/data-processor/pydantic-2.13.4.dist-info/INSTALLER b/functions/python/data-processor/pydantic-2.13.4.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/pydantic-2.13.4.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/pydantic-2.13.4.dist-info/METADATA b/functions/python/data-processor/pydantic-2.13.4.dist-info/METADATA deleted file mode 100644 index 44d82dcac..000000000 --- a/functions/python/data-processor/pydantic-2.13.4.dist-info/METADATA +++ /dev/null @@ -1,1294 +0,0 @@ -Metadata-Version: 2.4 -Name: pydantic -Version: 2.13.4 -Summary: Data validation using Python type hints -Project-URL: Homepage, https://github.com/pydantic/pydantic -Project-URL: Documentation, https://docs.pydantic.dev -Project-URL: Funding, https://github.com/sponsors/samuelcolvin -Project-URL: Source, https://github.com/pydantic/pydantic -Project-URL: Changelog, https://docs.pydantic.dev/latest/changelog/ -Author-email: Samuel Colvin , Eric Jolibois , Hasan Ramezani , Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, Terrence Dorsey , David Montague , Serge Matveenko , Marcelo Trylesinski , Sydney Runkle , David Hewitt , Alex Hall , Victorien Plot -License-Expression: MIT -License-File: LICENSE -Classifier: Development Status :: 5 - Production/Stable -Classifier: Framework :: Hypothesis -Classifier: Framework :: Pydantic -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Information Technology -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Internet -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Requires-Python: >=3.9 -Requires-Dist: annotated-types>=0.6.0 -Requires-Dist: pydantic-core==2.46.4 -Requires-Dist: typing-extensions>=4.14.1 -Requires-Dist: typing-inspection>=0.4.2 -Provides-Extra: email -Requires-Dist: email-validator>=2.0.0; extra == 'email' -Provides-Extra: timezone -Requires-Dist: tzdata; (python_version >= '3.9' and platform_system == 'Windows') and extra == 'timezone' -Description-Content-Type: text/markdown - -# Pydantic Validation - -[![CI](https://img.shields.io/github/actions/workflow/status/pydantic/pydantic/ci.yml?branch=main&logo=github&label=CI)](https://github.com/pydantic/pydantic/actions?query=event%3Apush+branch%3Amain+workflow%3ACI) -[![Coverage](https://coverage-badge.samuelcolvin.workers.dev/pydantic/pydantic.svg)](https://coverage-badge.samuelcolvin.workers.dev/redirect/pydantic/pydantic) -[![pypi](https://img.shields.io/pypi/v/pydantic.svg)](https://pypi.python.org/pypi/pydantic) -[![CondaForge](https://img.shields.io/conda/v/conda-forge/pydantic.svg)](https://anaconda.org/conda-forge/pydantic) -[![downloads](https://static.pepy.tech/badge/pydantic/month)](https://pepy.tech/project/pydantic) -[![versions](https://img.shields.io/pypi/pyversions/pydantic.svg)](https://github.com/pydantic/pydantic) -[![license](https://img.shields.io/github/license/pydantic/pydantic.svg)](https://github.com/pydantic/pydantic/blob/main/LICENSE) -[![Pydantic v2](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/pydantic/pydantic/main/docs/badge/v2.json)](https://docs.pydantic.dev/latest/contributing/#badges) -[![llms.txt](https://img.shields.io/badge/llms.txt-green)](https://docs.pydantic.dev/latest/llms.txt) - -Data validation using Python type hints. - -Fast and extensible, Pydantic plays nicely with your linters/IDE/brain. -Define how data should be in pure, canonical Python 3.9+; validate it with Pydantic. - -## Pydantic Logfire :fire: - -We've launched Pydantic Logfire to help you monitor your applications. -[Learn more](https://pydantic.dev/logfire/?utm_source=pydantic_validation) - -## Pydantic V1.10 vs. V2 - -Pydantic V2 is a ground-up rewrite that offers many new features, performance improvements, and some breaking changes compared to Pydantic V1. - -If you're using Pydantic V1 you may want to look at the -[pydantic V1.10 Documentation](https://docs.pydantic.dev/) or, -[`1.10.X-fixes` git branch](https://github.com/pydantic/pydantic/tree/1.10.X-fixes). Pydantic V2 also ships with the latest version of Pydantic V1 built in so that you can incrementally upgrade your code base and projects: `from pydantic import v1 as pydantic_v1`. - -## Help - -See [documentation](https://docs.pydantic.dev/) for more details. - -## Installation - -Install using `pip install -U pydantic` or `conda install pydantic -c conda-forge`. -For more installation options to make Pydantic even faster, -see the [Install](https://docs.pydantic.dev/install/) section in the documentation. - -## A Simple Example - -```python -from datetime import datetime -from typing import Optional -from pydantic import BaseModel - -class User(BaseModel): - id: int - name: str = 'John Doe' - signup_ts: Optional[datetime] = None - friends: list[int] = [] - -external_data = {'id': '123', 'signup_ts': '2017-06-01 12:22', 'friends': [1, '2', b'3']} -user = User(**external_data) -print(user) -#> User id=123 name='John Doe' signup_ts=datetime.datetime(2017, 6, 1, 12, 22) friends=[1, 2, 3] -print(user.id) -#> 123 -``` - -## Contributing - -For guidance on setting up a development environment and how to make a -contribution to Pydantic, see -[Contributing to Pydantic](https://docs.pydantic.dev/contributing/). - -## Reporting a Security Vulnerability - -See our [security policy](https://github.com/pydantic/pydantic/security/policy). - -## Changelog - - - - - -## v2.13.4 (2026-05-06) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.4) - -### What's Changed - -#### Packaging - -* Bump libc from 0.2.155 to 0.2.185 by [@Viicos](https://github.com/Viicos) in [#13109](https://github.com/pydantic/pydantic/pull/13109) -* Adapt `pydantic-core` linker flags on macOS by [@washingtoneg](https://github.com/washingtoneg) and [@Viicos](https://github.com/Viicos) in [#13147](https://github.com/pydantic/pydantic/pull/13147) - -#### Fixes - -* Preserve `RootModel` core metadata by [@Viicos](https://github.com/Viicos) in [#13129](https://github.com/pydantic/pydantic/pull/13129) - -## v2.13.3 (2026-04-20) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.3) - -### What's Changed - -#### Fixes - -* Handle `AttributeError` subclasses with `from_attributes` by [@Viicos](https://github.com/Viicos) in [#13096](https://github.com/pydantic/pydantic/pull/13096) - -## v2.13.2 (2026-04-17) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.2) - -### What's Changed - -#### Fixes - -* Fix `ValidationInfo.field_name` missing with `model_validate_json()` by [@Viicos](https://github.com/Viicos) in [#13084](https://github.com/pydantic/pydantic/pull/13084) - -## v2.13.1 (2026-04-15) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.1) - -### What's Changed - -#### Fixes - -* Fix `ValidationInfo.data` missing with `model_validate_json()` by [@davidhewitt](https://github.com/davidhewitt) in [#13079](https://github.com/pydantic/pydantic/pull/13079) - -## v2.13.0 (2026-04-13) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.0) - -The highlights of the v2.13 release are available in the [blog post](https://pydantic.dev/articles/pydantic-v2-13-release). -Several minor changes (considered non-breaking changes according to our [versioning policy](https://pydantic.dev/docs/validation/2.13/get-started/version-policy/#pydantic-v2)) -are also included in this release. Make sure to look into them before upgrading. - -This release contains the updated `pydantic.v1` namespace, matching version 1.10.26 which includes support for Python 3.14. - -### What's Changed - -See the beta releases for all changes sinces 2.12. - -#### New Features - -* Allow default factories of private attributes to take validated model data by [@Viicos](https://github.com/Viicos) in [#13013](https://github.com/pydantic/pydantic/pull/13013) - -#### Changes - -* Warn when serializing fixed length tuples with too few items by [@arvindsaripalli](https://github.com/arvindsaripalli) in [#13016](https://github.com/pydantic/pydantic/pull/13016) - -#### Fixes - -* Change type of `Any` when synthesizing `_build_sources` for `BaseSettings.__init__()` signature in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#13049](https://github.com/pydantic/pydantic/pull/13049) -* Fix model equality when using runtime `extra` configuration by [@Viicos](https://github.com/Viicos) in [#13062](https://github.com/pydantic/pydantic/pull/13062) - -#### Packaging - -* Add zizmor for GitHub Actions workflow linting by [@Viicos](https://github.com/Viicos) in [#13039](https://github.com/pydantic/pydantic/pull/13039) -* Update jiter to v0.14.0 to fix a segmentation fault on musl Linux by [@Viicos](https://github.com/Viicos) in [#13064](https://github.com/pydantic/pydantic/pull/13064) - -### New Contributors - -* [@arvindsaripalli](https://github.com/arvindsaripalli) made their first contribution in [#13016](https://github.com/pydantic/pydantic/pull/13016) - -## v2.13.0b3 (2026-03-31) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.0b3) - -### What's Changed - -#### New Features - -* Add `ascii_only` option to `StringConstraints` by [@ai-man-codes](https://github.com/ai-man-codes) in [#12907](https://github.com/pydantic/pydantic/pull/12907) -* Support `exclude_if` in computed fields by [@andresliszt](https://github.com/andresliszt) in [#12748](https://github.com/pydantic/pydantic/pull/12748) -* Push down constraints in unions involving `MISSING` sentinel by [@Viicos](https://github.com/Viicos) in [#12908](https://github.com/pydantic/pydantic/pull/12908) - -#### Changes - -* Track extra fields set after init in `model_fields_set` by [@navalprakhar](https://github.com/navalprakhar) in [#12817](https://github.com/pydantic/pydantic/pull/12817) -* Do not include annotations that are not part of named tuple fields by [@galuszkak](https://github.com/galuszkak) in [#12951](https://github.com/pydantic/pydantic/pull/12951) -* No longer fall back to trying all union members when the variant selected by discriminator fails to serialize by [@navalprakhar](https://github.com/navalprakhar) in [#12825](https://github.com/pydantic/pydantic/pull/12825) - -#### Fixes - -* Support discriminator metadata outside union type alias by [@Viicos](https://github.com/Viicos) in [#12785](https://github.com/pydantic/pydantic/pull/12785) -* Respect `extras_schema` when only `extra_fields_behavior` is set on the config in JSON Schema generation for typed dictionaries by [@Viicos](https://github.com/Viicos) in [#12810](https://github.com/pydantic/pydantic/pull/12810) -* Ensure `__pydantic_private__` is set in `model_construct()` with user-defined `model_post_init()` by [@nightcityblade](https://github.com/nightcityblade) in [#12816](https://github.com/pydantic/pydantic/pull/12816) -* Handle all schema generation errors in `InstanceOf` by [@Viicos](https://github.com/Viicos) in [#12705](https://github.com/pydantic/pydantic/pull/12705) -* Allow dynamic models created with `create_model()` to be used as annotations in the Mypy plugin by [@Br1an67](https://github.com/Br1an67) in [#12879](https://github.com/pydantic/pydantic/pull/12879) -* Check for `PlaceholderNode` in Mypy plugin by [@Viicos](https://github.com/Viicos) in [#12929](https://github.com/pydantic/pydantic/pull/12929) -* Try other branches in smart union in case of omit errors by [@mikeedjones](https://github.com/mikeedjones) in [#12758](https://github.com/pydantic/pydantic/pull/12758) -* Patch unset attributes with `MISSING` during model serialization with `exclude_unset` by [@davidhewitt](https://github.com/davidhewitt) in [#12905](https://github.com/pydantic/pydantic/pull/12905) -* Ensure custom `__init__()` is called when using `model_validate_strings()` by [@siewcapital](https://github.com/siewcapital) in [#12897](https://github.com/pydantic/pydantic/pull/12897) - -#### Packaging - -* Add riscv64 build target for manylinux by [@boosterl](https://github.com/boosterl) in [#12723](https://github.com/pydantic/pydantic/pull/12723) - -### New Contributors - -* [@kelsonbrito50](https://github.com/kelsonbrito50) made their first contribution in [#12860](https://github.com/pydantic/pydantic/pull/12860) -* [@boosterl](https://github.com/boosterl) made their first contribution in [#12723](https://github.com/pydantic/pydantic/pull/12723) -* [@adityagiri3600](https://github.com/adityagiri3600) made their first contribution in [#12868](https://github.com/pydantic/pydantic/pull/12868) -* [@navalprakhar](https://github.com/navalprakhar) made their first contribution in [#12817](https://github.com/pydantic/pydantic/pull/12817) -* [@Br1an67](https://github.com/Br1an67) made their first contribution in [#12879](https://github.com/pydantic/pydantic/pull/12879) -* [@rmorshea](https://github.com/rmorshea) made their first contribution in [#12910](https://github.com/pydantic/pydantic/pull/12910) -* [@N3XT3R1337](https://github.com/N3XT3R1337) made their first contribution in [#12922](https://github.com/pydantic/pydantic/pull/12922) -* [@ai-man-codes](https://github.com/ai-man-codes) made their first contribution in [#12907](https://github.com/pydantic/pydantic/pull/12907) -* [@Yume05-dev](https://github.com/Yume05-dev) made their first contribution in [#12953](https://github.com/pydantic/pydantic/pull/12953) -* [@galuszkak](https://github.com/galuszkak) made their first contribution in [#12951](https://github.com/pydantic/pydantic/pull/12951) -* [@siewcapital](https://github.com/siewcapital) made their first contribution in [#12897](https://github.com/pydantic/pydantic/pull/12897) - -## v2.13.0b2 (2026-02-24) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.0b2) - -### What's Changed - -#### Fixes - -* Fix backported V1 namespace by [@Viicos](https://github.com/Viicos) in [#12855](https://github.com/pydantic/pydantic/pull/12855) -* Allow any type form to be used in `validate_as()` by [@bledden](https://github.com/bledden) in [#12846](https://github.com/pydantic/pydantic/pull/12846) -* Fix walrus operator precedence in `UrlConstraints.__get_pydantic_core_schema__()` by [@bysiber](https://github.com/bysiber) in [#12826](https://github.com/pydantic/pydantic/pull/12826) - -### New Contributors - -* [@bledden](https://github.com/bledden) made their first contribution in [#12846](https://github.com/pydantic/pydantic/pull/12846) -* [@bysiber](https://github.com/bysiber) made their first contribution in [#12827](https://github.com/pydantic/pydantic/pull/12827) - -## v2.13.0b1 (2026-02-23) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.13.0b1) - -This is the first beta release of the 2.13 version, mainly providing bug fixes and performance improvements -for validation and serialization. - -Notable changes include: - -* Add a new `polymorphic_serialization` option, solving issues with `serialize_as_any` introduced in 2.12. -* Latest V1.10.26 release under the `pydantic.v1` namespace. This version includes support for Python 3.14. -* The [`pydantic-core`](https://github.com/pydantic/pydantic-core/) repository was merged inside the main `pydantic` one. - -### What's Changed - -#### New Features - -* Add `polymorphic_serialization` option by [@davidhewitt](https://github.com/davidhewitt) in [#12518](https://github.com/pydantic/pydantic/pull/12518) -* Support Root models with `Literal` root types as discriminator field types by [@YassinNouh21](https://github.com/YassinNouh21) in [#12680](https://github.com/pydantic/pydantic/pull/12680) - -#### Changes - -* Migrate `pydantic-core` CI by [@Viicos](https://github.com/Viicos) in [#12752](https://github.com/pydantic/pydantic/pull/12752) -* Import `pydantic-core` into pydantic by [@davidhewitt](https://github.com/davidhewitt) in [#12481](https://github.com/pydantic/pydantic/pull/12481) -* Backport V1 changes up to v1.10.26 by [@Viicos](https://github.com/Viicos) in [#12663](https://github.com/pydantic/pydantic/pull/12663) -* Use the `complex()` constructor unconditionally when validating `complex` Python data by [@tanmaymunjal](https://github.com/tanmaymunjal) in [#12498](https://github.com/pydantic/pydantic/pull/12498) -* Add support for three-tuple input for `Decimal` by [@tanmaymunjal](https://github.com/tanmaymunjal) in [#12500](https://github.com/pydantic/pydantic/pull/12500) -* Align `@field_serializer` logic with `@field_validator` by [@Viicos](https://github.com/Viicos) in [#12577](https://github.com/pydantic/pydantic/pull/12577) -* Make `PydanticUserError` a `RuntimeError` instead of a `TypeError` by [@poliakovva](https://github.com/poliakovva) in [#12579](https://github.com/pydantic/pydantic/pull/12579) -* Remove redundant serialization attempts in nested unions by [@davidhewitt](https://github.com/davidhewitt) in [#12604](https://github.com/pydantic/pydantic/pull/12604) -* Copy `root` value when making root model shallow copies by [@YassinNouh21](https://github.com/YassinNouh21) in [#12679](https://github.com/pydantic/pydantic/pull/12679) -* Ensure deterministic JSON schema defaults by sorting sets by [@drshvik](https://github.com/drshvik) in [#12760](https://github.com/pydantic/pydantic/pull/12760) - -#### Performance - -* Refactor `DecoratorInfos.build()` implementation by [@Viicos](https://github.com/Viicos) in [#12536](https://github.com/pydantic/pydantic/pull/12536) -* Cache compiled regex in `pydantic-core` by [@Viicos](https://github.com/Viicos) in [#12549](https://github.com/pydantic/pydantic/pull/12549) -* Optimize creation of `Literal` validators by [@davidhewitt](https://github.com/davidhewitt) in [#12569](https://github.com/pydantic/pydantic/pull/12569) -* Optimize implementation of `LookupKey` by [@davidhewitt](https://github.com/davidhewitt) in [#12571](https://github.com/pydantic/pydantic/pull/12571) -* Use python strings for field names by [@davidhewitt](https://github.com/davidhewitt) in [#12631](https://github.com/pydantic/pydantic/pull/12631) -* Optimize datetime formatting code by [@davidhewitt](https://github.com/davidhewitt) in [#12626](https://github.com/pydantic/pydantic/pull/12626) -* Validate JSON model data by iteration by [@davidhewitt](https://github.com/davidhewitt) in [#12550](https://github.com/pydantic/pydantic/pull/12550) -* Optimize annotations evaluation of Pydantic models by [@Viicos](https://github.com/Viicos) in [#12681](https://github.com/pydantic/pydantic/pull/12681) -* Optimize `FieldInfo._copy()` by [@Viicos](https://github.com/Viicos) in [#12727](https://github.com/pydantic/pydantic/pull/12727) - -#### Fixes - -* Fix `FieldInfo` rebuilding when parameterizing generic models with an `Annotated` type by [@Viicos](https://github.com/Viicos) in [#12463](https://github.com/pydantic/pydantic/pull/12463) -* Fix nested model schema deduplication in JSON schema generation by [@marwan-alloreview](https://github.com/marwan-alloreview) in [#12494](https://github.com/pydantic/pydantic/pull/12494) -* Fix `InitVar` being ignored when using with the `pydantic.Field()` function by [@Viicos](https://github.com/Viicos) in [#12495](https://github.com/pydantic/pydantic/pull/12495) -* Fix support for enums with `NamedTuple` as values by [@Viicos](https://github.com/Viicos) in [#12506](https://github.com/pydantic/pydantic/pull/12506) -* Do not delete mock validator/serializer in `rebuild_dataclass()` by [@Viicos](https://github.com/Viicos) in [#12513](https://github.com/pydantic/pydantic/pull/12513) -* Require test suite to pass with free threading, switch back to global generic types cache by [@davidhewitt](https://github.com/davidhewitt) in [#12537](https://github.com/pydantic/pydantic/pull/12537) -* Refactor `__pydantic_extra__` annotation handling by [@Viicos](https://github.com/Viicos) in [#12563](https://github.com/pydantic/pydantic/pull/12563) -* Do not add claim of UUID "safety" provision by [@davidhewitt](https://github.com/davidhewitt) in [#12567](https://github.com/pydantic/pydantic/pull/12567) -* Use Python hash to perform lookup in tagged union serializer by [@davidhewitt](https://github.com/davidhewitt) in [#12594](https://github.com/pydantic/pydantic/pull/12594) -* Do not emit serialization warning `MISSING` sentinel is present in a nested model by [@Viicos](https://github.com/Viicos) in [#12635](https://github.com/pydantic/pydantic/pull/12635) -* Do not eagerly evaluate annotations in signature logic by [@Viicos](https://github.com/Viicos) in [#12660](https://github.com/pydantic/pydantic/pull/12660) -* Fix serialization of typed dict unions when `exclude_none` is set by [@davidhewitt](https://github.com/davidhewitt) in [#12677](https://github.com/pydantic/pydantic/pull/12677) -* Do not reuse prebuilt serializers/validators on rebuilds by [@lmmx](https://github.com/lmmx) in [#12689](https://github.com/pydantic/pydantic/pull/12689) -* Fix type annotation of `field_definitions` in `create_model()` by [@lehmann-hqs](https://github.com/lehmann-hqs) in [#12734](https://github.com/pydantic/pydantic/pull/12734) -* Fix incorrect dataclass constructor signature when overriding class `kw_only` with `Field()` by [@jfadia](https://github.com/jfadia) in [#12741](https://github.com/pydantic/pydantic/pull/12741) -* Use `typing.Union` when replacing types under Python 3.14 by [@Viicos](https://github.com/Viicos) in [#12733](https://github.com/pydantic/pydantic/pull/12733) -* Improve ImportString error when internal imports fail by [@tsembp](https://github.com/tsembp) in [#12740](https://github.com/pydantic/pydantic/pull/12740) -* Fix serializing complex numbers with negative zero imaginary part by [@lhnwrk](https://github.com/lhnwrk) in [#12770](https://github.com/pydantic/pydantic/pull/12770) -* Preserve custom docstrings on stdlib dataclasses in JSON schema by [@nightcityblade](https://github.com/nightcityblade) in [#12815](https://github.com/pydantic/pydantic/pull/12815) - -#### Packaging - -* Bump Rust url dependency from 2.5.4 to 2.5.7 in `pydantic-core` by [@dependabot](https://github.com/dependabot)[bot] in [#12508](https://github.com/pydantic/pydantic/pull/12508) -* Bump Rust minimum version to 1.88, use edition 2024 by [@davidhewitt](https://github.com/davidhewitt) and [@Viicos](https://github.com/Viicos) in [#12551](https://github.com/pydantic/pydantic/pull/12551) and [#12752](https://github.com/pydantic/pydantic/pull/12752) -* Bump PyO3 to 0.28, jiter to 0.13 by [@davidhewitt](https://github.com/davidhewitt) in [#12767](https://github.com/pydantic/pydantic/pull/12767) - -### New Contributors - -* [@marwan-alloreview](https://github.com/marwan-alloreview) made their first contribution in [#12494](https://github.com/pydantic/pydantic/pull/12494) -* [@tanmaymunjal](https://github.com/tanmaymunjal) made their first contribution in [#12498](https://github.com/pydantic/pydantic/pull/12498) -* [@poliakovva](https://github.com/poliakovva) made their first contribution in [#12579](https://github.com/pydantic/pydantic/pull/12579) -* [@lehmann-hqs](https://github.com/lehmann-hqs) made their first contribution in [#12734](https://github.com/pydantic/pydantic/pull/12734) -* [@jfadia](https://github.com/jfadia) made their first contribution in [#12741](https://github.com/pydantic/pydantic/pull/12741) -* [@tsembp](https://github.com/tsembp) made their first contribution in [#12740](https://github.com/pydantic/pydantic/pull/12740) -* [@drshvik](https://github.com/drshvik) made their first contribution in [#12760](https://github.com/pydantic/pydantic/pull/12760) -* [@lhnwrk](https://github.com/lhnwrk) made their first contribution in [#12770](https://github.com/pydantic/pydantic/pull/12770) -* [@nightcityblade](https://github.com/nightcityblade) made their first contribution in [#12815](https://github.com/pydantic/pydantic/pull/12811) - -## v2.12.5 (2025-11-26) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.5) - -This is the fifth 2.12 patch release, addressing an issue with the `MISSING` sentinel and providing several documentation improvements. - -The next 2.13 minor release will be published in a couple weeks, and will include a new *polymorphic serialization* feature addressing -the remaining unexpected changes to the *serialize as any* behavior. - -* Fix pickle error when using `model_construct()` on a model with `MISSING` as a default value by [@ornariece](https://github.com/ornariece) in [#12522](https://github.com/pydantic/pydantic/pull/12522). -* Several updates to the documentation by [@Viicos](https://github.com/Viicos). - -## v2.12.4 (2025-11-05) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.4) - -This is the fourth 2.12 patch release, fixing more regressions, and reverting a change in the `build()` method -of the [`AnyUrl` and Dsn types](https://docs.pydantic.dev/latest/api/networks/). - -This patch release also fixes an issue with the serialization of IP address types, when `serialize_as_any` is used. The next patch release -will try to address the remaining issues with *serialize as any* behavior by introducing a new *polymorphic serialization* feature, that -should be used in most cases in place of *serialize as any*. - -* Fix issue with forward references in parent `TypedDict` classes by [@Viicos](https://github.com/Viicos) in [#12427](https://github.com/pydantic/pydantic/pull/12427). - - This issue is only relevant on Python 3.14 and greater. - -* Exclude fields with `exclude_if` from JSON Schema required fields by [@Viicos](https://github.com/Viicos) in [#12430](https://github.com/pydantic/pydantic/pull/12430) -* Revert URL percent-encoding of credentials in the `build()` method - of the [`AnyUrl` and Dsn types](https://docs.pydantic.dev/latest/api/networks/) by [@davidhewitt](https://github.com/davidhewitt) in - [pydantic-core#1833](https://github.com/pydantic/pydantic-core/pull/1833). - - This was initially considered as a bugfix, but caused regressions and as such was fully reverted. The next release will include - an opt-in option to percent-encode components of the URL. - -* Add type inference for IP address types by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1868](https://github.com/pydantic/pydantic-core/pull/1868). - - The 2.12 changes to the `serialize_as_any` behavior made it so that IP address types could not properly serialize to JSON. - -* Avoid getting default values from defaultdict by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1853](https://github.com/pydantic/pydantic-core/pull/1853). - - This fixes a subtle regression in the validation behavior of the [`collections.defaultdict`](https://docs.python.org/3/library/collections.html#collections.defaultdict) - type. - -* Fix issue with field serializers on nested typed dictionaries by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1879](https://github.com/pydantic/pydantic-core/pull/1879). -* Add more `pydantic-core` builds for the three-threaded version of Python 3.14 by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1864](https://github.com/pydantic/pydantic-core/pull/1864). - -## v2.12.3 (2025-10-17) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.3) - -### What's Changed - -This is the third 2.12 patch release, fixing issues related to the `FieldInfo` class, and reverting a change to the supported -[*after* model validator](https://docs.pydantic.dev/latest/concepts/validators/#model-validators) function signatures. - -* Raise a warning when an invalid after model validator function signature is raised by [@Viicos](https://github.com/Viicos) in [#12414](https://github.com/pydantic/pydantic/pull/12414). - Starting in 2.12.0, using class methods for *after* model validators raised an error, but the error wasn't raised concistently. We decided - to emit a deprecation warning instead. -* Add [`FieldInfo.asdict()`](https://docs.pydantic.dev/latest/api/fields/#pydantic.fields.FieldInfo.asdict) method, improve documentation around `FieldInfo` by [@Viicos](https://github.com/Viicos) in [#12411](https://github.com/pydantic/pydantic/pull/12411). - This also add back support for mutations on `FieldInfo` classes, that are reused as `Annotated` metadata. **However**, note that this is still - *not* a supported pattern. Instead, please refer to the [added example](https://docs.pydantic.dev/latest/examples/dynamic_models/) in the documentation. - -The [blog post](https://pydantic.dev/articles/pydantic-v2-12-release#changes) section on changes was also updated to document the changes related to `serialize_as_any`. - -## v2.12.2 (2025-10-14) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.2) - -### What's Changed - -#### Fixes - -* Release a new `pydantic-core` version, as a corrupted CPython 3.10 `manylinux2014_aarch64` wheel got uploaded ([pydantic-core#1843](https://github.com/pydantic/pydantic-core/pull/1843)). -* Fix issue with recursive generic models with a parent model class by [@Viicos](https://github.com/Viicos) in [#12398](https://github.com/pydantic/pydantic/pull/12398) - -## v2.12.1 (2025-10-13) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.1) - -### What's Changed - -This is the first 2.12 patch release, addressing most (but not all yet) regressions from the initial 2.12.0 release. - -#### Fixes - -* Do not evaluate annotations when inspecting validators and serializers by [@Viicos](https://github.com/Viicos) in [#12355](https://github.com/pydantic/pydantic/pull/12355) -* Make sure `None` is converted as `NoneType` in Python 3.14 by [@Viicos](https://github.com/Viicos) in [#12370](https://github.com/pydantic/pydantic/pull/12370) -* Backport V1 runtime warning when using Python 3.14 by [@Viicos](https://github.com/Viicos) in [#12367](https://github.com/pydantic/pydantic/pull/12367) -* Fix error message for invalid validator signatures by [@Viicos](https://github.com/Viicos) in [#12366](https://github.com/pydantic/pydantic/pull/12366) -* Populate field name in `ValidationInfo` for validation of default value by [@Viicos](https://github.com/Viicos) in [pydantic-core#1826](https://github.com/pydantic/pydantic-core/pull/1826) -* Encode credentials in `MultiHostUrl` builder by [@willswire](https://github.com/willswire) in [pydantic-core#1829](https://github.com/pydantic/pydantic-core/pull/1829) -* Respect field serializers when using `serialize_as_any` serialization flag by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1829](https://github.com/pydantic/pydantic-core/pull/1829) -* Fix various `RootModel` serialization issues by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1836](https://github.com/pydantic/pydantic-core/pull/1836) - -### New Contributors - -* [@willswire](https://github.com/willswire) made their first contribution in [pydantic-core#1829](https://github.com/pydantic/pydantic-core/pull/1829) - -## v2.12.0 (2025-10-07) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.0) - -This is the final 2.12 release. It features the work of 20 external contributors and provides useful new features, along with initial Python 3.14 support. -Several minor changes (considered non-breaking changes according to our [versioning policy](https://docs.pydantic.dev/2.12/version-policy/#pydantic-v2)) -are also included in this release. Make sure to look into them before upgrading. - -**Note that Pydantic V1 is not compatible with Python 3.14 and greater**. - -### What's Changed - -See the beta releases for all changes sinces 2.11. - -#### New Features - -* Add `extra` parameter to the validate functions by [@anvilpete](https://github.com/anvilpete) in [#12233](https://github.com/pydantic/pydantic/pull/12233) -* Add `exclude_computed_fields` serialization option by [@Viicos](https://github.com/Viicos) in [#12334](https://github.com/pydantic/pydantic/pull/12334) -* Add `preverse_empty_path` URL options by [@Viicos](https://github.com/Viicos) in [#12336](https://github.com/pydantic/pydantic/pull/12336) -* Add `union_format` parameter to JSON Schema generation by [@Viicos](https://github.com/Viicos) in [#12147](https://github.com/pydantic/pydantic/pull/12147) -* Add `__qualname__` parameter for `create_model` by [@Atry](https://github.com/Atry) in [#12001](https://github.com/pydantic/pydantic/pull/12001) - -#### Fixes - -* Do not try to infer name from lambda definitions in pipelines API by [@Viicos](https://github.com/Viicos) in [#12289](https://github.com/pydantic/pydantic/pull/12289) -* Use proper namespace for functions in `TypeAdapter` by [@Viicos](https://github.com/Viicos) in [#12324](https://github.com/pydantic/pydantic/pull/12324) -* Use `Any` for context type annotation in `TypeAdapter` by [@inducer](https://github.com/inducer) in [#12279](https://github.com/pydantic/pydantic/pull/12279) -* Expose `FieldInfo` in `pydantic.fields.__all__` by [@Viicos](https://github.com/Viicos) in [#12339](https://github.com/pydantic/pydantic/pull/12339) -* Respect `validation_alias` in `@validate_call` by [@Viicos](https://github.com/Viicos) in [#12340](https://github.com/pydantic/pydantic/pull/12340) -* Use `Any` as context annotation in plugin API by [@Viicos](https://github.com/Viicos) in [#12341](https://github.com/pydantic/pydantic/pull/12341) -* Use proper `stacklevel` in warnings when possible by [@Viicos](https://github.com/Viicos) in [#12342](https://github.com/pydantic/pydantic/pull/12342) - -#### Packaging - -* Update V1 copy to v1.10.24 by [@Viicos](https://github.com/Viicos) in [#12338](https://github.com/pydantic/pydantic/pull/12338) - -### New Contributors - -* [@anvilpete](https://github.com/anvilpete) made their first contribution in [#12233](https://github.com/pydantic/pydantic/pull/12233) -* [@JonathanWindell](https://github.com/JonathanWindell) made their first contribution in [#12327](https://github.com/pydantic/pydantic/pull/12327) -* [@inducer](https://github.com/inducer) made their first contribution in [#12279](https://github.com/pydantic/pydantic/pull/12279) -* [@Atry](https://github.com/Atry) made their first contribution in [#12001](https://github.com/pydantic/pydantic/pull/12001) - -## v2.12.0b1 (2025-10-03) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.0b1) - -This is the first beta release of the upcoming 2.12 release. - -### What's Changed - -#### New Features - -* Add support for `exclude_if` at the field level by [@andresliszt](https://github.com/andresliszt) in [#12141](https://github.com/pydantic/pydantic/pull/12141) -* Add `ValidateAs` annotation helper by [@Viicos](https://github.com/Viicos) in [#11942](https://github.com/pydantic/pydantic/pull/11942) -* Add configuration options for validation and JSON serialization of temporal types by [@ollz272](https://github.com/ollz272) in [#12068](https://github.com/pydantic/pydantic/pull/12068) -* Add support for PEP 728 by [@Viicos](https://github.com/Viicos) in [#12179](https://github.com/pydantic/pydantic/pull/12179) -* Add field name in serialization error by [@NicolasPllr1](https://github.com/NicolasPllr1) in [pydantic-core#1799](https://github.com/pydantic/pydantic-core/pull/1799) -* Add option to preserve empty URL paths by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1789](https://github.com/pydantic/pydantic-core/pull/1789) - -#### Changes - -* Raise error if an incompatible `pydantic-core` version is installed by [@Viicos](https://github.com/Viicos) in [#12196](https://github.com/pydantic/pydantic/pull/12196) -* Remove runtime warning for experimental features by [@Viicos](https://github.com/Viicos) in [#12265](https://github.com/pydantic/pydantic/pull/12265) -* Warn if registering virtual subclasses on Pydantic models by [@Viicos](https://github.com/Viicos) in [#11669](https://github.com/pydantic/pydantic/pull/11669) - -#### Fixes - -* Fix `__getattr__()` behavior on Pydantic models when a property raised an `AttributeError` and extra values are present by [@raspuchin](https://github.com/raspuchin) in [#12106](https://github.com/pydantic/pydantic/pull/12106) -* Add test to prevent regression with Pydantic models used as annotated metadata by [@Viicos](https://github.com/Viicos) in [#12133](https://github.com/pydantic/pydantic/pull/12133) -* Allow to use property setters on Pydantic dataclasses with `validate_assignment` set by [@Viicos](https://github.com/Viicos) in [#12173](https://github.com/pydantic/pydantic/pull/12173) -* Fix mypy v2 plugin for upcoming mypy release by [@cdce8p](https://github.com/cdce8p) in [#12209](https://github.com/pydantic/pydantic/pull/12209) -* Respect custom title in functions JSON Schema by [@Viicos](https://github.com/Viicos) in [#11892](https://github.com/pydantic/pydantic/pull/11892) -* Fix `ImportString` JSON serialization for objects with a `name` attribute by [@chr1sj0nes](https://github.com/chr1sj0nes) in [#12219](https://github.com/pydantic/pydantic/pull/12219) -* Do not error on fields overridden by methods in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#12290](https://github.com/pydantic/pydantic/pull/12290) - -#### Packaging - -* Bump `pydantic-core` to v2.40.1 by [@Viicos](https://github.com/Viicos) in [#12314](https://github.com/pydantic/pydantic/pull/12314) - -### New Contributors - -* [@raspuchin](https://github.com/raspuchin) made their first contribution in [#12106](https://github.com/pydantic/pydantic/pull/12106) -* [@chr1sj0nes](https://github.com/chr1sj0nes) made their first contribution in [#12219](https://github.com/pydantic/pydantic/pull/12219) - -## v2.12.0a1 (2025-07-26) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.12.0a1) - -This is the first alpha release of the upcoming 2.12 release, which adds initial support for Python 3.14. - -### What's Changed - -#### New Features - -* Add `__pydantic_on_complete__()` hook that is called once model is fully ready to be used by [@DouweM](https://github.com/DouweM) in [#11762](https://github.com/pydantic/pydantic/pull/11762) -* Add initial support for Python 3.14 by [@Viicos](https://github.com/Viicos) in [#11991](https://github.com/pydantic/pydantic/pull/11991) -* Add regex patterns to JSON schema for `Decimal` type by [@Dima-Bulavenko](https://github.com/Dima-Bulavenko) in [#11987](https://github.com/pydantic/pydantic/pull/11987) -* Add support for `doc` attribute on dataclass fields by [@Viicos](https://github.com/Viicos) in [#12077](https://github.com/pydantic/pydantic/pull/12077) -* Add experimental `MISSING` sentinel by [@Viicos](https://github.com/Viicos) in [#11883](https://github.com/pydantic/pydantic/pull/11883) - -#### Changes - -* Allow config and bases to be specified together in `create_model()` by [@Viicos](https://github.com/Viicos) in [#11714](https://github.com/pydantic/pydantic/pull/11714) -* Move some field logic out of the `GenerateSchema` class by [@Viicos](https://github.com/Viicos) in [#11733](https://github.com/pydantic/pydantic/pull/11733) -* Always make use of `inspect.getsourcelines()` for docstring extraction on Python 3.13 and greater by [@Viicos](https://github.com/Viicos) in [#11829](https://github.com/pydantic/pydantic/pull/11829) -* Only support the latest Mypy version by [@Viicos](https://github.com/Viicos) in [#11832](https://github.com/pydantic/pydantic/pull/11832) -* Do not implicitly convert after model validators to class methods by [@Viicos](https://github.com/Viicos) in [#11957](https://github.com/pydantic/pydantic/pull/11957) -* Refactor `FieldInfo` creation implementation by [@Viicos](https://github.com/Viicos) in [#11898](https://github.com/pydantic/pydantic/pull/11898) -* Make `Secret` covariant by [@bluenote10](https://github.com/bluenote10) in [#12008](https://github.com/pydantic/pydantic/pull/12008) -* Emit warning when field-specific metadata is used in invalid contexts by [@Viicos](https://github.com/Viicos) in [#12028](https://github.com/pydantic/pydantic/pull/12028) - -#### Fixes - -* Properly fetch plain serializer function when serializing default value in JSON Schema by [@Viicos](https://github.com/Viicos) in [#11721](https://github.com/pydantic/pydantic/pull/11721) -* Remove generics cache workaround by [@Viicos](https://github.com/Viicos) in [#11755](https://github.com/pydantic/pydantic/pull/11755) -* Remove coercion of decimal constraints by [@Viicos](https://github.com/Viicos) in [#11772](https://github.com/pydantic/pydantic/pull/11772) -* Fix crash when expanding root type in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#11735](https://github.com/pydantic/pydantic/pull/11735) -* Only mark model as complete once all fields are complete by [@DouweM](https://github.com/DouweM) in [#11759](https://github.com/pydantic/pydantic/pull/11759) -* Do not provide `field_name` in validator core schemas by [@DouweM](https://github.com/DouweM) in [#11761](https://github.com/pydantic/pydantic/pull/11761) -* Fix issue with recursive generic models by [@Viicos](https://github.com/Viicos) in [#11775](https://github.com/pydantic/pydantic/pull/11775) -* Fix qualified name comparison of private attributes during namespace inspection by [@karta9821](https://github.com/karta9821) in [#11803](https://github.com/pydantic/pydantic/pull/11803) -* Make sure Pydantic dataclasses with slots and `validate_assignment` can be unpickled by [@Viicos](https://github.com/Viicos) in [#11769](https://github.com/pydantic/pydantic/pull/11769) -* Traverse `function-before` schemas during schema gathering by [@Viicos](https://github.com/Viicos) in [#11801](https://github.com/pydantic/pydantic/pull/11801) -* Fix check for stdlib dataclasses by [@Viicos](https://github.com/Viicos) in [#11822](https://github.com/pydantic/pydantic/pull/11822) -* Check if `FieldInfo` is complete after applying type variable map by [@Viicos](https://github.com/Viicos) in [#11855](https://github.com/pydantic/pydantic/pull/11855) -* Do not delete mock validator/serializer in `model_rebuild()` by [@Viicos](https://github.com/Viicos) in [#11890](https://github.com/pydantic/pydantic/pull/11890) -* Rebuild dataclass fields before schema generation by [@Viicos](https://github.com/Viicos) in [#11949](https://github.com/pydantic/pydantic/pull/11949) -* Always store the original field assignment on `FieldInfo` by [@Viicos](https://github.com/Viicos) in [#11946](https://github.com/pydantic/pydantic/pull/11946) -* Do not use deprecated methods as default field values by [@Viicos](https://github.com/Viicos) in [#11914](https://github.com/pydantic/pydantic/pull/11914) -* Allow callable discriminator to be applied on PEP 695 type aliases by [@Viicos](https://github.com/Viicos) in [#11941](https://github.com/pydantic/pydantic/pull/11941) -* Suppress core schema generation warning when using `SkipValidation` by [@ygsh0816](https://github.com/ygsh0816) in [#12002](https://github.com/pydantic/pydantic/pull/12002) -* Do not emit typechecking error for invalid `Field()` default with `validate_default` set to `True` by [@Viicos](https://github.com/Viicos) in [#11988](https://github.com/pydantic/pydantic/pull/11988) -* Refactor logic to support Pydantic's `Field()` function in dataclasses by [@Viicos](https://github.com/Viicos) in [#12051](https://github.com/pydantic/pydantic/pull/12051) - -#### Packaging - -* Update project metadata to use PEP 639 by [@Viicos](https://github.com/Viicos) in [#11694](https://github.com/pydantic/pydantic/pull/11694) -* Bump `mkdocs-llmstxt` to v0.2.0 by [@Viicos](https://github.com/Viicos) in [#11725](https://github.com/pydantic/pydantic/pull/11725) -* Bump `pydantic-core` to v2.35.1 by [@Viicos](https://github.com/Viicos) in [#11963](https://github.com/pydantic/pydantic/pull/11963) -* Bump dawidd6/action-download-artifact from 10 to 11 by [@dependabot](https://github.com/dependabot)[bot] in [#12033](https://github.com/pydantic/pydantic/pull/12033) -* Bump astral-sh/setup-uv from 5 to 6 by [@dependabot](https://github.com/dependabot)[bot] in [#11826](https://github.com/pydantic/pydantic/pull/11826) -* Update mypy to 1.17.0 by [@Viicos](https://github.com/Viicos) in [#12076](https://github.com/pydantic/pydantic/pull/12076) - -### New Contributors - -* [@parth-paradkar](https://github.com/parth-paradkar) made their first contribution in [#11695](https://github.com/pydantic/pydantic/pull/11695) -* [@dqkqd](https://github.com/dqkqd) made their first contribution in [#11739](https://github.com/pydantic/pydantic/pull/11739) -* [@fhightower](https://github.com/fhightower) made their first contribution in [#11722](https://github.com/pydantic/pydantic/pull/11722) -* [@gbaian10](https://github.com/gbaian10) made their first contribution in [#11766](https://github.com/pydantic/pydantic/pull/11766) -* [@DouweM](https://github.com/DouweM) made their first contribution in [#11759](https://github.com/pydantic/pydantic/pull/11759) -* [@bowenliang123](https://github.com/bowenliang123) made their first contribution in [#11719](https://github.com/pydantic/pydantic/pull/11719) -* [@rawwar](https://github.com/rawwar) made their first contribution in [#11799](https://github.com/pydantic/pydantic/pull/11799) -* [@karta9821](https://github.com/karta9821) made their first contribution in [#11803](https://github.com/pydantic/pydantic/pull/11803) -* [@jinnovation](https://github.com/jinnovation) made their first contribution in [#11834](https://github.com/pydantic/pydantic/pull/11834) -* [@zmievsa](https://github.com/zmievsa) made their first contribution in [#11861](https://github.com/pydantic/pydantic/pull/11861) -* [@Otto-AA](https://github.com/Otto-AA) made their first contribution in [#11860](https://github.com/pydantic/pydantic/pull/11860) -* [@ygsh0816](https://github.com/ygsh0816) made their first contribution in [#12002](https://github.com/pydantic/pydantic/pull/12002) -* [@lukland](https://github.com/lukland) made their first contribution in [#12015](https://github.com/pydantic/pydantic/pull/12015) -* [@Dima-Bulavenko](https://github.com/Dima-Bulavenko) made their first contribution in [#11987](https://github.com/pydantic/pydantic/pull/11987) -* [@GSemikozov](https://github.com/GSemikozov) made their first contribution in [#12050](https://github.com/pydantic/pydantic/pull/12050) -* [@hannah-heywa](https://github.com/hannah-heywa) made their first contribution in [#12082](https://github.com/pydantic/pydantic/pull/12082) - -## v2.11.10 (2025-10-04) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.10) - -### What's Changed - -#### Fixes - -* Backport v1.10.24 changes by [@Viicos](https://github.com/Viicos) - -## v2.11.9 (2025-09-13) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.9) - -### What's Changed - -#### Fixes - -* Backport v1.10.23 changes by [@Viicos](https://github.com/Viicos) - -## v2.11.8 (2025-09-13) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.8) - -### What's Changed - -#### Fixes - -* Fix mypy plugin for mypy 1.18 by [@cdce8p](https://github.com/cdce8p) in [#12209](https://github.com/pydantic/pydantic/pull/12209) - -## v2.11.7 (2025-06-14) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.7) - -### What's Changed - -#### Fixes - -* Copy `FieldInfo` instance if necessary during `FieldInfo` build by [@Viicos](https://github.com/Viicos) in [#11898](https://github.com/pydantic/pydantic/pull/11898) - -## v2.11.6 (2025-06-13) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.6) - -### What's Changed - -#### Fixes - -* Rebuild dataclass fields before schema generation by [@Viicos](https://github.com/Viicos) in [#11949](https://github.com/pydantic/pydantic/pull/11949) -* Always store the original field assignment on `FieldInfo` by [@Viicos](https://github.com/Viicos) in [#11946](https://github.com/pydantic/pydantic/pull/11946) - -## v2.11.5 (2025-05-22) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.5) - -### What's Changed - -#### Fixes - -* Check if `FieldInfo` is complete after applying type variable map by [@Viicos](https://github.com/Viicos) in [#11855](https://github.com/pydantic/pydantic/pull/11855) -* Do not delete mock validator/serializer in `model_rebuild()` by [@Viicos](https://github.com/Viicos) in [#11890](https://github.com/pydantic/pydantic/pull/11890) -* Do not duplicate metadata on model rebuild by [@Viicos](https://github.com/Viicos) in [#11902](https://github.com/pydantic/pydantic/pull/11902) - -## v2.11.4 (2025-04-29) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.4) - -### What's Changed - -#### Changes - -* Allow config and bases to be specified together in `create_model()` by [@Viicos](https://github.com/Viicos) in [#11714](https://github.com/pydantic/pydantic/pull/11714). - This change was backported as it was previously possible (although not meant to be supported) - to provide `model_config` as a field, which would make it possible to provide both configuration - and bases. - -#### Fixes - -* Remove generics cache workaround by [@Viicos](https://github.com/Viicos) in [#11755](https://github.com/pydantic/pydantic/pull/11755) -* Remove coercion of decimal constraints by [@Viicos](https://github.com/Viicos) in [#11772](https://github.com/pydantic/pydantic/pull/11772) -* Fix crash when expanding root type in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#11735](https://github.com/pydantic/pydantic/pull/11735) -* Fix issue with recursive generic models by [@Viicos](https://github.com/Viicos) in [#11775](https://github.com/pydantic/pydantic/pull/11775) -* Traverse `function-before` schemas during schema gathering by [@Viicos](https://github.com/Viicos) in [#11801](https://github.com/pydantic/pydantic/pull/11801) - -#### Packaging - -* Bump `mkdocs-llmstxt` to v0.2.0 by [@Viicos](https://github.com/Viicos) in [#11725](https://github.com/pydantic/pydantic/pull/11725) - -## v2.11.3 (2025-04-08) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.3) - -### What's Changed - -#### Fixes - -* Preserve field description when rebuilding model fields by [@Viicos](https://github.com/Viicos) in [#11698](https://github.com/pydantic/pydantic/pull/11698) - -#### Packaging - -* Update V1 copy to v1.10.21 by [@Viicos](https://github.com/Viicos) in [#11706](https://github.com/pydantic/pydantic/pull/11706) - -## v2.11.2 (2025-04-03) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.2) - -### What's Changed - -#### Fixes - -* Bump `pydantic-core` to v2.33.1 by [@Viicos](https://github.com/Viicos) in [#11678](https://github.com/pydantic/pydantic/pull/11678) -* Make sure `__pydantic_private__` exists before setting private attributes by [@Viicos](https://github.com/Viicos) in [#11666](https://github.com/pydantic/pydantic/pull/11666) -* Do not override `FieldInfo._complete` when using field from parent class by [@Viicos](https://github.com/Viicos) in [#11668](https://github.com/pydantic/pydantic/pull/11668) -* Provide the available definitions when applying discriminated unions by [@Viicos](https://github.com/Viicos) in [#11670](https://github.com/pydantic/pydantic/pull/11670) -* Do not expand root type in the mypy plugin for variables by [@Viicos](https://github.com/Viicos) in [#11676](https://github.com/pydantic/pydantic/pull/11676) -* Mention the attribute name in model fields deprecation message by [@Viicos](https://github.com/Viicos) in [#11674](https://github.com/pydantic/pydantic/pull/11674) -* Properly validate parameterized mappings by [@Viicos](https://github.com/Viicos) in [#11658](https://github.com/pydantic/pydantic/pull/11658) - -## v2.11.1 (2025-03-28) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.1) - -### What's Changed - -#### Fixes - -* Do not override `'definitions-ref'` schemas containing serialization schemas or metadata by [@Viicos](https://github.com/Viicos) in [#11644](https://github.com/pydantic/pydantic/pull/11644) - -## v2.11.0 (2025-03-27) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.0) - -### What's Changed - -Pydantic v2.11 is a version strongly focused on build time performance of Pydantic models (and core schema generation in general). -See the [blog post](https://pydantic.dev/articles/pydantic-v2-11-release) for more details. - -#### New Features - -* Add `encoded_string()` method to the URL types by [@YassinNouh21](https://github.com/YassinNouh21) in [#11580](https://github.com/pydantic/pydantic/pull/11580) -* Add support for `defer_build` with `@validate_call` decorator by [@Viicos](https://github.com/Viicos) in [#11584](https://github.com/pydantic/pydantic/pull/11584) -* Allow `@with_config` decorator to be used with keyword arguments by [@Viicos](https://github.com/Viicos) in [#11608](https://github.com/pydantic/pydantic/pull/11608) -* Simplify customization of default value inclusion in JSON Schema generation by [@Viicos](https://github.com/Viicos) in [#11634](https://github.com/pydantic/pydantic/pull/11634) -* Add `generate_arguments_schema()` function by [@Viicos](https://github.com/Viicos) in [#11572](https://github.com/pydantic/pydantic/pull/11572) - -#### Fixes - -* Allow generic typed dictionaries to be used for unpacked variadic keyword parameters by [@Viicos](https://github.com/Viicos) in [#11571](https://github.com/pydantic/pydantic/pull/11571) -* Fix runtime error when computing model string representation involving cached properties and self-referenced models by [@Viicos](https://github.com/Viicos) in [#11579](https://github.com/pydantic/pydantic/pull/11579) -* Preserve other steps when using the ellipsis in the pipeline API by [@Viicos](https://github.com/Viicos) in [#11626](https://github.com/pydantic/pydantic/pull/11626) -* Fix deferred discriminator application logic by [@Viicos](https://github.com/Viicos) in [#11591](https://github.com/pydantic/pydantic/pull/11591) - -#### Packaging - -* Bump `pydantic-core` to v2.33.0 by [@Viicos](https://github.com/Viicos) in [#11631](https://github.com/pydantic/pydantic/pull/11631) - -### New Contributors - -* [@cmenon12](https://github.com/cmenon12) made their first contribution in [#11562](https://github.com/pydantic/pydantic/pull/11562) -* [@Jeukoh](https://github.com/Jeukoh) made their first contribution in [#11611](https://github.com/pydantic/pydantic/pull/11611) - -## v2.11.0b2 (2025-03-17) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.0b2) - -### What's Changed - -#### New Features - -* Add experimental support for free threading by [@Viicos](https://github.com/Viicos) in [#11516](https://github.com/pydantic/pydantic/pull/11516) - -#### Fixes - -* Fix `NotRequired` qualifier not taken into account in stringified annotation by [@Viicos](https://github.com/Viicos) in [#11559](https://github.com/pydantic/pydantic/pull/11559) - -#### Packaging - -* Bump `pydantic-core` to v2.32.0 by [@Viicos](https://github.com/Viicos) in [#11567](https://github.com/pydantic/pydantic/pull/11567) - -### New Contributors - -* [@joren485](https://github.com/joren485) made their first contribution in [#11547](https://github.com/pydantic/pydantic/pull/11547) - -## v2.11.0b1 (2025-03-06) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.0b1) - -### What's Changed - -#### New Features - -* Support unsubstituted type variables with both a default and a bound or constraints by [@FyZzyss](https://github.com/FyZzyss) in https://github.com/pydantic/pydantic/pull/10789 -* Add a `default_factory_takes_validated_data` property to `FieldInfo` by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11034 -* Raise a better error when a generic alias is used inside `type[]` by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11088 -* Properly support PEP 695 generics syntax by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11189 -* Properly support type variable defaults by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11332 -* Add support for validating v6, v7, v8 UUIDs by [@astei](https://github.com/astei) in https://github.com/pydantic/pydantic/pull/11436 -* Improve alias configuration APIs by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11468 - -#### Changes - -* Rework `create_model` field definitions format by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11032 -* Raise a deprecation warning when a field is annotated as final with a default value by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11168 -* Deprecate accessing `model_fields` and `model_computed_fields` on instances by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11169 -* **Breaking Change:** Move core schema generation logic for path types inside the `GenerateSchema` class by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/10846 -* Remove Python 3.8 Support by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11258 -* Optimize calls to `get_type_ref` by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/10863 -* Disable `pydantic-core` core schema validation by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11271 - -#### Performance - -* Only evaluate `FieldInfo` annotations if required during schema building by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/10769 -* Improve `__setattr__` performance of Pydantic models by caching setter functions by [@MarkusSintonen](https://github.com/MarkusSintonen) in https://github.com/pydantic/pydantic/pull/10868 -* Improve annotation application performance by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11186 -* Improve performance of `_typing_extra` module by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11255 -* Refactor and optimize schema cleaning logic by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11244 -* Create a single dictionary when creating a `CoreConfig` instance by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11384 -* Bump `pydantic-core` and thus use `SchemaValidator` and `SchemaSerializer` caching by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11402 -* Reuse cached core schemas for parametrized generic Pydantic models by [@MarkusSintonen](https://github.com/MarkusSintonen) in https://github.com/pydantic/pydantic/pull/11434 - -#### Fixes - -* Improve `TypeAdapter` instance repr by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/10872 -* Use the correct frame when instantiating a parametrized `TypeAdapter` by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/10893 -* Infer final fields with a default value as class variables in the mypy plugin by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11121 -* Recursively unpack `Literal` values if using PEP 695 type aliases by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11114 -* Override `__subclasscheck__` on `ModelMetaclass` to avoid memory leak and performance issues by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11116 -* Remove unused `_extract_get_pydantic_json_schema()` parameter by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11155 -* Improve discriminated union error message for invalid union variants by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11161 -* Unpack PEP 695 type aliases if using the `Annotated` form by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11109 -* Add missing stacklevel in `deprecated_instance_property` warning by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11200 -* Copy `WithJsonSchema` schema to avoid sharing mutated data by [@thejcannon](https://github.com/thejcannon) in https://github.com/pydantic/pydantic/pull/11014 -* Do not cache parametrized models when in the process of parametrizing another model by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/10704 -* Add discriminated union related metadata entries to the `CoreMetadata` definition by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11216 -* Consolidate schema definitions logic in the `_Definitions` class by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11208 -* Support initializing root model fields with values of the `root` type in the mypy plugin by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11212 -* Fix various issues with dataclasses and `use_attribute_docstrings` by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11246 -* Only compute normalized decimal places if necessary in `decimal_places_validator` by [@misrasaurabh1](https://github.com/misrasaurabh1) in https://github.com/pydantic/pydantic/pull/11281 -* Add support for `validation_alias` in the mypy plugin by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11295 -* Fix JSON Schema reference collection with `"examples"` keys by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11305 -* Do not transform model serializer functions as class methods in the mypy plugin by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11298 -* Simplify `GenerateJsonSchema.literal_schema()` implementation by [@misrasaurabh1](https://github.com/misrasaurabh1) in https://github.com/pydantic/pydantic/pull/11321 -* Add additional allowed schemes for `ClickHouseDsn` by [@Maze21127](https://github.com/Maze21127) in https://github.com/pydantic/pydantic/pull/11319 -* Coerce decimal constraints to `Decimal` instances by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11350 -* Use the correct JSON Schema mode when handling function schemas by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11367 -* Improve exception message when encountering recursion errors during type evaluation by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11356 -* Always include `additionalProperties: True` for arbitrary dictionary schemas by [@austinyu](https://github.com/austinyu) in https://github.com/pydantic/pydantic/pull/11392 -* Expose `fallback` parameter in serialization methods by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11398 -* Fix path serialization behavior by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11416 -* Do not reuse validators and serializers during model rebuild by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11429 -* Collect model fields when rebuilding a model by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11388 -* Allow cached properties to be altered on frozen models by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11432 -* Fix tuple serialization for `Sequence` types by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11435 -* Fix: do not check for `__get_validators__` on classes where `__get_pydantic_core_schema__` is also defined by [@tlambert03](https://github.com/tlambert03) in https://github.com/pydantic/pydantic/pull/11444 -* Allow callable instances to be used as serializers by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11451 -* Improve error thrown when overriding field with a property by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11459 -* Fix JSON Schema generation with referenceable core schemas holding JSON metadata by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11475 -* Support strict specification on union member types by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11481 -* Implicitly set `validate_by_name` to `True` when `validate_by_alias` is `False` by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11503 -* Change type of `Any` when synthesizing `BaseSettings.__init__` signature in the mypy plugin by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11497 -* Support type variable defaults referencing other type variables by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11520 -* Fix `ValueError` on year zero by [@davidhewitt](https://github.com/davidhewitt) in https://github.com/pydantic/pydantic-core/pull/1583 -* `dataclass` `InitVar` shouldn't be required on serialization by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic-core/pull/1602 - -#### Packaging - -* Add a `check_pydantic_core_version()` function by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11324 -* Remove `greenlet` development dependency by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11351 -* Use the `typing-inspection` library by [@Viicos](https://github.com/Viicos) in https://github.com/pydantic/pydantic/pull/11479 -* Bump `pydantic-core` to `v2.31.1` by [@sydney-runkle](https://github.com/sydney-runkle) in https://github.com/pydantic/pydantic/pull/11526 - -## New Contributors - -* [@FyZzyss](https://github.com/FyZzyss) made their first contribution in https://github.com/pydantic/pydantic/pull/10789 -* [@tamird](https://github.com/tamird) made their first contribution in https://github.com/pydantic/pydantic/pull/10948 -* [@felixxm](https://github.com/felixxm) made their first contribution in https://github.com/pydantic/pydantic/pull/11077 -* [@alexprabhat99](https://github.com/alexprabhat99) made their first contribution in https://github.com/pydantic/pydantic/pull/11082 -* [@Kharianne](https://github.com/Kharianne) made their first contribution in https://github.com/pydantic/pydantic/pull/11111 -* [@mdaffad](https://github.com/mdaffad) made their first contribution in https://github.com/pydantic/pydantic/pull/11177 -* [@thejcannon](https://github.com/thejcannon) made their first contribution in https://github.com/pydantic/pydantic/pull/11014 -* [@thomasfrimannkoren](https://github.com/thomasfrimannkoren) made their first contribution in https://github.com/pydantic/pydantic/pull/11251 -* [@usernameMAI](https://github.com/usernameMAI) made their first contribution in https://github.com/pydantic/pydantic/pull/11275 -* [@ananiavito](https://github.com/ananiavito) made their first contribution in https://github.com/pydantic/pydantic/pull/11302 -* [@pawamoy](https://github.com/pawamoy) made their first contribution in https://github.com/pydantic/pydantic/pull/11311 -* [@Maze21127](https://github.com/Maze21127) made their first contribution in https://github.com/pydantic/pydantic/pull/11319 -* [@kauabh](https://github.com/kauabh) made their first contribution in https://github.com/pydantic/pydantic/pull/11369 -* [@jaceklaskowski](https://github.com/jaceklaskowski) made their first contribution in https://github.com/pydantic/pydantic/pull/11353 -* [@tmpbeing](https://github.com/tmpbeing) made their first contribution in https://github.com/pydantic/pydantic/pull/11375 -* [@petyosi](https://github.com/petyosi) made their first contribution in https://github.com/pydantic/pydantic/pull/11405 -* [@austinyu](https://github.com/austinyu) made their first contribution in https://github.com/pydantic/pydantic/pull/11392 -* [@mikeedjones](https://github.com/mikeedjones) made their first contribution in https://github.com/pydantic/pydantic/pull/11402 -* [@astei](https://github.com/astei) made their first contribution in https://github.com/pydantic/pydantic/pull/11436 -* [@dsayling](https://github.com/dsayling) made their first contribution in https://github.com/pydantic/pydantic/pull/11522 -* [@sobolevn](https://github.com/sobolevn) made their first contribution in https://github.com/pydantic/pydantic-core/pull/1645 - -## v2.11.0a2 (2025-02-10) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.0a2) - -### What's Changed - -Pydantic v2.11 is a version strongly focused on build time performance of Pydantic models (and core schema generation in general). -This is another early alpha release, meant to collect early feedback from users having issues with core schema builds. - -#### Performance - -* Create a single dictionary when creating a `CoreConfig` instance by [@sydney-runkle](https://github.com/sydney-runkle) in [#11384](https://github.com/pydantic/pydantic/pull/11384) - -#### Fixes - -* Use the correct JSON Schema mode when handling function schemas by [@Viicos](https://github.com/Viicos) in [#11367](https://github.com/pydantic/pydantic/pull/11367) -* Fix JSON Schema reference logic with `examples` keys by [@Viicos](https://github.com/Viicos) in [#11366](https://github.com/pydantic/pydantic/pull/11366) -* Improve exception message when encountering recursion errors during type evaluation by [@Viicos](https://github.com/Viicos) in [#11356](https://github.com/pydantic/pydantic/pull/11356) -* Always include `additionalProperties: True` for arbitrary dictionary schemas by [@austinyu](https://github.com/austinyu) in [#11392](https://github.com/pydantic/pydantic/pull/11392) -* Expose `fallback` parameter in serialization methods by [@Viicos](https://github.com/Viicos) in [#11398](https://github.com/pydantic/pydantic/pull/11398) -* Fix path serialization behavior by [@sydney-runkle](https://github.com/sydney-runkle) in [#11416](https://github.com/pydantic/pydantic/pull/11416) - -#### Packaging - -* Bump `ruff` from 0.9.2 to 0.9.5 by [@Viicos](https://github.com/Viicos) in [#11407](https://github.com/pydantic/pydantic/pull/11407) -* Bump `pydantic-core` to v2.29.0 by [@mikeedjones](https://github.com/mikeedjones) in [#11402](https://github.com/pydantic/pydantic/pull/11402) -* Use locally-built rust with symbols & pgo by [@davidhewitt](https://github.com/davidhewitt) in [#11403](https://github.com/pydantic/pydantic/pull/11403) - -### New Contributors - -* [@kauabh](https://github.com/kauabh) made their first contribution in [#11369](https://github.com/pydantic/pydantic/pull/11369) -* [@jaceklaskowski](https://github.com/jaceklaskowski) made their first contribution in [#11353](https://github.com/pydantic/pydantic/pull/11353) -* [@tmpbeing](https://github.com/tmpbeing) made their first contribution in [#11375](https://github.com/pydantic/pydantic/pull/11375) -* [@petyosi](https://github.com/petyosi) made their first contribution in [#11405](https://github.com/pydantic/pydantic/pull/11405) -* [@austinyu](https://github.com/austinyu) made their first contribution in [#11392](https://github.com/pydantic/pydantic/pull/11392) -* [@mikeedjones](https://github.com/mikeedjones) made their first contribution in [#11402](https://github.com/pydantic/pydantic/pull/11402) - -## v2.11.0a1 (2025-01-30) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.11.0a1) - -### What's Changed - -Pydantic v2.11 is a version strongly focused on build time performance of Pydantic models (and core schema generation in general). -This is an early alpha release, meant to collect early feedback from users having issues with core schema builds. - -#### New Features - -* Support unsubstituted type variables with both a default and a bound or constraints by [@FyZzyss](https://github.com/FyZzyss) in [#10789](https://github.com/pydantic/pydantic/pull/10789) -* Add a `default_factory_takes_validated_data` property to `FieldInfo` by [@Viicos](https://github.com/Viicos) in [#11034](https://github.com/pydantic/pydantic/pull/11034) -* Raise a better error when a generic alias is used inside `type[]` by [@Viicos](https://github.com/Viicos) in [#11088](https://github.com/pydantic/pydantic/pull/11088) -* Properly support PEP 695 generics syntax by [@Viicos](https://github.com/Viicos) in [#11189](https://github.com/pydantic/pydantic/pull/11189) -* Properly support type variable defaults by [@Viicos](https://github.com/Viicos) in [#11332](https://github.com/pydantic/pydantic/pull/11332) - -#### Changes - -* Rework `create_model` field definitions format by [@Viicos](https://github.com/Viicos) in [#11032](https://github.com/pydantic/pydantic/pull/11032) -* Raise a deprecation warning when a field is annotated as final with a default value by [@Viicos](https://github.com/Viicos) in [#11168](https://github.com/pydantic/pydantic/pull/11168) -* Deprecate accessing `model_fields` and `model_computed_fields` on instances by [@Viicos](https://github.com/Viicos) in [#11169](https://github.com/pydantic/pydantic/pull/11169) -* Move core schema generation logic for path types inside the `GenerateSchema` class by [@sydney-runkle](https://github.com/sydney-runkle) in [#10846](https://github.com/pydantic/pydantic/pull/10846) -* Move `deque` schema gen to `GenerateSchema` class by [@sydney-runkle](https://github.com/sydney-runkle) in [#11239](https://github.com/pydantic/pydantic/pull/11239) -* Move `Mapping` schema gen to `GenerateSchema` to complete removal of `prepare_annotations_for_known_type` workaround by [@sydney-runkle](https://github.com/sydney-runkle) in [#11247](https://github.com/pydantic/pydantic/pull/11247) -* Remove Python 3.8 Support by [@sydney-runkle](https://github.com/sydney-runkle) in [#11258](https://github.com/pydantic/pydantic/pull/11258) -* Disable `pydantic-core` core schema validation by [@sydney-runkle](https://github.com/sydney-runkle) in [#11271](https://github.com/pydantic/pydantic/pull/11271) - -#### Performance - -* Only evaluate `FieldInfo` annotations if required during schema building by [@Viicos](https://github.com/Viicos) in [#10769](https://github.com/pydantic/pydantic/pull/10769) -* Optimize calls to `get_type_ref` by [@Viicos](https://github.com/Viicos) in [#10863](https://github.com/pydantic/pydantic/pull/10863) -* Improve `__setattr__` performance of Pydantic models by caching setter functions by [@MarkusSintonen](https://github.com/MarkusSintonen) in [#10868](https://github.com/pydantic/pydantic/pull/10868) -* Improve annotation application performance by [@Viicos](https://github.com/Viicos) in [#11186](https://github.com/pydantic/pydantic/pull/11186) -* Improve performance of `_typing_extra` module by [@Viicos](https://github.com/Viicos) in [#11255](https://github.com/pydantic/pydantic/pull/11255) -* Refactor and optimize schema cleaning logic by [@Viicos](https://github.com/Viicos) and [@MarkusSintonen](https://github.com/MarkusSintonen) in [#11244](https://github.com/pydantic/pydantic/pull/11244) - -#### Fixes - -* Add validation tests for `_internal/_validators.py` by [@tkasuz](https://github.com/tkasuz) in [#10763](https://github.com/pydantic/pydantic/pull/10763) -* Improve `TypeAdapter` instance repr by [@sydney-runkle](https://github.com/sydney-runkle) in [#10872](https://github.com/pydantic/pydantic/pull/10872) -* Revert "ci: use locally built pydantic-core with debug symbols by [@sydney-runkle](https://github.com/sydney-runkle) in [#10942](https://github.com/pydantic/pydantic/pull/10942) -* Re-enable all FastAPI tests by [@tamird](https://github.com/tamird) in [#10948](https://github.com/pydantic/pydantic/pull/10948) -* Fix typo in HISTORY.md. by [@felixxm](https://github.com/felixxm) in [#11077](https://github.com/pydantic/pydantic/pull/11077) -* Infer final fields with a default value as class variables in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#11121](https://github.com/pydantic/pydantic/pull/11121) -* Recursively unpack `Literal` values if using PEP 695 type aliases by [@Viicos](https://github.com/Viicos) in [#11114](https://github.com/pydantic/pydantic/pull/11114) -* Override `__subclasscheck__` on `ModelMetaclass` to avoid memory leak and performance issues by [@Viicos](https://github.com/Viicos) in [#11116](https://github.com/pydantic/pydantic/pull/11116) -* Remove unused `_extract_get_pydantic_json_schema()` parameter by [@Viicos](https://github.com/Viicos) in [#11155](https://github.com/pydantic/pydantic/pull/11155) -* Add FastAPI and SQLModel to third-party tests by [@sydney-runkle](https://github.com/sydney-runkle) in [#11044](https://github.com/pydantic/pydantic/pull/11044) -* Fix conditional expressions syntax for third-party tests by [@Viicos](https://github.com/Viicos) in [#11162](https://github.com/pydantic/pydantic/pull/11162) -* Move FastAPI tests to third-party workflow by [@Viicos](https://github.com/Viicos) in [#11164](https://github.com/pydantic/pydantic/pull/11164) -* Improve discriminated union error message for invalid union variants by [@Viicos](https://github.com/Viicos) in [#11161](https://github.com/pydantic/pydantic/pull/11161) -* Unpack PEP 695 type aliases if using the `Annotated` form by [@Viicos](https://github.com/Viicos) in [#11109](https://github.com/pydantic/pydantic/pull/11109) -* Include `openapi-python-client` check in issue creation for third-party failures, use `main` branch by [@sydney-runkle](https://github.com/sydney-runkle) in [#11182](https://github.com/pydantic/pydantic/pull/11182) -* Add pandera third-party tests by [@Viicos](https://github.com/Viicos) in [#11193](https://github.com/pydantic/pydantic/pull/11193) -* Add ODMantic third-party tests by [@sydney-runkle](https://github.com/sydney-runkle) in [#11197](https://github.com/pydantic/pydantic/pull/11197) -* Add missing stacklevel in `deprecated_instance_property` warning by [@Viicos](https://github.com/Viicos) in [#11200](https://github.com/pydantic/pydantic/pull/11200) -* Copy `WithJsonSchema` schema to avoid sharing mutated data by [@thejcannon](https://github.com/thejcannon) in [#11014](https://github.com/pydantic/pydantic/pull/11014) -* Do not cache parametrized models when in the process of parametrizing another model by [@Viicos](https://github.com/Viicos) in [#10704](https://github.com/pydantic/pydantic/pull/10704) -* Re-enable Beanie third-party tests by [@Viicos](https://github.com/Viicos) in [#11214](https://github.com/pydantic/pydantic/pull/11214) -* Add discriminated union related metadata entries to the `CoreMetadata` definition by [@Viicos](https://github.com/Viicos) in [#11216](https://github.com/pydantic/pydantic/pull/11216) -* Consolidate schema definitions logic in the `_Definitions` class by [@Viicos](https://github.com/Viicos) in [#11208](https://github.com/pydantic/pydantic/pull/11208) -* Support initializing root model fields with values of the `root` type in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#11212](https://github.com/pydantic/pydantic/pull/11212) -* Fix various issues with dataclasses and `use_attribute_docstrings` by [@Viicos](https://github.com/Viicos) in [#11246](https://github.com/pydantic/pydantic/pull/11246) -* Only compute normalized decimal places if necessary in `decimal_places_validator` by [@misrasaurabh1](https://github.com/misrasaurabh1) in [#11281](https://github.com/pydantic/pydantic/pull/11281) -* Fix two misplaced sentences in validation errors documentation by [@ananiavito](https://github.com/ananiavito) in [#11302](https://github.com/pydantic/pydantic/pull/11302) -* Fix mkdocstrings inventory example in documentation by [@pawamoy](https://github.com/pawamoy) in [#11311](https://github.com/pydantic/pydantic/pull/11311) -* Add support for `validation_alias` in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#11295](https://github.com/pydantic/pydantic/pull/11295) -* Do not transform model serializer functions as class methods in the mypy plugin by [@Viicos](https://github.com/Viicos) in [#11298](https://github.com/pydantic/pydantic/pull/11298) -* Simplify `GenerateJsonSchema.literal_schema()` implementation by [@misrasaurabh1](https://github.com/misrasaurabh1) in [#11321](https://github.com/pydantic/pydantic/pull/11321) -* Add additional allowed schemes for `ClickHouseDsn` by [@Maze21127](https://github.com/Maze21127) in [#11319](https://github.com/pydantic/pydantic/pull/11319) -* Coerce decimal constraints to `Decimal` instances by [@Viicos](https://github.com/Viicos) in [#11350](https://github.com/pydantic/pydantic/pull/11350) -* Fix `ValueError` on year zero by [@davidhewitt](https://github.com/davidhewitt) in [pydantic-core#1583](https://github.com/pydantic/pydantic-core/pull/1583) - -#### Packaging - -* Bump dawidd6/action-download-artifact from 6 to 7 by [@dependabot](https://github.com/dependabot) in [#11018](https://github.com/pydantic/pydantic/pull/11018) -* Re-enable memray related tests on Python 3.12+ by [@Viicos](https://github.com/Viicos) in [#11191](https://github.com/pydantic/pydantic/pull/11191) -* Bump astral-sh/setup-uv to 5 by [@dependabot](https://github.com/dependabot) in [#11205](https://github.com/pydantic/pydantic/pull/11205) -* Bump `ruff` to v0.9.0 by [@sydney-runkle](https://github.com/sydney-runkle) in [#11254](https://github.com/pydantic/pydantic/pull/11254) -* Regular `uv.lock` deps update by [@sydney-runkle](https://github.com/sydney-runkle) in [#11333](https://github.com/pydantic/pydantic/pull/11333) -* Add a `check_pydantic_core_version()` function by [@Viicos](https://github.com/Viicos) in [#11324](https://github.com/pydantic/pydantic/pull/11324) -* Remove `greenlet` development dependency by [@Viicos](https://github.com/Viicos) in [#11351](https://github.com/pydantic/pydantic/pull/11351) -* Bump `pydantic-core` to v2.28.0 by [@Viicos](https://github.com/Viicos) in [#11364](https://github.com/pydantic/pydantic/pull/11364) - -### New Contributors - -* [@FyZzyss](https://github.com/FyZzyss) made their first contribution in [#10789](https://github.com/pydantic/pydantic/pull/10789) -* [@tamird](https://github.com/tamird) made their first contribution in [#10948](https://github.com/pydantic/pydantic/pull/10948) -* [@felixxm](https://github.com/felixxm) made their first contribution in [#11077](https://github.com/pydantic/pydantic/pull/11077) -* [@alexprabhat99](https://github.com/alexprabhat99) made their first contribution in [#11082](https://github.com/pydantic/pydantic/pull/11082) -* [@Kharianne](https://github.com/Kharianne) made their first contribution in [#11111](https://github.com/pydantic/pydantic/pull/11111) -* [@mdaffad](https://github.com/mdaffad) made their first contribution in [#11177](https://github.com/pydantic/pydantic/pull/11177) -* [@thejcannon](https://github.com/thejcannon) made their first contribution in [#11014](https://github.com/pydantic/pydantic/pull/11014) -* [@thomasfrimannkoren](https://github.com/thomasfrimannkoren) made their first contribution in [#11251](https://github.com/pydantic/pydantic/pull/11251) -* [@usernameMAI](https://github.com/usernameMAI) made their first contribution in [#11275](https://github.com/pydantic/pydantic/pull/11275) -* [@ananiavito](https://github.com/ananiavito) made their first contribution in [#11302](https://github.com/pydantic/pydantic/pull/11302) -* [@pawamoy](https://github.com/pawamoy) made their first contribution in [#11311](https://github.com/pydantic/pydantic/pull/11311) -* [@Maze21127](https://github.com/Maze21127) made their first contribution in [#11319](https://github.com/pydantic/pydantic/pull/11319) - -## v2.10.6 (2025-01-23) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.6) - -### What's Changed - -#### Fixes - -* Fix JSON Schema reference collection with `'examples'` keys by [@Viicos](https://github.com/Viicos) in [#11325](https://github.com/pydantic/pydantic/pull/11325) -* Fix url python serialization by [@sydney-runkle](https://github.com/sydney-runkle) in [#11331](https://github.com/pydantic/pydantic/pull/11331) - -## v2.10.5 (2025-01-08) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.5) - -### What's Changed - -#### Fixes - -* Remove custom MRO implementation of Pydantic models by [@Viicos](https://github.com/Viicos) in [#11184](https://github.com/pydantic/pydantic/pull/11184) -* Fix URL serialization for unions by [@sydney-runkle](https://github.com/sydney-runkle) in [#11233](https://github.com/pydantic/pydantic/pull/11233) - -## v2.10.4 (2024-12-18) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.4) - -### What's Changed - -#### Fixes - -* Fix for comparison of `AnyUrl` objects by [@alexprabhat99](https://github.com/alexprabhat99) in [#11082](https://github.com/pydantic/pydantic/pull/11082) -* Properly fetch PEP 695 type params for functions, do not fetch annotations from signature by [@Viicos](https://github.com/Viicos) in [#11093](https://github.com/pydantic/pydantic/pull/11093) -* Include JSON Schema input core schema in function schemas by [@Viicos](https://github.com/Viicos) in [#11085](https://github.com/pydantic/pydantic/pull/11085) -* Add `len` to `_BaseUrl` to avoid TypeError by [@Kharianne](https://github.com/Kharianne) in [#11111](https://github.com/pydantic/pydantic/pull/11111) -* Make sure the type reference is removed from the seen references by [@Viicos](https://github.com/Viicos) in [#11143](https://github.com/pydantic/pydantic/pull/11143) - -### New Contributors - -* [@FyZzyss](https://github.com/FyZzyss) made their first contribution in [#10789](https://github.com/pydantic/pydantic/pull/10789) -* [@tamird](https://github.com/tamird) made their first contribution in [#10948](https://github.com/pydantic/pydantic/pull/10948) -* [@felixxm](https://github.com/felixxm) made their first contribution in [#11077](https://github.com/pydantic/pydantic/pull/11077) -* [@alexprabhat99](https://github.com/alexprabhat99) made their first contribution in [#11082](https://github.com/pydantic/pydantic/pull/11082) -* [@Kharianne](https://github.com/Kharianne) made their first contribution in [#11111](https://github.com/pydantic/pydantic/pull/11111) - -#### Packaging - -* Bump `pydantic-core` to v2.27.2 by [@davidhewitt](https://github.com/davidhewitt) in [#11138](https://github.com/pydantic/pydantic/pull/11138) - -## v2.10.3 (2024-12-03) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.3) - -### What's Changed - -#### Fixes - -* Set fields when `defer_build` is set on Pydantic dataclasses by [@Viicos](https://github.com/Viicos) in [#10984](https://github.com/pydantic/pydantic/pull/10984) -* Do not resolve the JSON Schema reference for `dict` core schema keys by [@Viicos](https://github.com/Viicos) in [#10989](https://github.com/pydantic/pydantic/pull/10989) -* Use the globals of the function when evaluating the return type for `PlainSerializer` and `WrapSerializer` functions by [@Viicos](https://github.com/Viicos) in [#11008](https://github.com/pydantic/pydantic/pull/11008) -* Fix host required enforcement for urls to be compatible with v2.9 behavior by [@sydney-runkle](https://github.com/sydney-runkle) in [#11027](https://github.com/pydantic/pydantic/pull/11027) -* Add a `default_factory_takes_validated_data` property to `FieldInfo` by [@Viicos](https://github.com/Viicos) in [#11034](https://github.com/pydantic/pydantic/pull/11034) -* Fix url json schema in `serialization` mode by [@sydney-runkle](https://github.com/sydney-runkle) in [#11035](https://github.com/pydantic/pydantic/pull/11035) - -## v2.10.2 (2024-11-25) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.2) - -### What's Changed - -#### Fixes - -* Only evaluate FieldInfo annotations if required during schema building by [@Viicos](https://github.com/Viicos) in [#10769](https://github.com/pydantic/pydantic/pull/10769) -* Do not evaluate annotations for private fields by [@Viicos](https://github.com/Viicos) in [#10962](https://github.com/pydantic/pydantic/pull/10962) -* Support serialization as any for `Secret` types and `Url` types by [@sydney-runkle](https://github.com/sydney-runkle) in [#10947](https://github.com/pydantic/pydantic/pull/10947) -* Fix type hint of `Field.default` to be compatible with Python 3.8 and 3.9 by [@Viicos](https://github.com/Viicos) in [#10972](https://github.com/pydantic/pydantic/pull/10972) -* Add hashing support for URL types by [@sydney-runkle](https://github.com/sydney-runkle) in [#10975](https://github.com/pydantic/pydantic/pull/10975) -* Hide `BaseModel.__replace__` definition from type checkers by [@Viicos](https://github.com/Viicos) in [#10979](https://github.com/pydantic/pydantic/pull/10979) - -## v2.10.1 (2024-11-21) - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.1) - -### What's Changed - -#### Fixes - -* Use the correct frame when instantiating a parametrized `TypeAdapter` by [@Viicos](https://github.com/Viicos) in [#10893](https://github.com/pydantic/pydantic/pull/10893) -* Relax check for validated data in `default_factory` utils by [@sydney-runkle](https://github.com/sydney-runkle) in [#10909](https://github.com/pydantic/pydantic/pull/10909) -* Fix type checking issue with `model_fields` and `model_computed_fields` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10911](https://github.com/pydantic/pydantic/pull/10911) -* Use the parent configuration during schema generation for stdlib `dataclass`es by [@sydney-runkle](https://github.com/sydney-runkle) in [#10928](https://github.com/pydantic/pydantic/pull/10928) -* Use the `globals` of the function when evaluating the return type of serializers and `computed_field`s by [@Viicos](https://github.com/Viicos) in [#10929](https://github.com/pydantic/pydantic/pull/10929) -* Fix URL constraint application by [@sydney-runkle](https://github.com/sydney-runkle) in [#10922](https://github.com/pydantic/pydantic/pull/10922) -* Fix URL equality with different validation methods by [@sydney-runkle](https://github.com/sydney-runkle) in [#10934](https://github.com/pydantic/pydantic/pull/10934) -* Fix JSON schema title when specified as `''` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10936](https://github.com/pydantic/pydantic/pull/10936) -* Fix `python` mode serialization for `complex` inference by [@sydney-runkle](https://github.com/sydney-runkle) in [pydantic-core#1549](https://github.com/pydantic/pydantic-core/pull/1549) - -#### Packaging - -* Bump `pydantic-core` version to `v2.27.1` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10938](https://github.com/pydantic/pydantic/pull/10938) - -### New Contributors - -## v2.10.0 (2024-11-20) - -The code released in v2.10.0 is practically identical to that of v2.10.0b2. - -[GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.0) - -See the [v2.10 release blog post](https://pydantic.dev/articles/pydantic-v2-10-release) for the highlights! - -### What's Changed - -#### New Features - -* Support `fractions.Fraction` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10318](https://github.com/pydantic/pydantic/pull/10318) -* Support `Hashable` for json validation by [@sydney-runkle](https://github.com/sydney-runkle) in [#10324](https://github.com/pydantic/pydantic/pull/10324) -* Add a `SocketPath` type for `linux` systems by [@theunkn0wn1](https://github.com/theunkn0wn1) in [#10378](https://github.com/pydantic/pydantic/pull/10378) -* Allow arbitrary refs in JSON schema `examples` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10417](https://github.com/pydantic/pydantic/pull/10417) -* Support `defer_build` for Pydantic dataclasses by [@Viicos](https://github.com/Viicos) in [#10313](https://github.com/pydantic/pydantic/pull/10313) -* Adding v1 / v2 incompatibility warning for nested v1 model by [@sydney-runkle](https://github.com/sydney-runkle) in [#10431](https://github.com/pydantic/pydantic/pull/10431) -* Add support for unpacked `TypedDict` to type hint variadic keyword arguments with `@validate_call` by [@Viicos](https://github.com/Viicos) in [#10416](https://github.com/pydantic/pydantic/pull/10416) -* Support compiled patterns in `protected_namespaces` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10522](https://github.com/pydantic/pydantic/pull/10522) -* Add support for `propertyNames` in JSON schema by [@FlorianSW](https://github.com/FlorianSW) in [#10478](https://github.com/pydantic/pydantic/pull/10478) -* Adding `__replace__` protocol for Python 3.13+ support by [@sydney-runkle](https://github.com/sydney-runkle) in [#10596](https://github.com/pydantic/pydantic/pull/10596) -* Expose public `sort` method for JSON schema generation by [@sydney-runkle](https://github.com/sydney-runkle) in [#10595](https://github.com/pydantic/pydantic/pull/10595) -* Add runtime validation of `@validate_call` callable argument by [@kc0506](https://github.com/kc0506) in [#10627](https://github.com/pydantic/pydantic/pull/10627) -* Add `experimental_allow_partial` support by [@samuelcolvin](https://github.com/samuelcolvin) in [#10748](https://github.com/pydantic/pydantic/pull/10748) -* Support default factories taking validated data as an argument by [@Viicos](https://github.com/Viicos) in [#10678](https://github.com/pydantic/pydantic/pull/10678) -* Allow subclassing `ValidationError` and `PydanticCustomError` by [@Youssefares](https://github.com/Youssefares) in [pydantic/pydantic-core#1413](https://github.com/pydantic/pydantic-core/pull/1413) -* Add `trailing-strings` support to `experimental_allow_partial` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10825](https://github.com/pydantic/pydantic/pull/10825) -* Add `rebuild()` method for `TypeAdapter` and simplify `defer_build` patterns by [@sydney-runkle](https://github.com/sydney-runkle) in [#10537](https://github.com/pydantic/pydantic/pull/10537) -* Improve `TypeAdapter` instance repr by [@sydney-runkle](https://github.com/sydney-runkle) in [#10872](https://github.com/pydantic/pydantic/pull/10872) - -#### Changes - -* Don't allow customization of `SchemaGenerator` until interface is more stable by [@sydney-runkle](https://github.com/sydney-runkle) in [#10303](https://github.com/pydantic/pydantic/pull/10303) -* Cleanly `defer_build` on `TypeAdapters`, removing experimental flag by [@sydney-runkle](https://github.com/sydney-runkle) in [#10329](https://github.com/pydantic/pydantic/pull/10329) -* Fix `mro` of generic subclass by [@kc0506](https://github.com/kc0506) in [#10100](https://github.com/pydantic/pydantic/pull/10100) -* Strip whitespaces on JSON Schema title generation by [@sydney-runkle](https://github.com/sydney-runkle) in [#10404](https://github.com/pydantic/pydantic/pull/10404) -* Use `b64decode` and `b64encode` for `Base64Bytes` type by [@sydney-runkle](https://github.com/sydney-runkle) in [#10486](https://github.com/pydantic/pydantic/pull/10486) -* Relax protected namespace config default by [@sydney-runkle](https://github.com/sydney-runkle) in [#10441](https://github.com/pydantic/pydantic/pull/10441) -* Revalidate parametrized generics if instance's origin is subclass of OG class by [@sydney-runkle](https://github.com/sydney-runkle) in [#10666](https://github.com/pydantic/pydantic/pull/10666) -* Warn if configuration is specified on the `@dataclass` decorator and with the `__pydantic_config__` attribute by [@sydney-runkle](https://github.com/sydney-runkle) in [#10406](https://github.com/pydantic/pydantic/pull/10406) -* Recommend against using `Ellipsis` (...) with `Field` by [@Viicos](https://github.com/Viicos) in [#10661](https://github.com/pydantic/pydantic/pull/10661) -* Migrate to subclassing instead of annotated approach for pydantic url types by [@sydney-runkle](https://github.com/sydney-runkle) in [#10662](https://github.com/pydantic/pydantic/pull/10662) -* Change JSON schema generation of `Literal`s and `Enums` by [@Viicos](https://github.com/Viicos) in [#10692](https://github.com/pydantic/pydantic/pull/10692) -* Simplify unions involving `Any` or `Never` when replacing type variables by [@Viicos](https://github.com/Viicos) in [#10338](https://github.com/pydantic/pydantic/pull/10338) -* Do not require padding when decoding `base64` bytes by [@bschoenmaeckers](https://github.com/bschoenmaeckers) in [pydantic/pydantic-core#1448](https://github.com/pydantic/pydantic-core/pull/1448) -* Support dates all the way to 1BC by [@changhc](https://github.com/changhc) in [pydantic/speedate#77](https://github.com/pydantic/speedate/pull/77) - -#### Performance - -* Schema cleaning: skip unnecessary copies during schema walking by [@Viicos](https://github.com/Viicos) in [#10286](https://github.com/pydantic/pydantic/pull/10286) -* Refactor namespace logic for annotations evaluation by [@Viicos](https://github.com/Viicos) in [#10530](https://github.com/pydantic/pydantic/pull/10530) -* Improve email regexp on edge cases by [@AlekseyLobanov](https://github.com/AlekseyLobanov) in [#10601](https://github.com/pydantic/pydantic/pull/10601) -* `CoreMetadata` refactor with an emphasis on documentation, schema build time performance, and reducing complexity by [@sydney-runkle](https://github.com/sydney-runkle) in [#10675](https://github.com/pydantic/pydantic/pull/10675) - -#### Packaging - -* Bump `pydantic-core` to `v2.27.0` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10825](https://github.com/pydantic/pydantic/pull/10825) -* Replaced pdm with uv by [@frfahim](https://github.com/frfahim) in [#10727](https://github.com/pydantic/pydantic/pull/10727) - -#### Fixes - -* Remove guarding check on `computed_field` with `field_serializer` by [@nix010](https://github.com/nix010) in [#10390](https://github.com/pydantic/pydantic/pull/10390) -* Fix `Predicate` issue in `v2.9.0` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10321](https://github.com/pydantic/pydantic/pull/10321) -* Fixing `annotated-types` bound by [@sydney-runkle](https://github.com/sydney-runkle) in [#10327](https://github.com/pydantic/pydantic/pull/10327) -* Turn `tzdata` install requirement into optional `timezone` dependency by [@jakob-keller](https://github.com/jakob-keller) in [#10331](https://github.com/pydantic/pydantic/pull/10331) -* Use correct types namespace when building `namedtuple` core schemas by [@Viicos](https://github.com/Viicos) in [#10337](https://github.com/pydantic/pydantic/pull/10337) -* Fix evaluation of stringified annotations during namespace inspection by [@Viicos](https://github.com/Viicos) in [#10347](https://github.com/pydantic/pydantic/pull/10347) -* Fix `IncEx` type alias definition by [@Viicos](https://github.com/Viicos) in [#10339](https://github.com/pydantic/pydantic/pull/10339) -* Do not error when trying to evaluate annotations of private attributes by [@Viicos](https://github.com/Viicos) in [#10358](https://github.com/pydantic/pydantic/pull/10358) -* Fix nested type statement by [@kc0506](https://github.com/kc0506) in [#10369](https://github.com/pydantic/pydantic/pull/10369) -* Improve typing of `ModelMetaclass.mro` by [@Viicos](https://github.com/Viicos) in [#10372](https://github.com/pydantic/pydantic/pull/10372) -* Fix class access of deprecated `computed_field`s by [@Viicos](https://github.com/Viicos) in [#10391](https://github.com/pydantic/pydantic/pull/10391) -* Make sure `inspect.iscoroutinefunction` works on coroutines decorated with `@validate_call` by [@MovisLi](https://github.com/MovisLi) in [#10374](https://github.com/pydantic/pydantic/pull/10374) -* Fix `NameError` when using `validate_call` with PEP 695 on a class by [@kc0506](https://github.com/kc0506) in [#10380](https://github.com/pydantic/pydantic/pull/10380) -* Fix `ZoneInfo` with various invalid types by [@sydney-runkle](https://github.com/sydney-runkle) in [#10408](https://github.com/pydantic/pydantic/pull/10408) -* Fix `PydanticUserError` on empty `model_config` with annotations by [@cdwilson](https://github.com/cdwilson) in [#10412](https://github.com/pydantic/pydantic/pull/10412) -* Fix variance issue in `_IncEx` type alias, only allow `True` by [@Viicos](https://github.com/Viicos) in [#10414](https://github.com/pydantic/pydantic/pull/10414) -* Fix serialization schema generation when using `PlainValidator` by [@Viicos](https://github.com/Viicos) in [#10427](https://github.com/pydantic/pydantic/pull/10427) -* Fix schema generation error when serialization schema holds references by [@Viicos](https://github.com/Viicos) in [#10444](https://github.com/pydantic/pydantic/pull/10444) -* Inline references if possible when generating schema for `json_schema_input_type` by [@Viicos](https://github.com/Viicos) in [#10439](https://github.com/pydantic/pydantic/pull/10439) -* Fix recursive arguments in `Representation` by [@Viicos](https://github.com/Viicos) in [#10480](https://github.com/pydantic/pydantic/pull/10480) -* Fix representation for builtin function types by [@kschwab](https://github.com/kschwab) in [#10479](https://github.com/pydantic/pydantic/pull/10479) -* Add python validators for decimal constraints (`max_digits` and `decimal_places`) by [@sydney-runkle](https://github.com/sydney-runkle) in [#10506](https://github.com/pydantic/pydantic/pull/10506) -* Only fetch `__pydantic_core_schema__` from the current class during schema generation by [@Viicos](https://github.com/Viicos) in [#10518](https://github.com/pydantic/pydantic/pull/10518) -* Fix `stacklevel` on deprecation warnings for `BaseModel` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10520](https://github.com/pydantic/pydantic/pull/10520) -* Fix warning `stacklevel` in `BaseModel.__init__` by [@Viicos](https://github.com/Viicos) in [#10526](https://github.com/pydantic/pydantic/pull/10526) -* Improve error handling for in-evaluable refs for discriminator application by [@sydney-runkle](https://github.com/sydney-runkle) in [#10440](https://github.com/pydantic/pydantic/pull/10440) -* Change the signature of `ConfigWrapper.core_config` to take the title directly by [@Viicos](https://github.com/Viicos) in [#10562](https://github.com/pydantic/pydantic/pull/10562) -* Do not use the previous config from the stack for dataclasses without config by [@Viicos](https://github.com/Viicos) in [#10576](https://github.com/pydantic/pydantic/pull/10576) -* Fix serialization for IP types with `mode='python'` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10594](https://github.com/pydantic/pydantic/pull/10594) -* Support constraint application for `Base64Etc` types by [@sydney-runkle](https://github.com/sydney-runkle) in [#10584](https://github.com/pydantic/pydantic/pull/10584) -* Fix `validate_call` ignoring `Field` in `Annotated` by [@kc0506](https://github.com/kc0506) in [#10610](https://github.com/pydantic/pydantic/pull/10610) -* Raise an error when `Self` is invalid by [@kc0506](https://github.com/kc0506) in [#10609](https://github.com/pydantic/pydantic/pull/10609) -* Using `core_schema.InvalidSchema` instead of metadata injection + checks by [@sydney-runkle](https://github.com/sydney-runkle) in [#10523](https://github.com/pydantic/pydantic/pull/10523) -* Tweak type alias logic by [@kc0506](https://github.com/kc0506) in [#10643](https://github.com/pydantic/pydantic/pull/10643) -* Support usage of `type` with `typing.Self` and type aliases by [@kc0506](https://github.com/kc0506) in [#10621](https://github.com/pydantic/pydantic/pull/10621) -* Use overloads for `Field` and `PrivateAttr` functions by [@Viicos](https://github.com/Viicos) in [#10651](https://github.com/pydantic/pydantic/pull/10651) -* Clean up the `mypy` plugin implementation by [@Viicos](https://github.com/Viicos) in [#10669](https://github.com/pydantic/pydantic/pull/10669) -* Properly check for `typing_extensions` variant of `TypeAliasType` by [@Daraan](https://github.com/Daraan) in [#10713](https://github.com/pydantic/pydantic/pull/10713) -* Allow any mapping in `BaseModel.model_copy()` by [@Viicos](https://github.com/Viicos) in [#10751](https://github.com/pydantic/pydantic/pull/10751) -* Fix `isinstance` behavior for urls by [@sydney-runkle](https://github.com/sydney-runkle) in [#10766](https://github.com/pydantic/pydantic/pull/10766) -* Ensure `cached_property` can be set on Pydantic models by [@Viicos](https://github.com/Viicos) in [#10774](https://github.com/pydantic/pydantic/pull/10774) -* Fix equality checks for primitives in literals by [@sydney-runkle](https://github.com/sydney-runkle) in [pydantic/pydantic-core#1459](https://github.com/pydantic/pydantic-core/pull/1459) -* Properly enforce `host_required` for URLs by [@Viicos](https://github.com/Viicos) in [pydantic/pydantic-core#1488](https://github.com/pydantic/pydantic-core/pull/1488) -* Fix when `coerce_numbers_to_str` enabled and string has invalid Unicode character by [@andrey-berenda](https://github.com/andrey-berenda) in [pydantic/pydantic-core#1515](https://github.com/pydantic/pydantic-core/pull/1515) -* Fix serializing `complex` values in `Enum`s by [@changhc](https://github.com/changhc) in [pydantic/pydantic-core#1524](https://github.com/pydantic/pydantic-core/pull/1524) -* Refactor `_typing_extra` module by [@Viicos](https://github.com/Viicos) in [#10725](https://github.com/pydantic/pydantic/pull/10725) -* Support intuitive equality for urls by [@sydney-runkle](https://github.com/sydney-runkle) in [#10798](https://github.com/pydantic/pydantic/pull/10798) -* Add `bytearray` to `TypeAdapter.validate_json` signature by [@samuelcolvin](https://github.com/samuelcolvin) in [#10802](https://github.com/pydantic/pydantic/pull/10802) -* Ensure class access of method descriptors is performed when used as a default with `Field` by [@Viicos](https://github.com/Viicos) in [#10816](https://github.com/pydantic/pydantic/pull/10816) -* Fix circular import with `validate_call` by [@sydney-runkle](https://github.com/sydney-runkle) in [#10807](https://github.com/pydantic/pydantic/pull/10807) -* Fix error when using type aliases referencing other type aliases by [@Viicos](https://github.com/Viicos) in [#10809](https://github.com/pydantic/pydantic/pull/10809) -* Fix `IncEx` type alias to be compatible with mypy by [@Viicos](https://github.com/Viicos) in [#10813](https://github.com/pydantic/pydantic/pull/10813) -* Make `__signature__` a lazy property, do not deepcopy defaults by [@Viicos](https://github.com/Viicos) in [#10818](https://github.com/pydantic/pydantic/pull/10818) -* Make `__signature__` lazy for dataclasses, too by [@sydney-runkle](https://github.com/sydney-runkle) in [#10832](https://github.com/pydantic/pydantic/pull/10832) -* Subclass all single host url classes from `AnyUrl` to preserve behavior from v2.9 by [@sydney-runkle](https://github.com/sydney-runkle) in [#10856](https://github.com/pydantic/pydantic/pull/10856) - -### New Contributors - -* [@jakob-keller](https://github.com/jakob-keller) made their first contribution in [#10331](https://github.com/pydantic/pydantic/pull/10331) -* [@MovisLi](https://github.com/MovisLi) made their first contribution in [#10374](https://github.com/pydantic/pydantic/pull/10374) -* [@joaopalmeiro](https://github.com/joaopalmeiro) made their first contribution in [#10405](https://github.com/pydantic/pydantic/pull/10405) -* [@theunkn0wn1](https://github.com/theunkn0wn1) made their first contribution in [#10378](https://github.com/pydantic/pydantic/pull/10378) -* [@cdwilson](https://github.com/cdwilson) made their first contribution in [#10412](https://github.com/pydantic/pydantic/pull/10412) -* [@dlax](https://github.com/dlax) made their first contribution in [#10421](https://github.com/pydantic/pydantic/pull/10421) -* [@kschwab](https://github.com/kschwab) made their first contribution in [#10479](https://github.com/pydantic/pydantic/pull/10479) -* [@santibreo](https://github.com/santibreo) made their first contribution in [#10453](https://github.com/pydantic/pydantic/pull/10453) -* [@FlorianSW](https://github.com/FlorianSW) made their first contribution in [#10478](https://github.com/pydantic/pydantic/pull/10478) -* [@tkasuz](https://github.com/tkasuz) made their first contribution in [#10555](https://github.com/pydantic/pydantic/pull/10555) -* [@AlekseyLobanov](https://github.com/AlekseyLobanov) made their first contribution in [#10601](https://github.com/pydantic/pydantic/pull/10601) -* [@NiclasvanEyk](https://github.com/NiclasvanEyk) made their first contribution in [#10667](https://github.com/pydantic/pydantic/pull/10667) -* [@mschoettle](https://github.com/mschoettle) made their first contribution in [#10677](https://github.com/pydantic/pydantic/pull/10677) -* [@Daraan](https://github.com/Daraan) made their first contribution in [#10713](https://github.com/pydantic/pydantic/pull/10713) -* [@k4nar](https://github.com/k4nar) made their first contribution in [#10736](https://github.com/pydantic/pydantic/pull/10736) -* [@UriyaHarpeness](https://github.com/UriyaHarpeness) made their first contribution in [#10740](https://github.com/pydantic/pydantic/pull/10740) -* [@frfahim](https://github.com/frfahim) made their first contribution in [#10727](https://github.com/pydantic/pydantic/pull/10727) - -## v2.10.0b2 (2024-11-13) - -Pre-release, see [the GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.0b2) for details. - -## v2.10.0b1 (2024-11-06) - -Pre-release, see [the GitHub release](https://github.com/pydantic/pydantic/releases/tag/v2.10.0b1) for details. - - -... see [here](https://docs.pydantic.dev/changelog/#v0322-2019-08-17) for earlier changes. diff --git a/functions/python/data-processor/pydantic-2.13.4.dist-info/RECORD b/functions/python/data-processor/pydantic-2.13.4.dist-info/RECORD deleted file mode 100644 index bca277bb3..000000000 --- a/functions/python/data-processor/pydantic-2.13.4.dist-info/RECORD +++ /dev/null @@ -1,217 +0,0 @@ -pydantic-2.13.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -pydantic-2.13.4.dist-info/METADATA,sha256=0eVsNBJJJRL7a1mp38Jat68qggRYXNe8oABRMCMTM5Q,109397 -pydantic-2.13.4.dist-info/RECORD,, -pydantic-2.13.4.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87 -pydantic-2.13.4.dist-info/licenses/LICENSE,sha256=qeGG88oWte74QxjnpwFyE1GgDLe4rjpDlLZ7SeNSnvM,1129 -pydantic/__init__.py,sha256=5iEnJ4wHv1OEzdKQPzaKaZKfO4pSQAC65ODrYI6_S8Y,15812 -pydantic/__pycache__/__init__.cpython-314.pyc,, -pydantic/__pycache__/_migration.cpython-314.pyc,, -pydantic/__pycache__/alias_generators.cpython-314.pyc,, -pydantic/__pycache__/aliases.cpython-314.pyc,, -pydantic/__pycache__/annotated_handlers.cpython-314.pyc,, -pydantic/__pycache__/class_validators.cpython-314.pyc,, -pydantic/__pycache__/color.cpython-314.pyc,, -pydantic/__pycache__/config.cpython-314.pyc,, -pydantic/__pycache__/dataclasses.cpython-314.pyc,, -pydantic/__pycache__/datetime_parse.cpython-314.pyc,, -pydantic/__pycache__/decorator.cpython-314.pyc,, -pydantic/__pycache__/env_settings.cpython-314.pyc,, -pydantic/__pycache__/error_wrappers.cpython-314.pyc,, -pydantic/__pycache__/errors.cpython-314.pyc,, -pydantic/__pycache__/fields.cpython-314.pyc,, -pydantic/__pycache__/functional_serializers.cpython-314.pyc,, -pydantic/__pycache__/functional_validators.cpython-314.pyc,, -pydantic/__pycache__/generics.cpython-314.pyc,, -pydantic/__pycache__/json.cpython-314.pyc,, -pydantic/__pycache__/json_schema.cpython-314.pyc,, -pydantic/__pycache__/main.cpython-314.pyc,, -pydantic/__pycache__/mypy.cpython-314.pyc,, -pydantic/__pycache__/networks.cpython-314.pyc,, -pydantic/__pycache__/parse.cpython-314.pyc,, -pydantic/__pycache__/root_model.cpython-314.pyc,, -pydantic/__pycache__/schema.cpython-314.pyc,, -pydantic/__pycache__/tools.cpython-314.pyc,, -pydantic/__pycache__/type_adapter.cpython-314.pyc,, -pydantic/__pycache__/types.cpython-314.pyc,, -pydantic/__pycache__/typing.cpython-314.pyc,, -pydantic/__pycache__/utils.cpython-314.pyc,, -pydantic/__pycache__/validate_call_decorator.cpython-314.pyc,, -pydantic/__pycache__/validators.cpython-314.pyc,, -pydantic/__pycache__/version.cpython-314.pyc,, -pydantic/__pycache__/warnings.cpython-314.pyc,, -pydantic/_internal/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -pydantic/_internal/__pycache__/__init__.cpython-314.pyc,, -pydantic/_internal/__pycache__/_config.cpython-314.pyc,, -pydantic/_internal/__pycache__/_core_metadata.cpython-314.pyc,, -pydantic/_internal/__pycache__/_core_utils.cpython-314.pyc,, -pydantic/_internal/__pycache__/_dataclasses.cpython-314.pyc,, -pydantic/_internal/__pycache__/_decorators.cpython-314.pyc,, -pydantic/_internal/__pycache__/_decorators_v1.cpython-314.pyc,, -pydantic/_internal/__pycache__/_discriminated_union.cpython-314.pyc,, -pydantic/_internal/__pycache__/_docs_extraction.cpython-314.pyc,, -pydantic/_internal/__pycache__/_fields.cpython-314.pyc,, -pydantic/_internal/__pycache__/_forward_ref.cpython-314.pyc,, -pydantic/_internal/__pycache__/_generate_schema.cpython-314.pyc,, -pydantic/_internal/__pycache__/_generics.cpython-314.pyc,, -pydantic/_internal/__pycache__/_git.cpython-314.pyc,, -pydantic/_internal/__pycache__/_import_utils.cpython-314.pyc,, -pydantic/_internal/__pycache__/_internal_dataclass.cpython-314.pyc,, -pydantic/_internal/__pycache__/_known_annotated_metadata.cpython-314.pyc,, -pydantic/_internal/__pycache__/_mock_val_ser.cpython-314.pyc,, -pydantic/_internal/__pycache__/_model_construction.cpython-314.pyc,, -pydantic/_internal/__pycache__/_namespace_utils.cpython-314.pyc,, -pydantic/_internal/__pycache__/_repr.cpython-314.pyc,, -pydantic/_internal/__pycache__/_schema_gather.cpython-314.pyc,, -pydantic/_internal/__pycache__/_schema_generation_shared.cpython-314.pyc,, -pydantic/_internal/__pycache__/_serializers.cpython-314.pyc,, -pydantic/_internal/__pycache__/_signature.cpython-314.pyc,, -pydantic/_internal/__pycache__/_typing_extra.cpython-314.pyc,, -pydantic/_internal/__pycache__/_utils.cpython-314.pyc,, -pydantic/_internal/__pycache__/_validate_call.cpython-314.pyc,, -pydantic/_internal/__pycache__/_validators.cpython-314.pyc,, -pydantic/_internal/_config.py,sha256=Rzys1Joffn4JczElcYDqsZLRgBgHn2lYWqDR55oASPA,14839 -pydantic/_internal/_core_metadata.py,sha256=Y_g2t3i7uluK-wXCZvzJfRFMPUM23aBYLfae4FzBPy0,5162 -pydantic/_internal/_core_utils.py,sha256=1jru4VbJ0x63R6dtVcuOI-dKQTC_d_lSnJWEBQzGNEQ,6487 -pydantic/_internal/_dataclasses.py,sha256=Zgqcm1WaJLBwTQQC5mGKNowjlTgX3mfX_J5e2vd24lM,13188 -pydantic/_internal/_decorators.py,sha256=RDEG_Jau5NiJcfO0xgdT7EOgsU1LgWIYlX7wN5rYtVs,33620 -pydantic/_internal/_decorators_v1.py,sha256=tfdfdpQKY4R2XCOwqHbZeoQMur6VNigRrfhudXBHx38,6185 -pydantic/_internal/_discriminated_union.py,sha256=JLx_MVLep7Mxl1zbpdNZjvHDcz-J3OEW6WcdV184dcM,26255 -pydantic/_internal/_docs_extraction.py,sha256=fyznSAHh5AzohnXZStV0HvH-nRbavNHPyg-knx-S_EE,4127 -pydantic/_internal/_fields.py,sha256=hXeb-zodGwTDvG9OK1um18P64b7cqHb0GOngP4jrgcY,31557 -pydantic/_internal/_forward_ref.py,sha256=5n3Y7-3AKLn8_FS3Yc7KutLiPUhyXmAtkEZOaFnonwM,611 -pydantic/_internal/_generate_schema.py,sha256=PjogUawIXmf8LuLNe9seJph7WLW4MJ7-GBsXwCsQC9Q,136348 -pydantic/_internal/_generics.py,sha256=CXjcInlvci8VejaWn1f39kv0AcfJL0R523qRLFZVD-s,23393 -pydantic/_internal/_git.py,sha256=IwPh3DPfa2Xq3rBuB9Nx8luR2A1i69QdeTfWWXIuCVg,809 -pydantic/_internal/_import_utils.py,sha256=TRhxD5OuY6CUosioBdBcJUs0om7IIONiZdYAV7zQ8jM,402 -pydantic/_internal/_internal_dataclass.py,sha256=_bedc1XbuuygRGiLZqkUkwwFpQaoR1hKLlR501nyySY,144 -pydantic/_internal/_known_annotated_metadata.py,sha256=PynQIFQ61__4Gcrzn0D5ENllg7jPq_cxoLTmuFQBY88,16805 -pydantic/_internal/_mock_val_ser.py,sha256=wmRRFSBvqfcLbI41PsFliB4u2AZ3mJpZeiERbD3xKTo,8885 -pydantic/_internal/_model_construction.py,sha256=JoKmY4JrDBu3nG_tCIrJgtJJE1uq6v29TvTz5ElHE5g,38928 -pydantic/_internal/_namespace_utils.py,sha256=hl3-TRAr82U2jTyPP3t-QqsvKLirxtkLfNfrN-fp0x8,12878 -pydantic/_internal/_repr.py,sha256=jQfnJuyDxQpSRNhG29II9PX8e4Nv2qWZrEw2lqih3UE,5172 -pydantic/_internal/_schema_gather.py,sha256=8nJ-uM6Y4z6xpasnGonEMubtNVX_mxeeRDFmd_qMVLA,9052 -pydantic/_internal/_schema_generation_shared.py,sha256=F_rbQbrkoomgxsskdHpP0jUJ7TCfe0BADAEkq6CJ4nM,4842 -pydantic/_internal/_serializers.py,sha256=YIWvSmAR5fnbGSWCOQduWt1yB4ZQY42eAruc-enrb6c,1491 -pydantic/_internal/_signature.py,sha256=i_b6wtluiVWZRh1ZY8UvB2UZziP1KjqSXZgC-HxwOT0,6808 -pydantic/_internal/_typing_extra.py,sha256=dDxqF46lzuqCoKLrAH_k95EDbayEeKb2lHjuTJ5OBoY,31574 -pydantic/_internal/_utils.py,sha256=gN48BsR-FDrJDibCmo69ttQg67WbuFrdy_1NQL3cvLI,15959 -pydantic/_internal/_validate_call.py,sha256=OD_BspHaL9FKzZ9XrndhiEuMnjF3SRIJUHtwv6yUffU,5366 -pydantic/_internal/_validators.py,sha256=7GTjXXWFMLib4dxQ-HeaiHlAZiR2B2G8byCYMGrmQ48,20563 -pydantic/_migration.py,sha256=VF73LRCUz3Irb5xVt13jb3NAcXVnEF6T1-J0OLfeZ5A,12160 -pydantic/alias_generators.py,sha256=KM1n3u4JfLSBl1UuYg3hoYHzXJD-yvgrnq8u1ccwh_A,2124 -pydantic/aliases.py,sha256=vhCHyoSWnX-EJ-wWb5qj4xyRssgGWnTQfzQp4GSZ9ug,4937 -pydantic/annotated_handlers.py,sha256=WfyFSqwoEIFXBh7T73PycKloI1DiX45GWi0-JOsCR4Y,4407 -pydantic/class_validators.py,sha256=i_V3j-PYdGLSLmj_IJZekTRjunO8SIVz8LMlquPyP7E,148 -pydantic/color.py,sha256=AzqGfVQHF92_ZctDcue0DM4yTp2P6tekkwRINTWrLIo,21481 -pydantic/config.py,sha256=o1P67FMWIQG-_RfGtKislkHwo4pXm_6jfsR41P88v78,44533 -pydantic/dataclasses.py,sha256=4X9We0jj1KLwBtvYSkAFXNon46zrpBmRZANf4LfwbXg,18963 -pydantic/datetime_parse.py,sha256=QC-WgMxMr_wQ_mNXUS7AVf-2hLEhvvsPY1PQyhSGOdk,150 -pydantic/decorator.py,sha256=YX-jUApu5AKaVWKPoaV-n-4l7UbS69GEt9Ra3hszmKI,145 -pydantic/deprecated/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -pydantic/deprecated/__pycache__/__init__.cpython-314.pyc,, -pydantic/deprecated/__pycache__/class_validators.cpython-314.pyc,, -pydantic/deprecated/__pycache__/config.cpython-314.pyc,, -pydantic/deprecated/__pycache__/copy_internals.cpython-314.pyc,, -pydantic/deprecated/__pycache__/decorator.cpython-314.pyc,, -pydantic/deprecated/__pycache__/json.cpython-314.pyc,, -pydantic/deprecated/__pycache__/parse.cpython-314.pyc,, -pydantic/deprecated/__pycache__/tools.cpython-314.pyc,, -pydantic/deprecated/class_validators.py,sha256=n0jYQOcb5YQiw0b7YXyi7NPiYdV7ujWR4KyZumjTPok,10281 -pydantic/deprecated/config.py,sha256=k_lsVk57paxLJOcBueH07cu1OgEgWdVBxm6lfaC3CCU,2663 -pydantic/deprecated/copy_internals.py,sha256=Ghd-vkMd5EYCCgyCGtPKO58np9cEKBQC6qkBeIEFI2g,7618 -pydantic/deprecated/decorator.py,sha256=TBm6bJ7wJsNih_8Wq5IzDcwP32m9_vfxs96desLuk00,10845 -pydantic/deprecated/json.py,sha256=HlWCG35RRrxyzuTS6LTQiZBwRhmDZWmeqQH8rLW6wA8,4657 -pydantic/deprecated/parse.py,sha256=Gzd6b_g8zJXcuE7QRq5adhx_EMJahXfcpXCF0RgrqqI,2511 -pydantic/deprecated/tools.py,sha256=Nrm9oFRZWp8-jlfvPgJILEsywp4YzZD52XIGPDLxHcI,3330 -pydantic/env_settings.py,sha256=6IHeeWEqlUPRUv3V-AXiF_W91fg2Jw_M3O0l34J_eyA,148 -pydantic/error_wrappers.py,sha256=RK6mqATc9yMD-KBD9IJS9HpKCprWHd8wo84Bnm-3fR8,150 -pydantic/errors.py,sha256=DrECPCWhSYrQ8Ba4O8hKzIAM2i9GBHTXWALzaFDpLf4,6013 -pydantic/experimental/__init__.py,sha256=QT7rKYdDsCiTJ9GEjmsQdWHScwpKrrNkGq6vqONP6RQ,104 -pydantic/experimental/__pycache__/__init__.cpython-314.pyc,, -pydantic/experimental/__pycache__/arguments_schema.cpython-314.pyc,, -pydantic/experimental/__pycache__/missing_sentinel.cpython-314.pyc,, -pydantic/experimental/__pycache__/pipeline.cpython-314.pyc,, -pydantic/experimental/arguments_schema.py,sha256=EFnjX_ulp-tPyUjQX5pmQtug1OFL_Acc8bcMbLd-fVY,1866 -pydantic/experimental/missing_sentinel.py,sha256=hQejgtF00wUuQMni9429evg-eXyIwpKvjsD8ofqfj-w,127 -pydantic/experimental/pipeline.py,sha256=auoW6l6g1FC41LciPmsI1M6ncf00Szde9B3C-yvb9mI,23956 -pydantic/fields.py,sha256=a8ZhJfI8FD6TQDD7-cWMm1ZXlQva0wMdT2Ey5VvVe-M,82023 -pydantic/functional_serializers.py,sha256=zwRAjZusORtEbtxpVU20kg8FpemyZz4Fq6wJk1mpwYQ,18117 -pydantic/functional_validators.py,sha256=7p-4jvP__9jZyvcdXiide5pAa_JlM7fJ1BgkqbjxSWM,31724 -pydantic/generics.py,sha256=0ZqZ9O9annIj_3mGBRqps4htey3b5lV1-d2tUxPMMnA,144 -pydantic/json.py,sha256=ZH8RkI7h4Bz-zp8OdTAxbJUoVvcoU-jhMdRZ0B-k0xc,140 -pydantic/json_schema.py,sha256=da3hQ9vQPLEhPsrDnUlijfQ1fuCCViGVnUzOrAZLgDs,125955 -pydantic/main.py,sha256=NbhCz-ku8wDgYLQMBi75Ov7hywdc8LewA3oUNIZ8JVQ,85334 -pydantic/mypy.py,sha256=sYmmZrL_GvoYSLcBuPRRDfSfgQWehkzw_ZvNkwDY2ME,60971 -pydantic/networks.py,sha256=gCB96gt0G7tiVDhVnJfpKr1ARL5qkH-SPZkuNkmG2O4,42102 -pydantic/parse.py,sha256=wkd82dgtvWtD895U_I6E1htqMlGhBSYEV39cuBSeo3A,141 -pydantic/plugin/__init__.py,sha256=a7Tw366U6K3kltCCNZY76nc9ss-7uGGQ40TXad9OypQ,7333 -pydantic/plugin/__pycache__/__init__.cpython-314.pyc,, -pydantic/plugin/__pycache__/_loader.cpython-314.pyc,, -pydantic/plugin/__pycache__/_schema_validator.cpython-314.pyc,, -pydantic/plugin/_loader.py,sha256=hAjgSljoIhGx3AVpIpuqw5SPttBNNeGBSTrqSMnNiJk,2213 -pydantic/plugin/_schema_validator.py,sha256=5M5Ic1bZnjhNDxtRDVKbRPTQ6po6QuKMY7MguMkHeW0,5445 -pydantic/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -pydantic/root_model.py,sha256=cOMoeWdp536KF85uRIcW-oz1O7d5GRCpx9HTS4-1uf8,6394 -pydantic/schema.py,sha256=Vqqjvq_LnapVknebUd3Bp_J1p2gXZZnZRgL48bVEG7o,142 -pydantic/tools.py,sha256=iHQpd8SJ5DCTtPV5atAV06T89bjSaMFeZZ2LX9lasZY,141 -pydantic/type_adapter.py,sha256=T05g8WQczBsVU_35RdKxgjIi7Y7LIip67pF3NI7X4GE,36123 -pydantic/types.py,sha256=fpYcGnAncK4QjaFm3jZtqYwxJYzB7rJJPfAl89emzyQ,105961 -pydantic/typing.py,sha256=P7feA35MwTcLsR1uL7db0S-oydBxobmXa55YDoBgajQ,138 -pydantic/utils.py,sha256=15nR2QpqTBFlQV4TNtTItMyTJx_fbyV-gPmIEY1Gooc,141 -pydantic/v1/__init__.py,sha256=SxQPklgBs4XHJwE6BZ9qoewYoGiNyYUnmHzEFCZbfnI,2946 -pydantic/v1/__pycache__/__init__.cpython-314.pyc,, -pydantic/v1/__pycache__/_hypothesis_plugin.cpython-314.pyc,, -pydantic/v1/__pycache__/annotated_types.cpython-314.pyc,, -pydantic/v1/__pycache__/class_validators.cpython-314.pyc,, -pydantic/v1/__pycache__/color.cpython-314.pyc,, -pydantic/v1/__pycache__/config.cpython-314.pyc,, -pydantic/v1/__pycache__/dataclasses.cpython-314.pyc,, -pydantic/v1/__pycache__/datetime_parse.cpython-314.pyc,, -pydantic/v1/__pycache__/decorator.cpython-314.pyc,, -pydantic/v1/__pycache__/env_settings.cpython-314.pyc,, -pydantic/v1/__pycache__/error_wrappers.cpython-314.pyc,, -pydantic/v1/__pycache__/errors.cpython-314.pyc,, -pydantic/v1/__pycache__/fields.cpython-314.pyc,, -pydantic/v1/__pycache__/generics.cpython-314.pyc,, -pydantic/v1/__pycache__/json.cpython-314.pyc,, -pydantic/v1/__pycache__/main.cpython-314.pyc,, -pydantic/v1/__pycache__/mypy.cpython-314.pyc,, -pydantic/v1/__pycache__/networks.cpython-314.pyc,, -pydantic/v1/__pycache__/parse.cpython-314.pyc,, -pydantic/v1/__pycache__/schema.cpython-314.pyc,, -pydantic/v1/__pycache__/tools.cpython-314.pyc,, -pydantic/v1/__pycache__/types.cpython-314.pyc,, -pydantic/v1/__pycache__/typing.cpython-314.pyc,, -pydantic/v1/__pycache__/utils.cpython-314.pyc,, -pydantic/v1/__pycache__/validators.cpython-314.pyc,, -pydantic/v1/__pycache__/version.cpython-314.pyc,, -pydantic/v1/_hypothesis_plugin.py,sha256=5ES5xWuw1FQAsymLezy8QgnVz0ZpVfU3jkmT74H27VQ,14847 -pydantic/v1/annotated_types.py,sha256=uk2NAAxqiNELKjiHhyhxKaIOh8F1lYW_LzrW3X7oZBc,3157 -pydantic/v1/class_validators.py,sha256=ULOaIUgYUDBsHL7EEVEarcM-UubKUggoN8hSbDonsFE,14672 -pydantic/v1/color.py,sha256=iZABLYp6OVoo2AFkP9Ipri_wSc6-Kklu8YuhSartd5g,16844 -pydantic/v1/config.py,sha256=a6P0Wer9x4cbwKW7Xv8poSUqM4WP-RLWwX6YMpYq9AA,6532 -pydantic/v1/dataclasses.py,sha256=784cqvInbwIPWr9usfpX3ch7z4t3J2tTK6N067_wk1o,18172 -pydantic/v1/datetime_parse.py,sha256=4Qy1kQpq3rNVZJeIHeSPDpuS2Bvhp1KPtzJG1xu-H00,7724 -pydantic/v1/decorator.py,sha256=zaaxxxoWPCm818D1bs0yhapRjXm32V8G0ZHWCdM1uXA,10339 -pydantic/v1/env_settings.py,sha256=A9VXwtRl02AY-jH0C0ouy5VNw3fi6F_pkzuHDjgAAOM,14105 -pydantic/v1/error_wrappers.py,sha256=6625Mfw9qkC2NwitB_JFAWe8B-Xv6zBU7rL9k28tfyo,5196 -pydantic/v1/errors.py,sha256=mIwPED5vGM5Q5v4C4Z1JPldTRH-omvEylH6ksMhOmPw,17726 -pydantic/v1/fields.py,sha256=VqWJCriUNiEyptXroDVJ501JpVA0en2VANcksqXL2b8,50649 -pydantic/v1/generics.py,sha256=YzyKTZN6x5Q1RGJ3WQ9jN-uwHJxL3W4qoZqwcZXqxWg,17829 -pydantic/v1/json.py,sha256=WQ5Hy_hIpfdR3YS8k6N2E6KMJzsdbBi_ldWOPJaV81M,3390 -pydantic/v1/main.py,sha256=vRB1TbpkzPN3P5ijJlc-cjNuO-HciNOpC4b8K3zZnfc,45697 -pydantic/v1/mypy.py,sha256=Cl8XRfCmIcVE3j5AEU52C8iDh8lcX__D3hz2jIWxMAs,38860 -pydantic/v1/networks.py,sha256=HYNtKAfOmOnKJpsDg1g6SIkj9WPhU_-i8l5e2JKBpG4,22124 -pydantic/v1/parse.py,sha256=BJtdqiZRtav9VRFCmOxoY-KImQmjPy-A_NoojiFUZxY,1821 -pydantic/v1/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -pydantic/v1/schema.py,sha256=aqBuA--cq8gAVkim5BJPFASHzOZ8dFtmFX_fNGr6ip4,47801 -pydantic/v1/tools.py,sha256=1lDdXHk0jL5uP3u5RCYAvUAlGClgAO-45lkq9j7fyBA,2881 -pydantic/v1/types.py,sha256=Bzl-RcnitPBHnqwwj9iv7JjHuN1GpnWH24dKkF3l9e8,35455 -pydantic/v1/typing.py,sha256=ovwtLpEZCbnghZaHfSNJupzetzHNkLXjn_66kgTnIV4,20102 -pydantic/v1/utils.py,sha256=1PqOIlz6OVWwGds3HBKlw4Et6asFou0UUpAto7jFOCs,26014 -pydantic/v1/validators.py,sha256=lyUkn1MWhHxlCX5ZfEgFj_CAHojoiPcaQeMdEM9XviU,22187 -pydantic/v1/version.py,sha256=YpHWOQKtGoxfyikzGrcmXJVKUVYB9EBdoCR994QxSnE,1039 -pydantic/validate_call_decorator.py,sha256=VLAi4hoFpjC-1eL0HixYaaWaEFO6htGcDctgsqa5VII,4416 -pydantic/validators.py,sha256=pwbIJXVb1CV2mAE4w_EGfNj7DwzsKaWw_tTL6cviTus,146 -pydantic/version.py,sha256=T5rziwDPrMjfaU0X--5fBdGVvh94QznNQ7QjD4LfgmA,3985 -pydantic/warnings.py,sha256=3QyQo6lN35cO7OXDbFEXWcNaPlGCRaregVZA-G-lZwI,4822 diff --git a/functions/python/data-processor/pydantic-2.13.4.dist-info/WHEEL b/functions/python/data-processor/pydantic-2.13.4.dist-info/WHEEL deleted file mode 100644 index b1b94fd58..000000000 --- a/functions/python/data-processor/pydantic-2.13.4.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.29.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/pydantic-2.13.4.dist-info/licenses/LICENSE b/functions/python/data-processor/pydantic-2.13.4.dist-info/licenses/LICENSE deleted file mode 100644 index 488c6260c..000000000 --- a/functions/python/data-processor/pydantic-2.13.4.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2017 to present Pydantic Services Inc. and individual contributors. - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/functions/python/data-processor/pydantic/__init__.py b/functions/python/data-processor/pydantic/__init__.py deleted file mode 100644 index 01212849e..000000000 --- a/functions/python/data-processor/pydantic/__init__.py +++ /dev/null @@ -1,456 +0,0 @@ -from importlib import import_module -from typing import TYPE_CHECKING -from warnings import warn - -from ._migration import getattr_migration -from .version import VERSION, _ensure_pydantic_core_version - -_ensure_pydantic_core_version() -del _ensure_pydantic_core_version - -if TYPE_CHECKING: - # import of virtually everything is supported via `__getattr__` below, - # but we need them here for type checking and IDE support - import pydantic_core - from pydantic_core.core_schema import ( - FieldSerializationInfo, - SerializationInfo, - SerializerFunctionWrapHandler, - ValidationInfo, - ValidatorFunctionWrapHandler, - ) - - from . import dataclasses - from .aliases import AliasChoices, AliasGenerator, AliasPath - from .annotated_handlers import GetCoreSchemaHandler, GetJsonSchemaHandler - from .config import ConfigDict, with_config - from .errors import * - from .fields import Field, PrivateAttr, computed_field - from .functional_serializers import ( - PlainSerializer, - SerializeAsAny, - WrapSerializer, - field_serializer, - model_serializer, - ) - from .functional_validators import ( - AfterValidator, - BeforeValidator, - InstanceOf, - ModelWrapValidatorHandler, - PlainValidator, - SkipValidation, - ValidateAs, - WrapValidator, - field_validator, - model_validator, - ) - from .json_schema import WithJsonSchema - from .main import * - from .networks import * - from .type_adapter import TypeAdapter - from .types import * - from .validate_call_decorator import validate_call - from .warnings import ( - PydanticDeprecatedSince20, - PydanticDeprecatedSince26, - PydanticDeprecatedSince29, - PydanticDeprecatedSince210, - PydanticDeprecatedSince211, - PydanticDeprecatedSince212, - PydanticDeprecationWarning, - PydanticExperimentalWarning, - ) - - # this encourages pycharm to import `ValidationError` from here, not pydantic_core - ValidationError = pydantic_core.ValidationError - from .deprecated.class_validators import root_validator, validator - from .deprecated.config import BaseConfig, Extra - from .deprecated.tools import * - from .root_model import RootModel - -__version__ = VERSION -__all__ = ( - # dataclasses - 'dataclasses', - # functional validators - 'field_validator', - 'model_validator', - 'AfterValidator', - 'BeforeValidator', - 'PlainValidator', - 'WrapValidator', - 'SkipValidation', - 'ValidateAs', - 'InstanceOf', - 'ModelWrapValidatorHandler', - # JSON Schema - 'WithJsonSchema', - # deprecated V1 functional validators, these are imported via `__getattr__` below - 'root_validator', - 'validator', - # functional serializers - 'field_serializer', - 'model_serializer', - 'PlainSerializer', - 'SerializeAsAny', - 'WrapSerializer', - # config - 'ConfigDict', - 'with_config', - # deprecated V1 config, these are imported via `__getattr__` below - 'BaseConfig', - 'Extra', - # validate_call - 'validate_call', - # errors - 'PydanticErrorCodes', - 'PydanticUserError', - 'PydanticSchemaGenerationError', - 'PydanticImportError', - 'PydanticUndefinedAnnotation', - 'PydanticInvalidForJsonSchema', - 'PydanticForbiddenQualifier', - # fields - 'Field', - 'computed_field', - 'PrivateAttr', - # alias - 'AliasChoices', - 'AliasGenerator', - 'AliasPath', - # main - 'BaseModel', - 'create_model', - # network - 'AnyUrl', - 'AnyHttpUrl', - 'FileUrl', - 'HttpUrl', - 'FtpUrl', - 'WebsocketUrl', - 'AnyWebsocketUrl', - 'UrlConstraints', - 'EmailStr', - 'NameEmail', - 'IPvAnyAddress', - 'IPvAnyInterface', - 'IPvAnyNetwork', - 'PostgresDsn', - 'CockroachDsn', - 'AmqpDsn', - 'RedisDsn', - 'MongoDsn', - 'KafkaDsn', - 'NatsDsn', - 'MySQLDsn', - 'MariaDBDsn', - 'ClickHouseDsn', - 'SnowflakeDsn', - 'validate_email', - # root_model - 'RootModel', - # deprecated tools, these are imported via `__getattr__` below - 'parse_obj_as', - 'schema_of', - 'schema_json_of', - # types - 'Strict', - 'StrictStr', - 'conbytes', - 'conlist', - 'conset', - 'confrozenset', - 'constr', - 'StringConstraints', - 'ImportString', - 'conint', - 'PositiveInt', - 'NegativeInt', - 'NonNegativeInt', - 'NonPositiveInt', - 'confloat', - 'PositiveFloat', - 'NegativeFloat', - 'NonNegativeFloat', - 'NonPositiveFloat', - 'FiniteFloat', - 'condecimal', - 'condate', - 'UUID1', - 'UUID3', - 'UUID4', - 'UUID5', - 'UUID6', - 'UUID7', - 'UUID8', - 'FilePath', - 'DirectoryPath', - 'NewPath', - 'Json', - 'Secret', - 'SecretStr', - 'SecretBytes', - 'SocketPath', - 'StrictBool', - 'StrictBytes', - 'StrictInt', - 'StrictFloat', - 'PaymentCardNumber', - 'ByteSize', - 'PastDate', - 'FutureDate', - 'PastDatetime', - 'FutureDatetime', - 'AwareDatetime', - 'NaiveDatetime', - 'AllowInfNan', - 'EncoderProtocol', - 'EncodedBytes', - 'EncodedStr', - 'Base64Encoder', - 'Base64Bytes', - 'Base64Str', - 'Base64UrlBytes', - 'Base64UrlStr', - 'GetPydanticSchema', - 'Tag', - 'Discriminator', - 'JsonValue', - 'FailFast', - # type_adapter - 'TypeAdapter', - # version - '__version__', - 'VERSION', - # warnings - 'PydanticDeprecatedSince20', - 'PydanticDeprecatedSince26', - 'PydanticDeprecatedSince29', - 'PydanticDeprecatedSince210', - 'PydanticDeprecatedSince211', - 'PydanticDeprecatedSince212', - 'PydanticDeprecationWarning', - 'PydanticExperimentalWarning', - # annotated handlers - 'GetCoreSchemaHandler', - 'GetJsonSchemaHandler', - # pydantic_core - 'ValidationError', - 'ValidationInfo', - 'SerializationInfo', - 'ValidatorFunctionWrapHandler', - 'FieldSerializationInfo', - 'SerializerFunctionWrapHandler', - 'OnErrorOmit', -) - -# A mapping of {: (package, )} defining dynamic imports -_dynamic_imports: 'dict[str, tuple[str, str]]' = { - 'dataclasses': (__spec__.parent, '__module__'), - # functional validators - 'field_validator': (__spec__.parent, '.functional_validators'), - 'model_validator': (__spec__.parent, '.functional_validators'), - 'AfterValidator': (__spec__.parent, '.functional_validators'), - 'BeforeValidator': (__spec__.parent, '.functional_validators'), - 'PlainValidator': (__spec__.parent, '.functional_validators'), - 'WrapValidator': (__spec__.parent, '.functional_validators'), - 'SkipValidation': (__spec__.parent, '.functional_validators'), - 'InstanceOf': (__spec__.parent, '.functional_validators'), - 'ValidateAs': (__spec__.parent, '.functional_validators'), - 'ModelWrapValidatorHandler': (__spec__.parent, '.functional_validators'), - # JSON Schema - 'WithJsonSchema': (__spec__.parent, '.json_schema'), - # functional serializers - 'field_serializer': (__spec__.parent, '.functional_serializers'), - 'model_serializer': (__spec__.parent, '.functional_serializers'), - 'PlainSerializer': (__spec__.parent, '.functional_serializers'), - 'SerializeAsAny': (__spec__.parent, '.functional_serializers'), - 'WrapSerializer': (__spec__.parent, '.functional_serializers'), - # config - 'ConfigDict': (__spec__.parent, '.config'), - 'with_config': (__spec__.parent, '.config'), - # validate call - 'validate_call': (__spec__.parent, '.validate_call_decorator'), - # errors - 'PydanticErrorCodes': (__spec__.parent, '.errors'), - 'PydanticUserError': (__spec__.parent, '.errors'), - 'PydanticSchemaGenerationError': (__spec__.parent, '.errors'), - 'PydanticImportError': (__spec__.parent, '.errors'), - 'PydanticUndefinedAnnotation': (__spec__.parent, '.errors'), - 'PydanticInvalidForJsonSchema': (__spec__.parent, '.errors'), - 'PydanticForbiddenQualifier': (__spec__.parent, '.errors'), - # fields - 'Field': (__spec__.parent, '.fields'), - 'computed_field': (__spec__.parent, '.fields'), - 'PrivateAttr': (__spec__.parent, '.fields'), - # alias - 'AliasChoices': (__spec__.parent, '.aliases'), - 'AliasGenerator': (__spec__.parent, '.aliases'), - 'AliasPath': (__spec__.parent, '.aliases'), - # main - 'BaseModel': (__spec__.parent, '.main'), - 'create_model': (__spec__.parent, '.main'), - # network - 'AnyUrl': (__spec__.parent, '.networks'), - 'AnyHttpUrl': (__spec__.parent, '.networks'), - 'FileUrl': (__spec__.parent, '.networks'), - 'HttpUrl': (__spec__.parent, '.networks'), - 'FtpUrl': (__spec__.parent, '.networks'), - 'WebsocketUrl': (__spec__.parent, '.networks'), - 'AnyWebsocketUrl': (__spec__.parent, '.networks'), - 'UrlConstraints': (__spec__.parent, '.networks'), - 'EmailStr': (__spec__.parent, '.networks'), - 'NameEmail': (__spec__.parent, '.networks'), - 'IPvAnyAddress': (__spec__.parent, '.networks'), - 'IPvAnyInterface': (__spec__.parent, '.networks'), - 'IPvAnyNetwork': (__spec__.parent, '.networks'), - 'PostgresDsn': (__spec__.parent, '.networks'), - 'CockroachDsn': (__spec__.parent, '.networks'), - 'AmqpDsn': (__spec__.parent, '.networks'), - 'RedisDsn': (__spec__.parent, '.networks'), - 'MongoDsn': (__spec__.parent, '.networks'), - 'KafkaDsn': (__spec__.parent, '.networks'), - 'NatsDsn': (__spec__.parent, '.networks'), - 'MySQLDsn': (__spec__.parent, '.networks'), - 'MariaDBDsn': (__spec__.parent, '.networks'), - 'ClickHouseDsn': (__spec__.parent, '.networks'), - 'SnowflakeDsn': (__spec__.parent, '.networks'), - 'validate_email': (__spec__.parent, '.networks'), - # root_model - 'RootModel': (__spec__.parent, '.root_model'), - # types - 'Strict': (__spec__.parent, '.types'), - 'StrictStr': (__spec__.parent, '.types'), - 'conbytes': (__spec__.parent, '.types'), - 'conlist': (__spec__.parent, '.types'), - 'conset': (__spec__.parent, '.types'), - 'confrozenset': (__spec__.parent, '.types'), - 'constr': (__spec__.parent, '.types'), - 'StringConstraints': (__spec__.parent, '.types'), - 'ImportString': (__spec__.parent, '.types'), - 'conint': (__spec__.parent, '.types'), - 'PositiveInt': (__spec__.parent, '.types'), - 'NegativeInt': (__spec__.parent, '.types'), - 'NonNegativeInt': (__spec__.parent, '.types'), - 'NonPositiveInt': (__spec__.parent, '.types'), - 'confloat': (__spec__.parent, '.types'), - 'PositiveFloat': (__spec__.parent, '.types'), - 'NegativeFloat': (__spec__.parent, '.types'), - 'NonNegativeFloat': (__spec__.parent, '.types'), - 'NonPositiveFloat': (__spec__.parent, '.types'), - 'FiniteFloat': (__spec__.parent, '.types'), - 'condecimal': (__spec__.parent, '.types'), - 'condate': (__spec__.parent, '.types'), - 'UUID1': (__spec__.parent, '.types'), - 'UUID3': (__spec__.parent, '.types'), - 'UUID4': (__spec__.parent, '.types'), - 'UUID5': (__spec__.parent, '.types'), - 'UUID6': (__spec__.parent, '.types'), - 'UUID7': (__spec__.parent, '.types'), - 'UUID8': (__spec__.parent, '.types'), - 'FilePath': (__spec__.parent, '.types'), - 'DirectoryPath': (__spec__.parent, '.types'), - 'NewPath': (__spec__.parent, '.types'), - 'Json': (__spec__.parent, '.types'), - 'Secret': (__spec__.parent, '.types'), - 'SecretStr': (__spec__.parent, '.types'), - 'SecretBytes': (__spec__.parent, '.types'), - 'StrictBool': (__spec__.parent, '.types'), - 'StrictBytes': (__spec__.parent, '.types'), - 'StrictInt': (__spec__.parent, '.types'), - 'StrictFloat': (__spec__.parent, '.types'), - 'PaymentCardNumber': (__spec__.parent, '.types'), - 'ByteSize': (__spec__.parent, '.types'), - 'PastDate': (__spec__.parent, '.types'), - 'SocketPath': (__spec__.parent, '.types'), - 'FutureDate': (__spec__.parent, '.types'), - 'PastDatetime': (__spec__.parent, '.types'), - 'FutureDatetime': (__spec__.parent, '.types'), - 'AwareDatetime': (__spec__.parent, '.types'), - 'NaiveDatetime': (__spec__.parent, '.types'), - 'AllowInfNan': (__spec__.parent, '.types'), - 'EncoderProtocol': (__spec__.parent, '.types'), - 'EncodedBytes': (__spec__.parent, '.types'), - 'EncodedStr': (__spec__.parent, '.types'), - 'Base64Encoder': (__spec__.parent, '.types'), - 'Base64Bytes': (__spec__.parent, '.types'), - 'Base64Str': (__spec__.parent, '.types'), - 'Base64UrlBytes': (__spec__.parent, '.types'), - 'Base64UrlStr': (__spec__.parent, '.types'), - 'GetPydanticSchema': (__spec__.parent, '.types'), - 'Tag': (__spec__.parent, '.types'), - 'Discriminator': (__spec__.parent, '.types'), - 'JsonValue': (__spec__.parent, '.types'), - 'OnErrorOmit': (__spec__.parent, '.types'), - 'FailFast': (__spec__.parent, '.types'), - # type_adapter - 'TypeAdapter': (__spec__.parent, '.type_adapter'), - # warnings - 'PydanticDeprecatedSince20': (__spec__.parent, '.warnings'), - 'PydanticDeprecatedSince26': (__spec__.parent, '.warnings'), - 'PydanticDeprecatedSince29': (__spec__.parent, '.warnings'), - 'PydanticDeprecatedSince210': (__spec__.parent, '.warnings'), - 'PydanticDeprecatedSince211': (__spec__.parent, '.warnings'), - 'PydanticDeprecatedSince212': (__spec__.parent, '.warnings'), - 'PydanticDeprecationWarning': (__spec__.parent, '.warnings'), - 'PydanticExperimentalWarning': (__spec__.parent, '.warnings'), - # annotated handlers - 'GetCoreSchemaHandler': (__spec__.parent, '.annotated_handlers'), - 'GetJsonSchemaHandler': (__spec__.parent, '.annotated_handlers'), - # pydantic_core stuff - 'ValidationError': ('pydantic_core', '.'), - 'ValidationInfo': ('pydantic_core', '.core_schema'), - 'SerializationInfo': ('pydantic_core', '.core_schema'), - 'ValidatorFunctionWrapHandler': ('pydantic_core', '.core_schema'), - 'FieldSerializationInfo': ('pydantic_core', '.core_schema'), - 'SerializerFunctionWrapHandler': ('pydantic_core', '.core_schema'), - # deprecated, mostly not included in __all__ - 'root_validator': (__spec__.parent, '.deprecated.class_validators'), - 'validator': (__spec__.parent, '.deprecated.class_validators'), - 'BaseConfig': (__spec__.parent, '.deprecated.config'), - 'Extra': (__spec__.parent, '.deprecated.config'), - 'parse_obj_as': (__spec__.parent, '.deprecated.tools'), - 'schema_of': (__spec__.parent, '.deprecated.tools'), - 'schema_json_of': (__spec__.parent, '.deprecated.tools'), - # deprecated dynamic imports - 'FieldValidationInfo': ('pydantic_core', '.core_schema'), - 'GenerateSchema': (__spec__.parent, '._internal._generate_schema'), -} -_deprecated_dynamic_imports = {'FieldValidationInfo', 'GenerateSchema'} - -_getattr_migration = getattr_migration(__name__) - - -def __getattr__(attr_name: str) -> object: - if attr_name in _deprecated_dynamic_imports: - from pydantic.warnings import PydanticDeprecatedSince20 - - warn( - f'Importing {attr_name} from `pydantic` is deprecated. This feature is either no longer supported, or is not public.', - PydanticDeprecatedSince20, - stacklevel=2, - ) - - dynamic_attr = _dynamic_imports.get(attr_name) - if dynamic_attr is None: - return _getattr_migration(attr_name) - - package, module_name = dynamic_attr - - if module_name == '__module__': - result = import_module(f'.{attr_name}', package=package) - globals()[attr_name] = result - return result - else: - module = import_module(module_name, package=package) - result = getattr(module, attr_name) - g = globals() - for k, (_, v_module_name) in _dynamic_imports.items(): - if v_module_name == module_name and k not in _deprecated_dynamic_imports: - g[k] = getattr(module, k) - return result - - -def __dir__() -> list[str]: - return list(__all__) diff --git a/functions/python/data-processor/pydantic/_internal/__init__.py b/functions/python/data-processor/pydantic/_internal/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/pydantic/_internal/_config.py b/functions/python/data-processor/pydantic/_internal/_config.py deleted file mode 100644 index 73b587089..000000000 --- a/functions/python/data-processor/pydantic/_internal/_config.py +++ /dev/null @@ -1,386 +0,0 @@ -from __future__ import annotations as _annotations - -import warnings -from contextlib import contextmanager -from re import Pattern -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Literal, - cast, -) - -from pydantic_core import core_schema -from typing_extensions import Self - -from ..aliases import AliasGenerator -from ..config import ConfigDict, ExtraValues, JsonDict, JsonEncoder, JsonSchemaExtraCallable -from ..errors import PydanticUserError -from ..warnings import PydanticDeprecatedSince20, PydanticDeprecatedSince210 - -if TYPE_CHECKING: - from .._internal._schema_generation_shared import GenerateSchema - from ..fields import ComputedFieldInfo, FieldInfo - -DEPRECATION_MESSAGE = 'Support for class-based `config` is deprecated, use ConfigDict instead.' - - -class ConfigWrapper: - """Internal wrapper for Config which exposes ConfigDict items as attributes.""" - - __slots__ = ('config_dict',) - - config_dict: ConfigDict - - # all annotations are copied directly from ConfigDict, and should be kept up to date, a test will fail if they - # stop matching - title: str | None - str_to_lower: bool - str_to_upper: bool - str_strip_whitespace: bool - str_min_length: int - str_max_length: int | None - extra: ExtraValues | None - frozen: bool - populate_by_name: bool - use_enum_values: bool - validate_assignment: bool - arbitrary_types_allowed: bool - from_attributes: bool - # whether to use the actual key provided in the data (e.g. alias or first alias for "field required" errors) instead of field_names - # to construct error `loc`s, default `True` - loc_by_alias: bool - alias_generator: Callable[[str], str] | AliasGenerator | None - model_title_generator: Callable[[type], str] | None - field_title_generator: Callable[[str, FieldInfo | ComputedFieldInfo], str] | None - ignored_types: tuple[type, ...] - allow_inf_nan: bool - json_schema_extra: JsonDict | JsonSchemaExtraCallable | None - json_encoders: dict[type[object], JsonEncoder] | None - - # new in V2 - strict: bool - # whether instances of models and dataclasses (including subclass instances) should re-validate, default 'never' - revalidate_instances: Literal['always', 'never', 'subclass-instances'] - ser_json_timedelta: Literal['iso8601', 'float'] - ser_json_temporal: Literal['iso8601', 'seconds', 'milliseconds'] - val_temporal_unit: Literal['seconds', 'milliseconds', 'infer'] - ser_json_bytes: Literal['utf8', 'base64', 'hex'] - val_json_bytes: Literal['utf8', 'base64', 'hex'] - ser_json_inf_nan: Literal['null', 'constants', 'strings'] - # whether to validate default values during validation, default False - validate_default: bool - validate_return: bool - protected_namespaces: tuple[str | Pattern[str], ...] - hide_input_in_errors: bool - defer_build: bool - plugin_settings: dict[str, object] | None - schema_generator: type[GenerateSchema] | None - json_schema_serialization_defaults_required: bool - json_schema_mode_override: Literal['validation', 'serialization', None] - coerce_numbers_to_str: bool - regex_engine: Literal['rust-regex', 'python-re'] - validation_error_cause: bool - use_attribute_docstrings: bool - cache_strings: bool | Literal['all', 'keys', 'none'] - validate_by_alias: bool - validate_by_name: bool - serialize_by_alias: bool - url_preserve_empty_path: bool - polymorphic_serialization: bool - - def __init__(self, config: ConfigDict | dict[str, Any] | type[Any] | None, *, check: bool = True): - if check: - self.config_dict = prepare_config(config) - else: - self.config_dict = cast(ConfigDict, config) - - @classmethod - def for_model( - cls, - bases: tuple[type[Any], ...], - namespace: dict[str, Any], - raw_annotations: dict[str, Any], - kwargs: dict[str, Any], - ) -> Self: - """Build a new `ConfigWrapper` instance for a `BaseModel`. - - The config wrapper built based on (in descending order of priority): - - options from `kwargs` - - options from the `namespace` - - options from the base classes (`bases`) - - Args: - bases: A tuple of base classes. - namespace: The namespace of the class being created. - raw_annotations: The (non-evaluated) annotations of the model. - kwargs: The kwargs passed to the class being created. - - Returns: - A `ConfigWrapper` instance for `BaseModel`. - """ - config_new = ConfigDict() - for base in bases: - config = getattr(base, 'model_config', None) - if config: - config_new.update(config.copy()) - - config_class_from_namespace = namespace.get('Config') - config_dict_from_namespace = namespace.get('model_config') - - if raw_annotations.get('model_config') and config_dict_from_namespace is None: - raise PydanticUserError( - '`model_config` cannot be used as a model field name. Use `model_config` for model configuration.', - code='model-config-invalid-field-name', - ) - - if config_class_from_namespace and config_dict_from_namespace: - raise PydanticUserError('"Config" and "model_config" cannot be used together', code='config-both') - - config_from_namespace = config_dict_from_namespace or prepare_config(config_class_from_namespace) - - config_new.update(config_from_namespace) - - for k in list(kwargs.keys()): - if k in config_keys: - config_new[k] = kwargs.pop(k) - - return cls(config_new) - - # we don't show `__getattr__` to type checkers so missing attributes cause errors - if not TYPE_CHECKING: # pragma: no branch - - def __getattr__(self, name: str) -> Any: - try: - return self.config_dict[name] - except KeyError: - try: - return config_defaults[name] - except KeyError: - raise AttributeError(f'Config has no attribute {name!r}') from None - - def core_config(self, title: str | None) -> core_schema.CoreConfig: - """Create a pydantic-core config. - - We don't use getattr here since we don't want to populate with defaults. - - Args: - title: The title to use if not set in config. - - Returns: - A `CoreConfig` object created from config. - """ - config = self.config_dict - - if config.get('schema_generator') is not None: - warnings.warn( - 'The `schema_generator` setting has been deprecated since v2.10. This setting no longer has any effect.', - PydanticDeprecatedSince210, - stacklevel=2, - ) - - if (populate_by_name := config.get('populate_by_name')) is not None: - # We include this patch for backwards compatibility purposes, but this config setting will be deprecated in v3.0, and likely removed in v4.0. - # Thus, the above warning and this patch can be removed then as well. - if config.get('validate_by_name') is None: - config['validate_by_alias'] = True - config['validate_by_name'] = populate_by_name - - # We dynamically patch validate_by_name to be True if validate_by_alias is set to False - # and validate_by_name is not explicitly set. - if config.get('validate_by_alias') is False and config.get('validate_by_name') is None: - config['validate_by_name'] = True - - if (not config.get('validate_by_alias', True)) and (not config.get('validate_by_name', False)): - raise PydanticUserError( - 'At least one of `validate_by_alias` or `validate_by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return core_schema.CoreConfig( - **{ # pyright: ignore[reportArgumentType] - k: v - for k, v in ( - ('title', config.get('title') or title or None), - ('extra_fields_behavior', config.get('extra')), - ('allow_inf_nan', config.get('allow_inf_nan')), - ('str_strip_whitespace', config.get('str_strip_whitespace')), - ('str_to_lower', config.get('str_to_lower')), - ('str_to_upper', config.get('str_to_upper')), - ('strict', config.get('strict')), - ('ser_json_timedelta', config.get('ser_json_timedelta')), - ('ser_json_temporal', config.get('ser_json_temporal')), - ('val_temporal_unit', config.get('val_temporal_unit')), - ('ser_json_bytes', config.get('ser_json_bytes')), - ('val_json_bytes', config.get('val_json_bytes')), - ('ser_json_inf_nan', config.get('ser_json_inf_nan')), - ('from_attributes', config.get('from_attributes')), - ('loc_by_alias', config.get('loc_by_alias')), - ('revalidate_instances', config.get('revalidate_instances')), - ('validate_default', config.get('validate_default')), - ('str_max_length', config.get('str_max_length')), - ('str_min_length', config.get('str_min_length')), - ('hide_input_in_errors', config.get('hide_input_in_errors')), - ('coerce_numbers_to_str', config.get('coerce_numbers_to_str')), - ('regex_engine', config.get('regex_engine')), - ('validation_error_cause', config.get('validation_error_cause')), - ('cache_strings', config.get('cache_strings')), - ('validate_by_alias', config.get('validate_by_alias')), - ('validate_by_name', config.get('validate_by_name')), - ('serialize_by_alias', config.get('serialize_by_alias')), - ('url_preserve_empty_path', config.get('url_preserve_empty_path')), - ('polymorphic_serialization', config.get('polymorphic_serialization')), - ) - if v is not None - } - ) - - def __repr__(self): - c = ', '.join(f'{k}={v!r}' for k, v in self.config_dict.items()) - return f'ConfigWrapper({c})' - - -class ConfigWrapperStack: - """A stack of `ConfigWrapper` instances.""" - - def __init__(self, config_wrapper: ConfigWrapper): - self._config_wrapper_stack: list[ConfigWrapper] = [config_wrapper] - - @property - def tail(self) -> ConfigWrapper: - return self._config_wrapper_stack[-1] - - @contextmanager - def push(self, config_wrapper: ConfigWrapper | ConfigDict | None): - if config_wrapper is None: - yield - return - - if not isinstance(config_wrapper, ConfigWrapper): - config_wrapper = ConfigWrapper(config_wrapper, check=False) - - self._config_wrapper_stack.append(config_wrapper) - try: - yield - finally: - self._config_wrapper_stack.pop() - - -config_defaults = ConfigDict( - title=None, - str_to_lower=False, - str_to_upper=False, - str_strip_whitespace=False, - str_min_length=0, - str_max_length=None, - # let the model / dataclass decide how to handle it - extra=None, - frozen=False, - populate_by_name=False, - use_enum_values=False, - validate_assignment=False, - arbitrary_types_allowed=False, - from_attributes=False, - loc_by_alias=True, - alias_generator=None, - model_title_generator=None, - field_title_generator=None, - ignored_types=(), - allow_inf_nan=True, - json_schema_extra=None, - strict=False, - revalidate_instances='never', - ser_json_timedelta='iso8601', - ser_json_temporal='iso8601', - val_temporal_unit='infer', - ser_json_bytes='utf8', - val_json_bytes='utf8', - ser_json_inf_nan='null', - validate_default=False, - validate_return=False, - protected_namespaces=('model_validate', 'model_dump'), - hide_input_in_errors=False, - json_encoders=None, - defer_build=False, - schema_generator=None, - plugin_settings=None, - json_schema_serialization_defaults_required=False, - json_schema_mode_override=None, - coerce_numbers_to_str=False, - regex_engine='rust-regex', - validation_error_cause=False, - use_attribute_docstrings=False, - cache_strings=True, - validate_by_alias=True, - validate_by_name=False, - serialize_by_alias=False, - url_preserve_empty_path=False, - polymorphic_serialization=False, -) - - -def prepare_config(config: ConfigDict | dict[str, Any] | type[Any] | None) -> ConfigDict: - """Create a `ConfigDict` instance from an existing dict, a class (e.g. old class-based config) or None. - - Args: - config: The input config. - - Returns: - A ConfigDict object created from config. - """ - if config is None: - return ConfigDict() - - if not isinstance(config, dict): - warnings.warn(DEPRECATION_MESSAGE, PydanticDeprecatedSince20, stacklevel=4) - config = {k: getattr(config, k) for k in dir(config) if not k.startswith('__')} - - config_dict = cast(ConfigDict, config) - check_deprecated(config_dict) - return config_dict - - -config_keys = set(ConfigDict.__annotations__.keys()) - - -V2_REMOVED_KEYS = { - 'allow_mutation', - 'error_msg_templates', - 'fields', - 'getter_dict', - 'smart_union', - 'underscore_attrs_are_private', - 'json_loads', - 'json_dumps', - 'copy_on_model_validation', - 'post_init_call', -} -V2_RENAMED_KEYS = { - 'allow_population_by_field_name': 'validate_by_name', - 'anystr_lower': 'str_to_lower', - 'anystr_strip_whitespace': 'str_strip_whitespace', - 'anystr_upper': 'str_to_upper', - 'keep_untouched': 'ignored_types', - 'max_anystr_length': 'str_max_length', - 'min_anystr_length': 'str_min_length', - 'orm_mode': 'from_attributes', - 'schema_extra': 'json_schema_extra', - 'validate_all': 'validate_default', -} - - -def check_deprecated(config_dict: ConfigDict) -> None: - """Check for deprecated config keys and warn the user. - - Args: - config_dict: The input config. - """ - deprecated_removed_keys = V2_REMOVED_KEYS & config_dict.keys() - deprecated_renamed_keys = V2_RENAMED_KEYS.keys() & config_dict.keys() - if deprecated_removed_keys or deprecated_renamed_keys: - renamings = {k: V2_RENAMED_KEYS[k] for k in sorted(deprecated_renamed_keys)} - renamed_bullets = [f'* {k!r} has been renamed to {v!r}' for k, v in renamings.items()] - removed_bullets = [f'* {k!r} has been removed' for k in sorted(deprecated_removed_keys)] - message = '\n'.join(['Valid config keys have changed in V2:'] + renamed_bullets + removed_bullets) - warnings.warn(message, UserWarning) diff --git a/functions/python/data-processor/pydantic/_internal/_core_metadata.py b/functions/python/data-processor/pydantic/_internal/_core_metadata.py deleted file mode 100644 index 9f2510c03..000000000 --- a/functions/python/data-processor/pydantic/_internal/_core_metadata.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import annotations as _annotations - -from typing import TYPE_CHECKING, Any, TypedDict, cast -from warnings import warn - -if TYPE_CHECKING: - from ..config import JsonDict, JsonSchemaExtraCallable - from ._schema_generation_shared import ( - GetJsonSchemaFunction, - ) - - -class CoreMetadata(TypedDict, total=False): - """A `TypedDict` for holding the metadata dict of the schema. - - Attributes: - pydantic_js_functions: List of JSON schema functions that resolve refs during application. - pydantic_js_annotation_functions: List of JSON schema functions that don't resolve refs during application. - pydantic_js_prefer_positional_arguments: Whether JSON schema generator will - prefer positional over keyword arguments for an 'arguments' schema. - custom validation function. Only applies to before, plain, and wrap validators. - pydantic_js_updates: key / value pair updates to apply to the JSON schema for a type. - pydantic_js_extra: WIP, either key/value pair updates to apply to the JSON schema, or a custom callable. - pydantic_internal_union_tag_key: Used internally by the `Tag` metadata to specify the tag used for a discriminated union. - pydantic_internal_union_discriminator: Used internally to specify the discriminator value for a discriminated union - when the discriminator was applied to a `'definition-ref'` schema, and that reference was missing at the time - of the annotation application. - - TODO: Perhaps we should move this structure to pydantic-core. At the moment, though, - it's easier to iterate on if we leave it in pydantic until we feel there is a semi-stable API. - - TODO: It's unfortunate how functionally oriented JSON schema generation is, especially that which occurs during - the core schema generation process. It's inevitable that we need to store some json schema related information - on core schemas, given that we generate JSON schemas directly from core schemas. That being said, debugging related - issues is quite difficult when JSON schema information is disguised via dynamically defined functions. - """ - - pydantic_js_functions: list[GetJsonSchemaFunction] - pydantic_js_annotation_functions: list[GetJsonSchemaFunction] - pydantic_js_prefer_positional_arguments: bool - pydantic_js_updates: JsonDict - pydantic_js_extra: JsonDict | JsonSchemaExtraCallable - pydantic_internal_union_tag_key: str - pydantic_internal_union_discriminator: str - - -def update_core_metadata( - core_metadata: Any, - /, - *, - pydantic_js_functions: list[GetJsonSchemaFunction] | None = None, - pydantic_js_annotation_functions: list[GetJsonSchemaFunction] | None = None, - pydantic_js_updates: JsonDict | None = None, - pydantic_js_extra: JsonDict | JsonSchemaExtraCallable | None = None, -) -> None: - from ..json_schema import PydanticJsonSchemaWarning - - """Update CoreMetadata instance in place. When we make modifications in this function, they - take effect on the `core_metadata` reference passed in as the first (and only) positional argument. - - First, cast to `CoreMetadata`, then finish with a cast to `dict[str, Any]` for core schema compatibility. - We do this here, instead of before / after each call to this function so that this typing hack - can be easily removed if/when we move `CoreMetadata` to `pydantic-core`. - - For parameter descriptions, see `CoreMetadata` above. - """ - core_metadata = cast(CoreMetadata, core_metadata) - - if pydantic_js_functions: - core_metadata.setdefault('pydantic_js_functions', []).extend(pydantic_js_functions) - - if pydantic_js_annotation_functions: - core_metadata.setdefault('pydantic_js_annotation_functions', []).extend(pydantic_js_annotation_functions) - - if pydantic_js_updates: - if (existing_updates := core_metadata.get('pydantic_js_updates')) is not None: - core_metadata['pydantic_js_updates'] = {**existing_updates, **pydantic_js_updates} - else: - core_metadata['pydantic_js_updates'] = pydantic_js_updates - - if pydantic_js_extra is not None: - existing_pydantic_js_extra = core_metadata.get('pydantic_js_extra') - if existing_pydantic_js_extra is None: - core_metadata['pydantic_js_extra'] = pydantic_js_extra - if isinstance(existing_pydantic_js_extra, dict): - if isinstance(pydantic_js_extra, dict): - core_metadata['pydantic_js_extra'] = {**existing_pydantic_js_extra, **pydantic_js_extra} - if callable(pydantic_js_extra): - warn( - 'Composing `dict` and `callable` type `json_schema_extra` is not supported.' - 'The `callable` type is being ignored.' - "If you'd like support for this behavior, please open an issue on pydantic.", - PydanticJsonSchemaWarning, - ) - if callable(existing_pydantic_js_extra): - # if ever there's a case of a callable, we'll just keep the last json schema extra spec - core_metadata['pydantic_js_extra'] = pydantic_js_extra diff --git a/functions/python/data-processor/pydantic/_internal/_core_utils.py b/functions/python/data-processor/pydantic/_internal/_core_utils.py deleted file mode 100644 index caa51e8c4..000000000 --- a/functions/python/data-processor/pydantic/_internal/_core_utils.py +++ /dev/null @@ -1,174 +0,0 @@ -from __future__ import annotations - -import inspect -from collections.abc import Mapping, Sequence -from typing import TYPE_CHECKING, Any, Union - -from pydantic_core import CoreSchema, core_schema -from typing_extensions import TypeGuard, get_args, get_origin -from typing_inspection import typing_objects - -from . import _repr -from ._typing_extra import is_generic_alias - -if TYPE_CHECKING: - from rich.console import Console - -AnyFunctionSchema = Union[ - core_schema.AfterValidatorFunctionSchema, - core_schema.BeforeValidatorFunctionSchema, - core_schema.WrapValidatorFunctionSchema, - core_schema.PlainValidatorFunctionSchema, -] - - -FunctionSchemaWithInnerSchema = Union[ - core_schema.AfterValidatorFunctionSchema, - core_schema.BeforeValidatorFunctionSchema, - core_schema.WrapValidatorFunctionSchema, -] - -CoreSchemaField = Union[ - core_schema.ModelField, core_schema.DataclassField, core_schema.TypedDictField, core_schema.ComputedField -] -CoreSchemaOrField = Union[core_schema.CoreSchema, CoreSchemaField] - -_CORE_SCHEMA_FIELD_TYPES = {'typed-dict-field', 'dataclass-field', 'model-field', 'computed-field'} -_FUNCTION_WITH_INNER_SCHEMA_TYPES = {'function-before', 'function-after', 'function-wrap'} -_LIST_LIKE_SCHEMA_WITH_ITEMS_TYPES = {'list', 'set', 'frozenset'} - - -def is_core_schema( - schema: CoreSchemaOrField, -) -> TypeGuard[CoreSchema]: - return schema['type'] not in _CORE_SCHEMA_FIELD_TYPES - - -def is_core_schema_field( - schema: CoreSchemaOrField, -) -> TypeGuard[CoreSchemaField]: - return schema['type'] in _CORE_SCHEMA_FIELD_TYPES - - -def is_function_with_inner_schema( - schema: CoreSchemaOrField, -) -> TypeGuard[FunctionSchemaWithInnerSchema]: - return schema['type'] in _FUNCTION_WITH_INNER_SCHEMA_TYPES - - -def is_list_like_schema_with_items_schema( - schema: CoreSchema, -) -> TypeGuard[core_schema.ListSchema | core_schema.SetSchema | core_schema.FrozenSetSchema]: - return schema['type'] in _LIST_LIKE_SCHEMA_WITH_ITEMS_TYPES - - -def get_type_ref(type_: Any, args_override: tuple[type[Any], ...] | None = None) -> str: - """Produces the ref to be used for this type by pydantic_core's core schemas. - - This `args_override` argument was added for the purpose of creating valid recursive references - when creating generic models without needing to create a concrete class. - """ - origin = get_origin(type_) or type_ - - args = get_args(type_) if is_generic_alias(type_) else (args_override or ()) - generic_metadata = getattr(type_, '__pydantic_generic_metadata__', None) - if generic_metadata: - origin = generic_metadata['origin'] or origin - args = generic_metadata['args'] or args - - module_name = getattr(origin, '__module__', '') - if typing_objects.is_typealiastype(origin): - type_ref = f'{module_name}.{origin.__name__}:{id(origin)}' - else: - try: - qualname = getattr(origin, '__qualname__', f'') - except Exception: - qualname = getattr(origin, '__qualname__', '') - type_ref = f'{module_name}.{qualname}:{id(origin)}' - - arg_refs: list[str] = [] - for arg in args: - if isinstance(arg, str): - # Handle string literals as a special case; we may be able to remove this special handling if we - # wrap them in a ForwardRef at some point. - arg_ref = f'{arg}:str-{id(arg)}' - else: - arg_ref = f'{_repr.display_as_type(arg)}:{id(arg)}' - arg_refs.append(arg_ref) - if arg_refs: - type_ref = f'{type_ref}[{",".join(arg_refs)}]' - return type_ref - - -def get_ref(s: core_schema.CoreSchema) -> None | str: - """Get the ref from the schema if it has one. - This exists just for type checking to work correctly. - """ - return s.get('ref', None) - - -def _clean_schema_for_pretty_print(obj: Any, strip_metadata: bool = True) -> Any: # pragma: no cover - """A utility function to remove irrelevant information from a core schema.""" - if isinstance(obj, Mapping): - new_dct = {} - for k, v in obj.items(): - if k == 'metadata' and strip_metadata: - new_metadata = {} - - for meta_k, meta_v in v.items(): - if meta_k in ('pydantic_js_functions', 'pydantic_js_annotation_functions'): - new_metadata['js_metadata'] = '' - else: - new_metadata[meta_k] = _clean_schema_for_pretty_print(meta_v, strip_metadata=strip_metadata) - - if list(new_metadata.keys()) == ['js_metadata']: - new_metadata = {''} - - new_dct[k] = new_metadata - # Remove some defaults: - elif k in ('custom_init', 'root_model') and not v: - continue - else: - new_dct[k] = _clean_schema_for_pretty_print(v, strip_metadata=strip_metadata) - - return new_dct - elif isinstance(obj, Sequence) and not isinstance(obj, str): - return [_clean_schema_for_pretty_print(v, strip_metadata=strip_metadata) for v in obj] - else: - return obj - - -def pretty_print_core_schema( - val: Any, - *, - console: Console | None = None, - max_depth: int | None = None, - strip_metadata: bool = True, -) -> None: # pragma: no cover - """Pretty-print a core schema using the `rich` library. - - Args: - val: The core schema to print, or a Pydantic model/dataclass/type adapter - (in which case the cached core schema is fetched and printed). - console: A rich console to use when printing. Defaults to the global rich console instance. - max_depth: The number of nesting levels which may be printed. - strip_metadata: Whether to strip metadata in the output. If `True` any known core metadata - attributes will be stripped (but custom attributes are kept). Defaults to `True`. - """ - # lazy import: - from rich.pretty import pprint - - # circ. imports: - from pydantic import BaseModel, TypeAdapter - from pydantic.dataclasses import is_pydantic_dataclass - - if (inspect.isclass(val) and issubclass(val, BaseModel)) or is_pydantic_dataclass(val): - val = val.__pydantic_core_schema__ - if isinstance(val, TypeAdapter): - val = val.core_schema - cleaned_schema = _clean_schema_for_pretty_print(val, strip_metadata=strip_metadata) - - pprint(cleaned_schema, console=console, max_depth=max_depth) - - -pps = pretty_print_core_schema diff --git a/functions/python/data-processor/pydantic/_internal/_dataclasses.py b/functions/python/data-processor/pydantic/_internal/_dataclasses.py deleted file mode 100644 index f00c8b777..000000000 --- a/functions/python/data-processor/pydantic/_internal/_dataclasses.py +++ /dev/null @@ -1,315 +0,0 @@ -"""Private logic for creating pydantic dataclasses.""" - -from __future__ import annotations as _annotations - -import copy -import dataclasses -import sys -import warnings -from collections.abc import Generator -from contextlib import contextmanager -from functools import partial -from typing import TYPE_CHECKING, Any, ClassVar, Protocol, cast - -from pydantic_core import ( - ArgsKwargs, - SchemaSerializer, - SchemaValidator, - core_schema, -) -from typing_extensions import TypeAlias, TypeIs - -from ..errors import PydanticUndefinedAnnotation -from ..fields import FieldInfo -from ..plugin._schema_validator import PluggableSchemaValidator, create_schema_validator -from ..warnings import PydanticDeprecatedSince20 -from . import _config, _decorators -from ._fields import collect_dataclass_fields -from ._generate_schema import GenerateSchema, InvalidSchemaError -from ._generics import get_standard_typevars_map -from ._mock_val_ser import set_dataclass_mocks -from ._namespace_utils import NsResolver -from ._signature import generate_pydantic_signature -from ._utils import LazyClassAttribute - -if TYPE_CHECKING: - from _typeshed import DataclassInstance as StandardDataclass - - from ..config import ConfigDict - - class PydanticDataclass(StandardDataclass, Protocol): - """A protocol containing attributes only available once a class has been decorated as a Pydantic dataclass. - - Attributes: - __pydantic_config__: Pydantic-specific configuration settings for the dataclass. - __pydantic_complete__: Whether dataclass building is completed, or if there are still undefined fields. - __pydantic_core_schema__: The pydantic-core schema used to build the SchemaValidator and SchemaSerializer. - __pydantic_decorators__: Metadata containing the decorators defined on the dataclass. - __pydantic_fields__: Metadata about the fields defined on the dataclass. - __pydantic_serializer__: The pydantic-core SchemaSerializer used to dump instances of the dataclass. - __pydantic_validator__: The pydantic-core SchemaValidator used to validate instances of the dataclass. - """ - - __pydantic_config__: ClassVar[ConfigDict] - __pydantic_complete__: ClassVar[bool] - __pydantic_core_schema__: ClassVar[core_schema.CoreSchema] - __pydantic_decorators__: ClassVar[_decorators.DecoratorInfos] - __pydantic_fields__: ClassVar[dict[str, FieldInfo]] - __pydantic_serializer__: ClassVar[SchemaSerializer] - __pydantic_validator__: ClassVar[SchemaValidator | PluggableSchemaValidator] - - @classmethod - def __pydantic_fields_complete__(cls) -> bool: ... - - -def set_dataclass_fields( - cls: type[StandardDataclass], - config_wrapper: _config.ConfigWrapper, - ns_resolver: NsResolver | None = None, -) -> None: - """Collect and set `cls.__pydantic_fields__`. - - Args: - cls: The class. - config_wrapper: The config wrapper instance. - ns_resolver: Namespace resolver to use when getting dataclass annotations. - """ - typevars_map = get_standard_typevars_map(cls) - fields = collect_dataclass_fields( - cls, ns_resolver=ns_resolver, typevars_map=typevars_map, config_wrapper=config_wrapper - ) - - cls.__pydantic_fields__ = fields # type: ignore - - -def complete_dataclass( - cls: type[Any], - config_wrapper: _config.ConfigWrapper, - *, - raise_errors: bool = True, - ns_resolver: NsResolver | None = None, - _force_build: bool = False, -) -> bool: - """Finish building a pydantic dataclass. - - This logic is called on a class which has already been wrapped in `dataclasses.dataclass()`. - - This is somewhat analogous to `pydantic._internal._model_construction.complete_model_class`. - - Args: - cls: The class. - config_wrapper: The config wrapper instance. - raise_errors: Whether to raise errors, defaults to `True`. - ns_resolver: The namespace resolver instance to use when collecting dataclass fields - and during schema building. - _force_build: Whether to force building the dataclass, no matter if - [`defer_build`][pydantic.config.ConfigDict.defer_build] is set. - - Returns: - `True` if building a pydantic dataclass is successfully completed, `False` otherwise. - - Raises: - PydanticUndefinedAnnotation: If `raise_error` is `True` and there is an undefined annotations. - """ - original_init = cls.__init__ - - # dataclass.__init__ must be defined here so its `__qualname__` can be changed since functions can't be copied, - # and so that the mock validator is used if building was deferred: - def __init__(__dataclass_self__: PydanticDataclass, *args: Any, **kwargs: Any) -> None: - __tracebackhide__ = True - s = __dataclass_self__ - s.__pydantic_validator__.validate_python(ArgsKwargs(args, kwargs), self_instance=s) - - __init__.__qualname__ = f'{cls.__qualname__}.__init__' - - cls.__init__ = __init__ # type: ignore - cls.__pydantic_config__ = config_wrapper.config_dict # type: ignore - - set_dataclass_fields(cls, config_wrapper=config_wrapper, ns_resolver=ns_resolver) - - if not _force_build and config_wrapper.defer_build: - set_dataclass_mocks(cls) - return False - - if hasattr(cls, '__post_init_post_parse__'): - warnings.warn( - 'Support for `__post_init_post_parse__` has been dropped, the method will not be called', - PydanticDeprecatedSince20, - ) - - typevars_map = get_standard_typevars_map(cls) - gen_schema = GenerateSchema( - config_wrapper, - ns_resolver=ns_resolver, - typevars_map=typevars_map, - ) - - # set __signature__ attr only for the class, but not for its instances - # (because instances can define `__call__`, and `inspect.signature` shouldn't - # use the `__signature__` attribute and instead generate from `__call__`). - cls.__signature__ = LazyClassAttribute( - '__signature__', - partial( - generate_pydantic_signature, - # It's important that we reference the `original_init` here, - # as it is the one synthesized by the stdlib `dataclass` module: - init=original_init, - fields=cls.__pydantic_fields__, # type: ignore - validate_by_name=config_wrapper.validate_by_name, - extra=config_wrapper.extra, - is_dataclass=True, - ), - ) - - try: - schema = gen_schema.generate_schema(cls) - except PydanticUndefinedAnnotation as e: - if raise_errors: - raise - set_dataclass_mocks(cls, f'`{e.name}`') - return False - - core_config = config_wrapper.core_config(title=cls.__name__) - - try: - schema = gen_schema.clean_schema(schema) - except InvalidSchemaError: - set_dataclass_mocks(cls) - return False - - # We are about to set all the remaining required properties expected for this cast; - # __pydantic_decorators__ and __pydantic_fields__ should already be set - cls = cast('type[PydanticDataclass]', cls) - - cls.__pydantic_core_schema__ = schema - cls.__pydantic_validator__ = create_schema_validator( - schema, cls, cls.__module__, cls.__qualname__, 'dataclass', core_config, config_wrapper.plugin_settings - ) - cls.__pydantic_serializer__ = SchemaSerializer(schema, core_config) - cls.__pydantic_complete__ = True - return True - - -def is_stdlib_dataclass(cls: type[Any], /) -> TypeIs[type[StandardDataclass]]: - """Returns `True` if the class is a stdlib dataclass and *not* a Pydantic dataclass. - - Unlike the stdlib `dataclasses.is_dataclass()` function, this does *not* include subclasses - of a dataclass that are themselves not dataclasses. - - Args: - cls: The class. - - Returns: - `True` if the class is a stdlib dataclass, `False` otherwise. - """ - return '__dataclass_fields__' in cls.__dict__ and not hasattr(cls, '__pydantic_validator__') - - -def as_dataclass_field(pydantic_field: FieldInfo) -> dataclasses.Field[Any]: - field_args: dict[str, Any] = {'default': pydantic_field} - - # Needed because if `doc` is set, the dataclass slots will be a dict (field name -> doc) instead of a tuple: - if sys.version_info >= (3, 14) and pydantic_field.description is not None: - field_args['doc'] = pydantic_field.description - - # Needed as the stdlib dataclass module processes kw_only in a specific way during class construction: - if sys.version_info >= (3, 10) and pydantic_field.kw_only is not None: - field_args['kw_only'] = pydantic_field.kw_only - - # Needed as the stdlib dataclass modules generates `__repr__()` during class construction: - if pydantic_field.repr is not True: - field_args['repr'] = pydantic_field.repr - - return dataclasses.field(**field_args) - - -DcFields: TypeAlias = dict[str, dataclasses.Field[Any]] - - -@contextmanager -def patch_base_fields(cls: type[Any]) -> Generator[None]: - """Temporarily patch the stdlib dataclasses bases of `cls` if the Pydantic `Field()` function is used. - - When creating a Pydantic dataclass, it is possible to inherit from stdlib dataclasses, where - the Pydantic `Field()` function is used. To create this Pydantic dataclass, we first apply - the stdlib `@dataclass` decorator on it. During the construction of the stdlib dataclass, - the `kw_only` and `repr` field arguments need to be understood by the stdlib *during* the - dataclass construction. To do so, we temporarily patch the fields dictionary of the affected - bases. - - For instance, with the following example: - - ```python {test="skip" lint="skip"} - import dataclasses as stdlib_dc - - import pydantic - import pydantic.dataclasses as pydantic_dc - - @stdlib_dc.dataclass - class A: - a: int = pydantic.Field(repr=False) - - # Notice that the `repr` attribute of the dataclass field is `True`: - A.__dataclass_fields__['a'] - #> dataclass.Field(default=FieldInfo(repr=False), repr=True, ...) - - @pydantic_dc.dataclass - class B(A): - b: int = pydantic.Field(repr=False) - ``` - - When passing `B` to the stdlib `@dataclass` decorator, it will look for fields in the parent classes - and reuse them directly. When this context manager is active, `A` will be temporarily patched to be - equivalent to: - - ```python {test="skip" lint="skip"} - @stdlib_dc.dataclass - class A: - a: int = stdlib_dc.field(default=Field(repr=False), repr=False) - ``` - - !!! note - This is only applied to the bases of `cls`, and not `cls` itself. The reason is that the Pydantic - dataclass decorator "owns" `cls` (in the previous example, `B`). As such, we instead modify the fields - directly (in the previous example, we simply do `setattr(B, 'b', as_dataclass_field(pydantic_field))`). - - !!! note - This approach is far from ideal, and can probably be the source of unwanted side effects/race conditions. - The previous implemented approach was mutating the `__annotations__` dict of `cls`, which is no longer a - safe operation in Python 3.14+, and resulted in unexpected behavior with field ordering anyway. - """ - # A list of two-tuples, the first element being a reference to the - # dataclass fields dictionary, the second element being a mapping between - # the field names that were modified, and their original `Field`: - original_fields_list: list[tuple[DcFields, DcFields]] = [] - - for base in cls.__mro__[1:]: - dc_fields: dict[str, dataclasses.Field[Any]] = base.__dict__.get('__dataclass_fields__', {}) - dc_fields_with_pydantic_field_defaults = { - field_name: field - for field_name, field in dc_fields.items() - if isinstance(field.default, FieldInfo) - # Only do the patching if one of the affected attributes is set: - and (field.default.description is not None or field.default.kw_only or field.default.repr is not True) - } - if dc_fields_with_pydantic_field_defaults: - original_fields_list.append((dc_fields, dc_fields_with_pydantic_field_defaults)) - for field_name, field in dc_fields_with_pydantic_field_defaults.items(): - default = cast(FieldInfo, field.default) - # `dataclasses.Field` isn't documented as working with `copy.copy()`. - # It is a class with `__slots__`, so should work (and we hope for the best): - new_dc_field = copy.copy(field) - # For base fields, no need to set `doc` from `FieldInfo.description`, this is only relevant - # for the class under construction and handled in `as_dataclass_field()`. - if sys.version_info >= (3, 10) and default.kw_only: - new_dc_field.kw_only = True - if default.repr is not True: - new_dc_field.repr = default.repr - dc_fields[field_name] = new_dc_field - - try: - yield - finally: - for fields, original_fields in original_fields_list: - for field_name, original_field in original_fields.items(): - fields[field_name] = original_field diff --git a/functions/python/data-processor/pydantic/_internal/_decorators.py b/functions/python/data-processor/pydantic/_internal/_decorators.py deleted file mode 100644 index e24ad2cf9..000000000 --- a/functions/python/data-processor/pydantic/_internal/_decorators.py +++ /dev/null @@ -1,873 +0,0 @@ -"""Logic related to validators applied to models etc. via the `@field_validator` and `@model_validator` decorators.""" - -from __future__ import annotations as _annotations - -import types -from collections import deque -from collections.abc import Iterable -from copy import copy -from dataclasses import dataclass, field -from functools import cached_property, partial, partialmethod -from inspect import Parameter, Signature, isdatadescriptor, ismethoddescriptor -from itertools import islice -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Generic, Literal, TypeVar, Union - -from pydantic_core import PydanticUndefined, PydanticUndefinedType, core_schema -from typing_extensions import Self, TypeAlias, is_typeddict - -from ..errors import PydanticUserError -from ._core_utils import get_type_ref -from ._internal_dataclass import slots_true -from ._namespace_utils import GlobalsNamespace, MappingNamespace -from ._typing_extra import get_function_type_hints, signature_no_eval -from ._utils import can_be_positional - -if TYPE_CHECKING: - from ..fields import ComputedFieldInfo - from ..functional_validators import FieldValidatorModes - from ._config import ConfigWrapper - - -@dataclass(**slots_true) -class ValidatorDecoratorInfo: - """A container for data from `@validator` so that we can access it - while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@validator'. - fields: A tuple of field names the validator should be called on. - mode: The proposed validator mode. - each_item: For complex objects (sets, lists etc.) whether to validate individual - elements rather than the whole object. - always: Whether this method and other validators should be called even if the value is missing. - check_fields: Whether to check that the fields actually exist on the model. - """ - - decorator_repr: ClassVar[str] = '@validator' - - fields: tuple[str, ...] - mode: Literal['before', 'after'] - each_item: bool - always: bool - check_fields: bool | None - - -@dataclass(**slots_true) -class FieldValidatorDecoratorInfo: - """A container for data from `@field_validator` so that we can access it - while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@field_validator'. - fields: A tuple of field names the validator should be called on. - mode: The proposed validator mode. - check_fields: Whether to check that the fields actually exist on the model. - json_schema_input_type: The input type of the function. This is only used to generate - the appropriate JSON Schema (in validation mode) and can only specified - when `mode` is either `'before'`, `'plain'` or `'wrap'`. - """ - - decorator_repr: ClassVar[str] = '@field_validator' - - fields: tuple[str, ...] - mode: FieldValidatorModes - check_fields: bool | None - json_schema_input_type: Any - - -@dataclass(**slots_true) -class RootValidatorDecoratorInfo: - """A container for data from `@root_validator` so that we can access it - while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@root_validator'. - mode: The proposed validator mode. - """ - - decorator_repr: ClassVar[str] = '@root_validator' - mode: Literal['before', 'after'] - - -@dataclass(**slots_true) -class FieldSerializerDecoratorInfo: - """A container for data from `@field_serializer` so that we can access it - while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@field_serializer'. - fields: A tuple of field names the serializer should be called on. - mode: The proposed serializer mode. - return_type: The type of the serializer's return value. - when_used: The serialization condition. Accepts a string with values `'always'`, `'unless-none'`, `'json'`, - and `'json-unless-none'`. - check_fields: Whether to check that the fields actually exist on the model. - """ - - decorator_repr: ClassVar[str] = '@field_serializer' - fields: tuple[str, ...] - mode: Literal['plain', 'wrap'] - return_type: Any - when_used: core_schema.WhenUsed - check_fields: bool | None - - -@dataclass(**slots_true) -class ModelSerializerDecoratorInfo: - """A container for data from `@model_serializer` so that we can access it - while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@model_serializer'. - mode: The proposed serializer mode. - return_type: The type of the serializer's return value. - when_used: The serialization condition. Accepts a string with values `'always'`, `'unless-none'`, `'json'`, - and `'json-unless-none'`. - """ - - decorator_repr: ClassVar[str] = '@model_serializer' - mode: Literal['plain', 'wrap'] - return_type: Any - when_used: core_schema.WhenUsed - - -@dataclass(**slots_true) -class ModelValidatorDecoratorInfo: - """A container for data from `@model_validator` so that we can access it - while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@model_validator'. - mode: The proposed serializer mode. - """ - - decorator_repr: ClassVar[str] = '@model_validator' - mode: Literal['wrap', 'before', 'after'] - - -DecoratorInfo: TypeAlias = """Union[ - ValidatorDecoratorInfo, - FieldValidatorDecoratorInfo, - RootValidatorDecoratorInfo, - FieldSerializerDecoratorInfo, - ModelSerializerDecoratorInfo, - ModelValidatorDecoratorInfo, - ComputedFieldInfo, -]""" - -ReturnType = TypeVar('ReturnType') -DecoratedType: TypeAlias = ( - 'Union[classmethod[Any, Any, ReturnType], staticmethod[Any, ReturnType], Callable[..., ReturnType], property]' -) - - -@dataclass # can't use slots here since we set attributes on `__post_init__` -class PydanticDescriptorProxy(Generic[ReturnType]): - """Wrap a classmethod, staticmethod, property or unbound function - and act as a descriptor that allows us to detect decorated items - from the class' attributes. - - This class' __get__ returns the wrapped item's __get__ result, - which makes it transparent for classmethods and staticmethods. - - Attributes: - wrapped: The decorator that has to be wrapped. - decorator_info: The decorator info. - shim: A wrapper function to wrap V1 style function. - """ - - wrapped: DecoratedType[ReturnType] - decorator_info: DecoratorInfo - shim: Callable[[Callable[..., Any]], Callable[..., Any]] | None = None - - def __post_init__(self): - for attr in 'setter', 'deleter': - if hasattr(self.wrapped, attr): - f = partial(self._call_wrapped_attr, name=attr) - setattr(self, attr, f) - - def _call_wrapped_attr(self, func: Callable[[Any], None], *, name: str) -> PydanticDescriptorProxy[ReturnType]: - self.wrapped = getattr(self.wrapped, name)(func) - if isinstance(self.wrapped, property): - # update ComputedFieldInfo.wrapped_property - from ..fields import ComputedFieldInfo - - if isinstance(self.decorator_info, ComputedFieldInfo): - self.decorator_info.wrapped_property = self.wrapped - return self - - def __get__(self, obj: object | None, obj_type: type[object] | None = None) -> PydanticDescriptorProxy[ReturnType]: - try: - return self.wrapped.__get__(obj, obj_type) # pyright: ignore[reportReturnType] - except AttributeError: - # not a descriptor, e.g. a partial object - return self.wrapped # type: ignore[return-value] - - def __set_name__(self, instance: Any, name: str) -> None: - if hasattr(self.wrapped, '__set_name__'): - self.wrapped.__set_name__(instance, name) # pyright: ignore[reportFunctionMemberAccess] - - def __getattr__(self, name: str, /) -> Any: - """Forward checks for __isabstractmethod__ and such.""" - return getattr(self.wrapped, name) - - -DecoratorInfoType = TypeVar('DecoratorInfoType', bound=DecoratorInfo) - - -@dataclass(**slots_true) -class Decorator(Generic[DecoratorInfoType]): - """A generic container class to join together the decorator metadata - (metadata from decorator itself, which we have when the - decorator is called but not when we are building the core-schema) - and the bound function (which we have after the class itself is created). - - Attributes: - cls_ref: The class ref. - cls_var_name: The decorated function name. - func: The decorated function. - shim: A wrapper function to wrap V1 style function. - info: The decorator info. - """ - - cls_ref: str - cls_var_name: str - func: Callable[..., Any] - shim: Callable[[Any], Any] | None - info: DecoratorInfoType - - @staticmethod - def build( - cls_: Any, - *, - cls_var_name: str, - shim: Callable[[Any], Any] | None, - info: DecoratorInfoType, - ) -> Decorator[DecoratorInfoType]: - """Build a new decorator. - - Args: - cls_: The class. - cls_var_name: The decorated function name. - shim: A wrapper function to wrap V1 style function. - info: The decorator info. - - Returns: - The new decorator instance. - """ - func = get_attribute_from_bases(cls_, cls_var_name) - if shim is not None: - func = shim(func) - func = unwrap_wrapped_function(func, unwrap_partial=False) - if not callable(func): - # TODO most likely this branch can be removed when we drop support for Python 3.12: - # This branch will get hit for classmethod properties - attribute = get_attribute_from_base_dicts(cls_, cls_var_name) # prevents the binding call to `__get__` - if isinstance(attribute, PydanticDescriptorProxy): - func = unwrap_wrapped_function(attribute.wrapped) - return Decorator( - cls_ref=get_type_ref(cls_), - cls_var_name=cls_var_name, - func=func, - shim=shim, - info=info, - ) - - def bind_to_cls(self, cls: Any) -> Decorator[DecoratorInfoType]: - """Bind the decorator to a class. - - Args: - cls: the class. - - Returns: - The new decorator instance. - """ - return self.build( - cls, - cls_var_name=self.cls_var_name, - shim=self.shim, - info=copy(self.info), - ) - - -def get_bases(tp: type[Any]) -> tuple[type[Any], ...]: - """Get the base classes of a class or typeddict. - - Args: - tp: The type or class to get the bases. - - Returns: - The base classes. - """ - if is_typeddict(tp): - return tp.__orig_bases__ # type: ignore - try: - return tp.__bases__ - except AttributeError: - return () - - -def mro(tp: type[Any]) -> tuple[type[Any], ...]: - """Calculate the Method Resolution Order of bases using the C3 algorithm. - - See https://www.python.org/download/releases/2.3/mro/ - """ - # try to use the existing mro, for performance mainly - # but also because it helps verify the implementation below - if not is_typeddict(tp): - try: - return tp.__mro__ - except AttributeError: - # GenericAlias and some other cases - pass - - bases = get_bases(tp) - return (tp,) + mro_for_bases(bases) - - -def mro_for_bases(bases: tuple[type[Any], ...]) -> tuple[type[Any], ...]: - def merge_seqs(seqs: list[deque[type[Any]]]) -> Iterable[type[Any]]: - while True: - non_empty = [seq for seq in seqs if seq] - if not non_empty: - # Nothing left to process, we're done. - return - candidate: type[Any] | None = None - for seq in non_empty: # Find merge candidates among seq heads. - candidate = seq[0] - not_head = [s for s in non_empty if candidate in islice(s, 1, None)] - if not_head: - # Reject the candidate. - candidate = None - else: - break - if not candidate: - raise TypeError('Inconsistent hierarchy, no C3 MRO is possible') - yield candidate - for seq in non_empty: - # Remove candidate. - if seq[0] == candidate: - seq.popleft() - - seqs = [deque(mro(base)) for base in bases] + [deque(bases)] - return tuple(merge_seqs(seqs)) - - -_sentinel = object() - - -def get_attribute_from_bases(tp: type[Any] | tuple[type[Any], ...], name: str) -> Any: - """Get the attribute from the next class in the MRO that has it, - aiming to simulate calling the method on the actual class. - - The reason for iterating over the mro instead of just getting - the attribute (which would do that for us) is to support TypedDict, - which lacks a real __mro__, but can have a virtual one constructed - from its bases (as done here). - - Args: - tp: The type or class to search for the attribute. If a tuple, this is treated as a set of base classes. - name: The name of the attribute to retrieve. - - Returns: - Any: The attribute value, if found. - - Raises: - AttributeError: If the attribute is not found in any class in the MRO. - """ - if isinstance(tp, tuple): - for base in mro_for_bases(tp): - attribute = base.__dict__.get(name, _sentinel) - if attribute is not _sentinel: - attribute_get = getattr(attribute, '__get__', None) - if attribute_get is not None: - return attribute_get(None, tp) - return attribute - raise AttributeError(f'{name} not found in {tp}') - else: - try: - return getattr(tp, name) - except AttributeError: - return get_attribute_from_bases(mro(tp), name) - - -def get_attribute_from_base_dicts(tp: type[Any], name: str) -> Any: - """Get an attribute out of the `__dict__` following the MRO. - This prevents the call to `__get__` on the descriptor, and allows - us to get the original function for classmethod properties. - - Args: - tp: The type or class to search for the attribute. - name: The name of the attribute to retrieve. - - Returns: - Any: The attribute value, if found. - - Raises: - KeyError: If the attribute is not found in any class's `__dict__` in the MRO. - """ - for base in reversed(mro(tp)): - if name in base.__dict__: - return base.__dict__[name] - return tp.__dict__[name] # raise the error - - -@dataclass(**slots_true) -class DecoratorInfos: - """Mapping of name in the class namespace to decorator info. - - note that the name in the class namespace is the function or attribute name - not the field name! - """ - - validators: dict[str, Decorator[ValidatorDecoratorInfo]] = field(default_factory=dict) - field_validators: dict[str, Decorator[FieldValidatorDecoratorInfo]] = field(default_factory=dict) - root_validators: dict[str, Decorator[RootValidatorDecoratorInfo]] = field(default_factory=dict) - field_serializers: dict[str, Decorator[FieldSerializerDecoratorInfo]] = field(default_factory=dict) - model_serializers: dict[str, Decorator[ModelSerializerDecoratorInfo]] = field(default_factory=dict) - model_validators: dict[str, Decorator[ModelValidatorDecoratorInfo]] = field(default_factory=dict) - computed_fields: dict[str, Decorator[ComputedFieldInfo]] = field(default_factory=dict) - - @classmethod - def build( - cls, - typ: type[Any], - # Default to `True` for backwards compatibility: - replace_wrapped_methods: bool = True, - ) -> Self: - """Build a `DecoratorInfos` instance for the given model, dataclass or `TypedDict` type. - - Decorators from parent classes are included, including "bare" classes (e.g. if `typ` - is a Pydantic model, non Pydantic parent model classes are also taken into account). - The collection of the decorators happens by respecting the MRO. - - If one of the bases has an `__pydantic_decorators__` attribute set, it is assumed to be - a `DecoratorInfos` instance and is used as-is. The `__pydantic_decorators__` attribute - is *not* being set on the provided `typ`. - - Args: - typ: The model, dataclass or `TypedDict` type to use when building the `DecoratorInfos` instance. - replace_wrapped_methods: Whether to replace the decorator's wrapped methods on `typ`. - This is useful e.g. for field validators which are initially class methods. This should - only be set to `True` if `typ` is a Pydantic model or dataclass (otherwise this results - in mutations of classes Pydantic doesn't "own"). - """ - # reminder: dicts are ordered and replacement does not alter the order - res = cls() - # Iterate over the bases, without the actual `typ`. - # `1:-1` because we don't need to include `object`/`TypedDict`: - for base in reversed(mro(typ)[1:-1]): - existing: DecoratorInfos | None = base.__dict__.get('__pydantic_decorators__') - if existing is None: - existing, _ = _decorator_infos_for_class(base, collect_to_replace=False) - res.validators.update({k: v.bind_to_cls(typ) for k, v in existing.validators.items()}) - res.field_validators.update({k: v.bind_to_cls(typ) for k, v in existing.field_validators.items()}) - res.root_validators.update({k: v.bind_to_cls(typ) for k, v in existing.root_validators.items()}) - res.field_serializers.update({k: v.bind_to_cls(typ) for k, v in existing.field_serializers.items()}) - res.model_serializers.update({k: v.bind_to_cls(typ) for k, v in existing.model_serializers.items()}) - res.model_validators.update({k: v.bind_to_cls(typ) for k, v in existing.model_validators.items()}) - res.computed_fields.update({k: v.bind_to_cls(typ) for k, v in existing.computed_fields.items()}) - - decorator_infos, to_replace = _decorator_infos_for_class(typ, collect_to_replace=True) - - res.validators.update(decorator_infos.validators) - res.field_validators.update(decorator_infos.field_validators) - res.root_validators.update(decorator_infos.root_validators) - res.field_serializers.update(decorator_infos.field_serializers) - res.model_serializers.update(decorator_infos.model_serializers) - res.model_validators.update(decorator_infos.model_validators) - res.computed_fields.update(decorator_infos.computed_fields) - - if replace_wrapped_methods and to_replace: - for name, value in to_replace: - setattr(typ, name, value) - - res._validate() - return res - - def _validate(self) -> None: - seen: set[str] = set() - for field_ser in self.field_serializers.values(): - for f_name in field_ser.info.fields: - if f_name in seen: - raise PydanticUserError( - f'Multiple field serializer functions were defined for field {f_name!r}, this is not allowed.', - code='multiple-field-serializers', - ) - seen.add(f_name) - - def update_from_config(self, config_wrapper: ConfigWrapper) -> None: - """Update the decorator infos from the configuration of the class they are attached to.""" - for name, computed_field_dec in self.computed_fields.items(): - computed_field_dec.info._update_from_config(config_wrapper, name) - - -def _decorator_infos_for_class( - typ: type[Any], - *, - collect_to_replace: bool, -) -> tuple[DecoratorInfos, list[tuple[str, Any]]]: - """Collect a `DecoratorInfos` for class, without looking into bases.""" - res = DecoratorInfos() - to_replace: list[tuple[str, Any]] = [] - - for var_name, var_value in vars(typ).items(): - if isinstance(var_value, PydanticDescriptorProxy): - info = var_value.decorator_info - if isinstance(info, ValidatorDecoratorInfo): - res.validators[var_name] = Decorator.build(typ, cls_var_name=var_name, shim=var_value.shim, info=info) - elif isinstance(info, FieldValidatorDecoratorInfo): - res.field_validators[var_name] = Decorator.build( - typ, cls_var_name=var_name, shim=var_value.shim, info=info - ) - elif isinstance(info, RootValidatorDecoratorInfo): - res.root_validators[var_name] = Decorator.build( - typ, cls_var_name=var_name, shim=var_value.shim, info=info - ) - elif isinstance(info, FieldSerializerDecoratorInfo): - res.field_serializers[var_name] = Decorator.build( - typ, cls_var_name=var_name, shim=var_value.shim, info=info - ) - elif isinstance(info, ModelValidatorDecoratorInfo): - res.model_validators[var_name] = Decorator.build( - typ, cls_var_name=var_name, shim=var_value.shim, info=info - ) - elif isinstance(info, ModelSerializerDecoratorInfo): - res.model_serializers[var_name] = Decorator.build( - typ, cls_var_name=var_name, shim=var_value.shim, info=info - ) - else: - from ..fields import ComputedFieldInfo - - isinstance(var_value, ComputedFieldInfo) - res.computed_fields[var_name] = Decorator.build(typ, cls_var_name=var_name, shim=None, info=info) - if collect_to_replace: - to_replace.append((var_name, var_value.wrapped)) - - return res, to_replace - - -def inspect_validator( - validator: Callable[..., Any], *, mode: FieldValidatorModes, type: Literal['field', 'model'] -) -> bool: - """Look at a field or model validator function and determine whether it takes an info argument. - - An error is raised if the function has an invalid signature. - - Args: - validator: The validator function to inspect. - mode: The proposed validator mode. - type: The type of validator, either 'field' or 'model'. - - Returns: - Whether the validator takes an info argument. - """ - try: - sig = signature_no_eval(validator) - except (ValueError, TypeError): - # `inspect.signature` might not be able to infer a signature, e.g. with C objects. - # In this case, we assume no info argument is present: - return False - n_positional = count_positional_required_params(sig) - if mode == 'wrap': - if n_positional == 3: - return True - elif n_positional == 2: - return False - else: - assert mode in {'before', 'after', 'plain'}, f"invalid mode: {mode!r}, expected 'before', 'after' or 'plain" - if n_positional == 2: - return True - elif n_positional == 1: - return False - - raise PydanticUserError( - f'Unrecognized {type} validator function signature for {validator} with `mode={mode}`: {sig}', - code='validator-signature', - ) - - -def inspect_field_serializer(serializer: Callable[..., Any], mode: Literal['plain', 'wrap']) -> tuple[bool, bool]: - """Look at a field serializer function and determine if it is a field serializer, - and whether it takes an info argument. - - An error is raised if the function has an invalid signature. - - Args: - serializer: The serializer function to inspect. - mode: The serializer mode, either 'plain' or 'wrap'. - - Returns: - Tuple of (is_field_serializer, info_arg). - """ - try: - sig = signature_no_eval(serializer) - except (ValueError, TypeError): - # `inspect.signature` might not be able to infer a signature, e.g. with C objects. - # In this case, we assume no info argument is present and this is not a method: - return (False, False) - - first = next(iter(sig.parameters.values()), None) - is_field_serializer = first is not None and first.name == 'self' - - n_positional = count_positional_required_params(sig) - if is_field_serializer: - # -1 to correct for self parameter - info_arg = _serializer_info_arg(mode, n_positional - 1) - else: - info_arg = _serializer_info_arg(mode, n_positional) - - if info_arg is None: - raise PydanticUserError( - f'Unrecognized field_serializer function signature for {serializer} with `mode={mode}`:{sig}', - code='field-serializer-signature', - ) - - return is_field_serializer, info_arg - - -def inspect_annotated_serializer(serializer: Callable[..., Any], mode: Literal['plain', 'wrap']) -> bool: - """Look at a serializer function used via `Annotated` and determine whether it takes an info argument. - - An error is raised if the function has an invalid signature. - - Args: - serializer: The serializer function to check. - mode: The serializer mode, either 'plain' or 'wrap'. - - Returns: - info_arg - """ - try: - sig = signature_no_eval(serializer) - except (ValueError, TypeError): - # `inspect.signature` might not be able to infer a signature, e.g. with C objects. - # In this case, we assume no info argument is present: - return False - info_arg = _serializer_info_arg(mode, count_positional_required_params(sig)) - if info_arg is None: - raise PydanticUserError( - f'Unrecognized field_serializer function signature for {serializer} with `mode={mode}`:{sig}', - code='field-serializer-signature', - ) - else: - return info_arg - - -def inspect_model_serializer(serializer: Callable[..., Any], mode: Literal['plain', 'wrap']) -> bool: - """Look at a model serializer function and determine whether it takes an info argument. - - An error is raised if the function has an invalid signature. - - Args: - serializer: The serializer function to check. - mode: The serializer mode, either 'plain' or 'wrap'. - - Returns: - `info_arg` - whether the function expects an info argument. - """ - if isinstance(serializer, (staticmethod, classmethod)) or not is_instance_method_from_sig(serializer): - raise PydanticUserError( - '`@model_serializer` must be applied to instance methods', code='model-serializer-instance-method' - ) - - sig = signature_no_eval(serializer) - info_arg = _serializer_info_arg(mode, count_positional_required_params(sig)) - if info_arg is None: - raise PydanticUserError( - f'Unrecognized model_serializer function signature for {serializer} with `mode={mode}`:{sig}', - code='model-serializer-signature', - ) - else: - return info_arg - - -def _serializer_info_arg(mode: Literal['plain', 'wrap'], n_positional: int) -> bool | None: - if mode == 'plain': - if n_positional == 1: - # (input_value: Any, /) -> Any - return False - elif n_positional == 2: - # (model: Any, input_value: Any, /) -> Any - return True - else: - assert mode == 'wrap', f"invalid mode: {mode!r}, expected 'plain' or 'wrap'" - if n_positional == 2: - # (input_value: Any, serializer: SerializerFunctionWrapHandler, /) -> Any - return False - elif n_positional == 3: - # (input_value: Any, serializer: SerializerFunctionWrapHandler, info: SerializationInfo, /) -> Any - return True - - return None - - -AnyDecoratorCallable: TypeAlias = ( - 'Union[classmethod[Any, Any, Any], staticmethod[Any, Any], partialmethod[Any], Callable[..., Any]]' -) - - -def is_instance_method_from_sig(function: AnyDecoratorCallable) -> bool: - """Whether the function is an instance method. - - It will consider a function as instance method if the first parameter of - function is `self`. - - Args: - function: The function to check. - - Returns: - `True` if the function is an instance method, `False` otherwise. - """ - sig = signature_no_eval(unwrap_wrapped_function(function)) - first = next(iter(sig.parameters.values()), None) - if first and first.name == 'self': - return True - return False - - -def ensure_classmethod_based_on_signature(function: AnyDecoratorCallable) -> Any: - """Apply the `@classmethod` decorator on the function. - - Args: - function: The function to apply the decorator on. - - Return: - The `@classmethod` decorator applied function. - """ - if not isinstance( - unwrap_wrapped_function(function, unwrap_class_static_method=False), classmethod - ) and _is_classmethod_from_sig(function): - return classmethod(function) # type: ignore[arg-type] - return function - - -def _is_classmethod_from_sig(function: AnyDecoratorCallable) -> bool: - sig = signature_no_eval(unwrap_wrapped_function(function)) - first = next(iter(sig.parameters.values()), None) - if first and first.name == 'cls': - return True - return False - - -def unwrap_wrapped_function( - func: Any, - *, - unwrap_partial: bool = True, - unwrap_class_static_method: bool = True, -) -> Any: - """Recursively unwraps a wrapped function until the underlying function is reached. - This handles property, functools.partial, functools.partialmethod, staticmethod, and classmethod. - - Args: - func: The function to unwrap. - unwrap_partial: If True (default), unwrap partial and partialmethod decorators. - unwrap_class_static_method: If True (default), also unwrap classmethod and staticmethod - decorators. If False, only unwrap partial and partialmethod decorators. - - Returns: - The underlying function of the wrapped function. - """ - # Define the types we want to check against as a single tuple. - unwrap_types = ( - (property, cached_property) - + ((partial, partialmethod) if unwrap_partial else ()) - + ((staticmethod, classmethod) if unwrap_class_static_method else ()) - ) - - while isinstance(func, unwrap_types): - if unwrap_class_static_method and isinstance(func, (classmethod, staticmethod)): - func = func.__func__ - elif isinstance(func, (partial, partialmethod)): - func = func.func - elif isinstance(func, property): - func = func.fget # arbitrary choice, convenient for computed fields - else: - # Make coverage happy as it can only get here in the last possible case - assert isinstance(func, cached_property) - func = func.func # type: ignore - - return func - - -_function_like = ( - partial, - partialmethod, - types.FunctionType, - types.BuiltinFunctionType, - types.MethodType, - types.WrapperDescriptorType, - types.MethodWrapperType, - types.MemberDescriptorType, -) - - -def get_callable_return_type( - callable_obj: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, -) -> Any | PydanticUndefinedType: - """Get the callable return type. - - Args: - callable_obj: The callable to analyze. - globalns: The globals namespace to use during type annotation evaluation. - localns: The locals namespace to use during type annotation evaluation. - - Returns: - The function return type. - """ - if isinstance(callable_obj, type): - # types are callables, and we assume the return type - # is the type itself (e.g. `int()` results in an instance of `int`). - return callable_obj - - if not isinstance(callable_obj, _function_like): - call_func = getattr(type(callable_obj), '__call__', None) # noqa: B004 - if call_func is not None: - callable_obj = call_func - - hints = get_function_type_hints( - unwrap_wrapped_function(callable_obj), - include_keys={'return'}, - globalns=globalns, - localns=localns, - ) - return hints.get('return', PydanticUndefined) - - -def count_positional_required_params(sig: Signature) -> int: - """Get the number of positional (required) arguments of a signature. - - This function should only be used to inspect signatures of validation and serialization functions. - The first argument (the value being serialized or validated) is counted as a required argument - even if a default value exists. - - Returns: - The number of positional arguments of a signature. - """ - parameters = list(sig.parameters.values()) - return sum( - 1 - for param in parameters - if can_be_positional(param) - # First argument is the value being validated/serialized, and can have a default value - # (e.g. `float`, which has signature `(x=0, /)`). We assume other parameters (the info arg - # for instance) should be required, and thus without any default value. - and (param.default is Parameter.empty or param is parameters[0]) - ) - - -def ensure_property(f: Any) -> Any: - """Ensure that a function is a `property` or `cached_property`, or is a valid descriptor. - - Args: - f: The function to check. - - Returns: - The function, or a `property` or `cached_property` instance wrapping the function. - """ - if ismethoddescriptor(f) or isdatadescriptor(f): - return f - else: - return property(f) diff --git a/functions/python/data-processor/pydantic/_internal/_decorators_v1.py b/functions/python/data-processor/pydantic/_internal/_decorators_v1.py deleted file mode 100644 index 34273779d..000000000 --- a/functions/python/data-processor/pydantic/_internal/_decorators_v1.py +++ /dev/null @@ -1,174 +0,0 @@ -"""Logic for V1 validators, e.g. `@validator` and `@root_validator`.""" - -from __future__ import annotations as _annotations - -from inspect import Parameter, signature -from typing import Any, Union, cast - -from pydantic_core import core_schema -from typing_extensions import Protocol - -from ..errors import PydanticUserError -from ._utils import can_be_positional - - -class V1OnlyValueValidator(Protocol): - """A simple validator, supported for V1 validators and V2 validators.""" - - def __call__(self, __value: Any) -> Any: ... - - -class V1ValidatorWithValues(Protocol): - """A validator with `values` argument, supported for V1 validators and V2 validators.""" - - def __call__(self, __value: Any, values: dict[str, Any]) -> Any: ... - - -class V1ValidatorWithValuesKwOnly(Protocol): - """A validator with keyword only `values` argument, supported for V1 validators and V2 validators.""" - - def __call__(self, __value: Any, *, values: dict[str, Any]) -> Any: ... - - -class V1ValidatorWithKwargs(Protocol): - """A validator with `kwargs` argument, supported for V1 validators and V2 validators.""" - - def __call__(self, __value: Any, **kwargs: Any) -> Any: ... - - -class V1ValidatorWithValuesAndKwargs(Protocol): - """A validator with `values` and `kwargs` arguments, supported for V1 validators and V2 validators.""" - - def __call__(self, __value: Any, values: dict[str, Any], **kwargs: Any) -> Any: ... - - -V1Validator = Union[ - V1ValidatorWithValues, V1ValidatorWithValuesKwOnly, V1ValidatorWithKwargs, V1ValidatorWithValuesAndKwargs -] - - -def can_be_keyword(param: Parameter) -> bool: - return param.kind in (Parameter.POSITIONAL_OR_KEYWORD, Parameter.KEYWORD_ONLY) - - -def make_generic_v1_field_validator(validator: V1Validator) -> core_schema.WithInfoValidatorFunction: - """Wrap a V1 style field validator for V2 compatibility. - - Args: - validator: The V1 style field validator. - - Returns: - A wrapped V2 style field validator. - - Raises: - PydanticUserError: If the signature is not supported or the parameters are - not available in Pydantic V2. - """ - sig = signature(validator) - - needs_values_kw = False - - for param_num, (param_name, parameter) in enumerate(sig.parameters.items()): - if can_be_keyword(parameter) and param_name in ('field', 'config'): - raise PydanticUserError( - 'The `field` and `config` parameters are not available in Pydantic V2, ' - 'please use the `info` parameter instead.', - code='validator-field-config-info', - ) - if parameter.kind is Parameter.VAR_KEYWORD: - needs_values_kw = True - elif can_be_keyword(parameter) and param_name == 'values': - needs_values_kw = True - elif can_be_positional(parameter) and param_num == 0: - # value - continue - elif parameter.default is Parameter.empty: # ignore params with defaults e.g. bound by functools.partial - raise PydanticUserError( - f'Unsupported signature for V1 style validator {validator}: {sig} is not supported.', - code='validator-v1-signature', - ) - - if needs_values_kw: - # (v, **kwargs), (v, values, **kwargs), (v, *, values, **kwargs) or (v, *, values) - val1 = cast(V1ValidatorWithValues, validator) - - def wrapper1(value: Any, info: core_schema.ValidationInfo) -> Any: - return val1(value, values=info.data) - - return wrapper1 - else: - val2 = cast(V1OnlyValueValidator, validator) - - def wrapper2(value: Any, _: core_schema.ValidationInfo) -> Any: - return val2(value) - - return wrapper2 - - -RootValidatorValues = dict[str, Any] -# technically tuple[model_dict, model_extra, fields_set] | tuple[dataclass_dict, init_vars] -RootValidatorFieldsTuple = tuple[Any, ...] - - -class V1RootValidatorFunction(Protocol): - """A simple root validator, supported for V1 validators and V2 validators.""" - - def __call__(self, __values: RootValidatorValues) -> RootValidatorValues: ... - - -class V2CoreBeforeRootValidator(Protocol): - """V2 validator with mode='before'.""" - - def __call__(self, __values: RootValidatorValues, __info: core_schema.ValidationInfo) -> RootValidatorValues: ... - - -class V2CoreAfterRootValidator(Protocol): - """V2 validator with mode='after'.""" - - def __call__( - self, __fields_tuple: RootValidatorFieldsTuple, __info: core_schema.ValidationInfo - ) -> RootValidatorFieldsTuple: ... - - -def make_v1_generic_root_validator( - validator: V1RootValidatorFunction, pre: bool -) -> V2CoreBeforeRootValidator | V2CoreAfterRootValidator: - """Wrap a V1 style root validator for V2 compatibility. - - Args: - validator: The V1 style field validator. - pre: Whether the validator is a pre validator. - - Returns: - A wrapped V2 style validator. - """ - if pre is True: - # mode='before' for pydantic-core - def _wrapper1(values: RootValidatorValues, _: core_schema.ValidationInfo) -> RootValidatorValues: - return validator(values) - - return _wrapper1 - - # mode='after' for pydantic-core - def _wrapper2(fields_tuple: RootValidatorFieldsTuple, _: core_schema.ValidationInfo) -> RootValidatorFieldsTuple: - if len(fields_tuple) == 2: - # dataclass, this is easy - values, init_vars = fields_tuple - values = validator(values) - return values, init_vars - else: - # ugly hack: to match v1 behaviour, we merge values and model_extra, then split them up based on fields - # afterwards - model_dict, model_extra, fields_set = fields_tuple - if model_extra: - fields = set(model_dict.keys()) - model_dict.update(model_extra) - model_dict_new = validator(model_dict) - for k in list(model_dict_new.keys()): - if k not in fields: - model_extra[k] = model_dict_new.pop(k) - else: - model_dict_new = validator(model_dict) - return model_dict_new, model_extra, fields_set - - return _wrapper2 diff --git a/functions/python/data-processor/pydantic/_internal/_discriminated_union.py b/functions/python/data-processor/pydantic/_internal/_discriminated_union.py deleted file mode 100644 index fc52f3088..000000000 --- a/functions/python/data-processor/pydantic/_internal/_discriminated_union.py +++ /dev/null @@ -1,494 +0,0 @@ -from __future__ import annotations as _annotations - -from collections.abc import Hashable, Sequence -from typing import TYPE_CHECKING, Any, cast - -from pydantic_core import CoreSchema, core_schema - -from ..errors import PydanticUserError -from . import _core_utils -from ._core_utils import ( - CoreSchemaField, -) - -if TYPE_CHECKING: - from ..types import Discriminator - from ._core_metadata import CoreMetadata - - -class MissingDefinitionForUnionRef(Exception): - """Raised when applying a discriminated union discriminator to a schema - requires a definition that is not yet defined - """ - - def __init__(self, ref: str) -> None: - self.ref = ref - super().__init__(f'Missing definition for ref {self.ref!r}') - - -def set_discriminator_in_metadata(schema: CoreSchema, discriminator: Any) -> None: - metadata = cast('CoreMetadata', schema.setdefault('metadata', {})) - metadata['pydantic_internal_union_discriminator'] = discriminator - - -def apply_discriminator( - schema: core_schema.CoreSchema, - discriminator: str | Discriminator, - definitions: dict[str, core_schema.CoreSchema] | None = None, -) -> core_schema.CoreSchema: - """Applies the discriminator and returns a new core schema. - - Args: - schema: The input schema. - discriminator: The name of the field which will serve as the discriminator. - definitions: A mapping of schema ref to schema. - - Returns: - The new core schema. - - Raises: - TypeError: - - If `discriminator` is used with invalid union variant. - - If `discriminator` is used with `Union` type with one variant. - - If `discriminator` value mapped to multiple choices. - MissingDefinitionForUnionRef: - If the definition for ref is missing. - PydanticUserError: - - If a model in union doesn't have a discriminator field. - - If discriminator field has a non-string alias. - - If discriminator fields have different aliases. - - If discriminator field not of type `Literal`. - """ - from ..types import Discriminator - - if isinstance(discriminator, Discriminator): - if isinstance(discriminator.discriminator, str): - discriminator = discriminator.discriminator - else: - return discriminator._convert_schema(schema) - - return _ApplyInferredDiscriminator(discriminator, definitions or {}).apply(schema) - - -class _ApplyInferredDiscriminator: - """This class is used to convert an input schema containing a union schema into one where that union is - replaced with a tagged-union, with all the associated debugging and performance benefits. - - This is done by: - * Validating that the input schema is compatible with the provided discriminator - * Introspecting the schema to determine which discriminator values should map to which union choices - * Handling various edge cases such as 'definitions', 'default', 'nullable' schemas, and more - - I have chosen to implement the conversion algorithm in this class, rather than a function, - to make it easier to maintain state while recursively walking the provided CoreSchema. - """ - - def __init__(self, discriminator: str, definitions: dict[str, core_schema.CoreSchema]): - # `discriminator` should be the name of the field which will serve as the discriminator. - # It must be the python name of the field, and *not* the field's alias. Note that as of now, - # all members of a discriminated union _must_ use a field with the same name as the discriminator. - # This may change if/when we expose a way to manually specify the TaggedUnionSchema's choices. - self.discriminator = discriminator - - # `definitions` should contain a mapping of schema ref to schema for all schemas which might - # be referenced by some choice - self.definitions = definitions - - # `_discriminator_alias` will hold the value, if present, of the alias for the discriminator - # - # Note: following the v1 implementation, we currently disallow the use of different aliases - # for different choices. This is not a limitation of pydantic_core, but if we try to handle - # this, the inference logic gets complicated very quickly, and could result in confusing - # debugging challenges for users making subtle mistakes. - # - # Rather than trying to do the most powerful inference possible, I think we should eventually - # expose a way to more-manually control the way the TaggedUnionSchema is constructed through - # the use of a new type which would be placed as an Annotation on the Union type. This would - # provide the full flexibility/power of pydantic_core's TaggedUnionSchema where necessary for - # more complex cases, without over-complicating the inference logic for the common cases. - self._discriminator_alias: str | None = None - - # `_should_be_nullable` indicates whether the converted union has `None` as an allowed value. - # If `None` is an acceptable value of the (possibly-wrapped) union, we ignore it while - # constructing the TaggedUnionSchema, but set the `_should_be_nullable` attribute to True. - # Once we have constructed the TaggedUnionSchema, if `_should_be_nullable` is True, we ensure - # that the final schema gets wrapped as a NullableSchema. This has the same semantics on the - # python side, but resolves the issue that `None` cannot correspond to any discriminator values. - self._should_be_nullable = False - - # `_is_nullable` is used to track if the final produced schema will definitely be nullable; - # we set it to True if the input schema is wrapped in a nullable schema that we know will be preserved - # as an indication that, even if None is discovered as one of the union choices, we will not need to wrap - # the final value in another nullable schema. - # - # This is more complicated than just checking for the final outermost schema having type 'nullable' thanks - # to the possible presence of other wrapper schemas such as DefinitionsSchema, WithDefaultSchema, etc. - self._is_nullable = False - - # `_choices_to_handle` serves as a stack of choices to add to the tagged union. Initially, choices - # from the union in the wrapped schema will be appended to this list, and the recursive choice-handling - # algorithm may add more choices to this stack as (nested) unions are encountered. - self._choices_to_handle: list[core_schema.CoreSchema] = [] - - # `_tagged_union_choices` is built during the call to `apply`, and will hold the choices to be included - # in the output TaggedUnionSchema that will replace the union from the input schema - self._tagged_union_choices: dict[Hashable, core_schema.CoreSchema] = {} - - # `_used` is changed to True after applying the discriminator to prevent accidental reuse - self._used = False - - def apply(self, schema: core_schema.CoreSchema) -> core_schema.CoreSchema: - """Return a new CoreSchema based on `schema` that uses a tagged-union with the discriminator provided - to this class. - - Args: - schema: The input schema. - - Returns: - The new core schema. - - Raises: - TypeError: - - If `discriminator` is used with invalid union variant. - - If `discriminator` is used with `Union` type with one variant. - - If `discriminator` value mapped to multiple choices. - ValueError: - If the definition for ref is missing. - PydanticUserError: - - If a model in union doesn't have a discriminator field. - - If discriminator field has a non-string alias. - - If discriminator fields have different aliases. - - If discriminator field not of type `Literal`. - """ - assert not self._used - schema = self._apply_to_root(schema) - if self._should_be_nullable and not self._is_nullable: - schema = core_schema.nullable_schema(schema) - self._used = True - return schema - - def _apply_to_root(self, schema: core_schema.CoreSchema) -> core_schema.CoreSchema: - """This method handles the outer-most stage of recursion over the input schema: - unwrapping nullable or definitions schemas, and calling the `_handle_choice` - method iteratively on the choices extracted (recursively) from the possibly-wrapped union. - """ - if schema['type'] == 'nullable': - self._is_nullable = True - wrapped = self._apply_to_root(schema['schema']) - nullable_wrapper = schema.copy() - nullable_wrapper['schema'] = wrapped - return nullable_wrapper - - if schema['type'] == 'definitions': - wrapped = self._apply_to_root(schema['schema']) - definitions_wrapper = schema.copy() - definitions_wrapper['schema'] = wrapped - return definitions_wrapper - - if schema['type'] == 'definition-ref': - schema_ref = schema['schema_ref'] - if schema_ref not in self.definitions: # pragma: no cover - raise MissingDefinitionForUnionRef(schema_ref) - - def_schema = self.definitions[schema_ref] - # If using a referenceable union as discriminated (e.g. `type Pet = Cat | Dog; field: Pet = Field(discriminator=...)`): - if def_schema['type'] == 'union': - schema = def_schema.copy() - schema.pop('ref') - - if schema['type'] != 'union': - # If the schema is not a union, it probably means it just had a single member and - # was flattened by pydantic_core. - # However, it still may make sense to apply the discriminator to this schema, - # as a way to get discriminated-union-style error messages, so we allow this here. - schema = core_schema.union_schema([schema]) - - # Reverse the choices list before extending the stack so that they get handled in the order they occur - choices_schemas = [v[0] if isinstance(v, tuple) else v for v in schema['choices'][::-1]] - self._choices_to_handle.extend(choices_schemas) - while self._choices_to_handle: - choice = self._choices_to_handle.pop() - self._handle_choice(choice) - - if self._discriminator_alias is not None and self._discriminator_alias != self.discriminator: - # * We need to annotate `discriminator` as a union here to handle both branches of this conditional - # * We need to annotate `discriminator` as list[list[str | int]] and not list[list[str]] due to the - # invariance of list, and because list[list[str | int]] is the type of the discriminator argument - # to tagged_union_schema below - # * See the docstring of pydantic_core.core_schema.tagged_union_schema for more details about how to - # interpret the value of the discriminator argument to tagged_union_schema. (The list[list[str]] here - # is the appropriate way to provide a list of fallback attributes to check for a discriminator value.) - discriminator: str | list[list[str | int]] = [[self.discriminator], [self._discriminator_alias]] - else: - discriminator = self.discriminator - return core_schema.tagged_union_schema( - choices=self._tagged_union_choices, - discriminator=discriminator, - custom_error_type=schema.get('custom_error_type'), - custom_error_message=schema.get('custom_error_message'), - custom_error_context=schema.get('custom_error_context'), - strict=False, - from_attributes=True, - ref=schema.get('ref'), - metadata=schema.get('metadata'), - serialization=schema.get('serialization'), - ) - - def _handle_choice(self, choice: core_schema.CoreSchema) -> None: - """This method handles the "middle" stage of recursion over the input schema. - Specifically, it is responsible for handling each choice of the outermost union - (and any "coalesced" choices obtained from inner unions). - - Here, "handling" entails: - * Coalescing nested unions and compatible tagged-unions - * Tracking the presence of 'none' and 'nullable' schemas occurring as choices - * Validating that each allowed discriminator value maps to a unique choice - * Updating the _tagged_union_choices mapping that will ultimately be used to build the TaggedUnionSchema. - """ - if choice['type'] == 'definition-ref': - if choice['schema_ref'] not in self.definitions: - raise MissingDefinitionForUnionRef(choice['schema_ref']) - - if choice['type'] == 'none': - self._should_be_nullable = True - elif choice['type'] == 'definitions': - self._handle_choice(choice['schema']) - elif choice['type'] == 'nullable': - self._should_be_nullable = True - self._handle_choice(choice['schema']) # unwrap the nullable schema - elif choice['type'] == 'union': - # Reverse the choices list before extending the stack so that they get handled in the order they occur - choices_schemas = [v[0] if isinstance(v, tuple) else v for v in choice['choices'][::-1]] - self._choices_to_handle.extend(choices_schemas) - elif choice['type'] not in { - 'model', - 'typed-dict', - 'tagged-union', - 'lax-or-strict', - 'dataclass', - 'dataclass-args', - 'definition-ref', - } and not _core_utils.is_function_with_inner_schema(choice): - # We should eventually handle 'definition-ref' as well - err_str = f'The core schema type {choice["type"]!r} is not a valid discriminated union variant.' - if choice['type'] == 'list': - err_str += ( - ' If you are making use of a list of union types, make sure the discriminator is applied to the ' - 'union type and not the list (e.g. `list[Annotated[ | , Field(discriminator=...)]]`).' - ) - raise TypeError(err_str) - else: - if choice['type'] == 'tagged-union' and self._is_discriminator_shared(choice): - # In this case, this inner tagged-union is compatible with the outer tagged-union, - # and its choices can be coalesced into the outer TaggedUnionSchema. - subchoices = [x for x in choice['choices'].values() if not isinstance(x, (str, int))] - # Reverse the choices list before extending the stack so that they get handled in the order they occur - self._choices_to_handle.extend(subchoices[::-1]) - return - - inferred_discriminator_values = self._infer_discriminator_values_for_choice(choice, source_name=None) - self._set_unique_choice_for_values(choice, inferred_discriminator_values) - - def _is_discriminator_shared(self, choice: core_schema.TaggedUnionSchema) -> bool: - """This method returns a boolean indicating whether the discriminator for the `choice` - is the same as that being used for the outermost tagged union. This is used to - determine whether this TaggedUnionSchema choice should be "coalesced" into the top level, - or whether it should be treated as a separate (nested) choice. - """ - inner_discriminator = choice['discriminator'] - return inner_discriminator == self.discriminator or ( - isinstance(inner_discriminator, list) - and (self.discriminator in inner_discriminator or [self.discriminator] in inner_discriminator) - ) - - def _infer_discriminator_values_for_choice( # noqa C901 - self, choice: core_schema.CoreSchema, source_name: str | None - ) -> list[str | int]: - """This function recurses over `choice`, extracting all discriminator values that should map to this choice. - - `model_name` is accepted for the purpose of producing useful error messages. - """ - if choice['type'] == 'definitions': - return self._infer_discriminator_values_for_choice(choice['schema'], source_name=source_name) - - elif _core_utils.is_function_with_inner_schema(choice): - return self._infer_discriminator_values_for_choice(choice['schema'], source_name=source_name) - - elif choice['type'] == 'lax-or-strict': - return sorted( - set( - self._infer_discriminator_values_for_choice(choice['lax_schema'], source_name=None) - + self._infer_discriminator_values_for_choice(choice['strict_schema'], source_name=None) - ) - ) - - elif choice['type'] == 'tagged-union': - values: list[str | int] = [] - # Ignore str/int "choices" since these are just references to other choices - subchoices = [x for x in choice['choices'].values() if not isinstance(x, (str, int))] - for subchoice in subchoices: - subchoice_values = self._infer_discriminator_values_for_choice(subchoice, source_name=None) - values.extend(subchoice_values) - return values - - elif choice['type'] == 'union': - values = [] - for subchoice in choice['choices']: - subchoice_schema = subchoice[0] if isinstance(subchoice, tuple) else subchoice - subchoice_values = self._infer_discriminator_values_for_choice(subchoice_schema, source_name=None) - values.extend(subchoice_values) - return values - - elif choice['type'] == 'nullable': - self._should_be_nullable = True - return self._infer_discriminator_values_for_choice(choice['schema'], source_name=None) - - elif choice['type'] == 'model': - return self._infer_discriminator_values_for_choice(choice['schema'], source_name=choice['cls'].__name__) - - elif choice['type'] == 'dataclass': - return self._infer_discriminator_values_for_choice(choice['schema'], source_name=choice['cls'].__name__) - - elif choice['type'] == 'model-fields': - return self._infer_discriminator_values_for_model_choice(choice, source_name=source_name) - - elif choice['type'] == 'dataclass-args': - return self._infer_discriminator_values_for_dataclass_choice(choice, source_name=source_name) - - elif choice['type'] == 'typed-dict': - return self._infer_discriminator_values_for_typed_dict_choice(choice, source_name=source_name) - - elif choice['type'] == 'definition-ref': - schema_ref = choice['schema_ref'] - if schema_ref not in self.definitions: - raise MissingDefinitionForUnionRef(schema_ref) - return self._infer_discriminator_values_for_choice(self.definitions[schema_ref], source_name=source_name) - else: - err_str = f'The core schema type {choice["type"]!r} is not a valid discriminated union variant.' - if choice['type'] == 'list': - err_str += ( - ' If you are making use of a list of union types, make sure the discriminator is applied to the ' - 'union type and not the list (e.g. `list[Annotated[ | , Field(discriminator=...)]]`).' - ) - raise TypeError(err_str) - - def _infer_discriminator_values_for_typed_dict_choice( - self, choice: core_schema.TypedDictSchema, source_name: str | None = None - ) -> list[str | int]: - """This method just extracts the _infer_discriminator_values_for_choice logic specific to TypedDictSchema - for the sake of readability. - """ - source = 'TypedDict' if source_name is None else f'TypedDict {source_name!r}' - field = choice['fields'].get(self.discriminator) - if field is None: - raise PydanticUserError( - f'{source} needs a discriminator field for key {self.discriminator!r}', code='discriminator-no-field' - ) - return self._infer_discriminator_values_for_field(field, source) - - def _infer_discriminator_values_for_model_choice( - self, choice: core_schema.ModelFieldsSchema, source_name: str | None = None - ) -> list[str | int]: - source = 'ModelFields' if source_name is None else f'Model {source_name!r}' - field = choice['fields'].get(self.discriminator) - if field is None: - raise PydanticUserError( - f'{source} needs a discriminator field for key {self.discriminator!r}', code='discriminator-no-field' - ) - return self._infer_discriminator_values_for_field(field, source) - - def _infer_discriminator_values_for_dataclass_choice( - self, choice: core_schema.DataclassArgsSchema, source_name: str | None = None - ) -> list[str | int]: - source = 'DataclassArgs' if source_name is None else f'Dataclass {source_name!r}' - for field in choice['fields']: - if field['name'] == self.discriminator: - break - else: - raise PydanticUserError( - f'{source} needs a discriminator field for key {self.discriminator!r}', code='discriminator-no-field' - ) - return self._infer_discriminator_values_for_field(field, source) - - def _infer_discriminator_values_for_field(self, field: CoreSchemaField, source: str) -> list[str | int]: - if field['type'] == 'computed-field': - # This should never occur as a discriminator, as it is only relevant to serialization - return [] - alias = field.get('validation_alias', self.discriminator) - if not isinstance(alias, str): - raise PydanticUserError( - f'Alias {alias!r} is not supported in a discriminated union', code='discriminator-alias-type' - ) - if self._discriminator_alias is None: - self._discriminator_alias = alias - elif self._discriminator_alias != alias: - raise PydanticUserError( - f'Aliases for discriminator {self.discriminator!r} must be the same ' - f'(got {alias}, {self._discriminator_alias})', - code='discriminator-alias', - ) - return self._infer_discriminator_values_for_inner_schema(field['schema'], source) - - def _infer_discriminator_values_for_inner_schema( - self, schema: core_schema.CoreSchema, source: str - ) -> list[str | int]: - """When inferring discriminator values for a field, we typically extract the expected values from a literal - schema. This function does that, but also handles nested unions and defaults. - """ - if schema['type'] == 'literal': - return schema['expected'] - - elif schema['type'] == 'union': - # Generally when multiple values are allowed they should be placed in a single `Literal`, but - # we add this case to handle the situation where a field is annotated as a `Union` of `Literal`s. - # For example, this lets us handle `Union[Literal['key'], Union[Literal['Key'], Literal['KEY']]]` - values: list[Any] = [] - for choice in schema['choices']: - choice_schema = choice[0] if isinstance(choice, tuple) else choice - choice_values = self._infer_discriminator_values_for_inner_schema(choice_schema, source) - values.extend(choice_values) - return values - - elif schema['type'] == 'default': - # This will happen if the field has a default value; we ignore it while extracting the discriminator values - return self._infer_discriminator_values_for_inner_schema(schema['schema'], source) - - elif schema['type'] == 'function-after': - # After validators don't affect the discriminator values - return self._infer_discriminator_values_for_inner_schema(schema['schema'], source) - - elif schema['type'] == 'model' and schema.get('root_model'): - # Support RootModel[Literal[...]] as discriminator field type - return self._infer_discriminator_values_for_inner_schema(schema['schema'], source) - - elif schema['type'] in {'function-before', 'function-wrap', 'function-plain'}: - validator_type = repr(schema['type'].split('-')[1]) - raise PydanticUserError( - f'Cannot use a mode={validator_type} validator in the' - f' discriminator field {self.discriminator!r} of {source}', - code='discriminator-validator', - ) - - else: - raise PydanticUserError( - f'{source} needs field {self.discriminator!r} to be of type `Literal`', - code='discriminator-needs-literal', - ) - - def _set_unique_choice_for_values(self, choice: core_schema.CoreSchema, values: Sequence[str | int]) -> None: - """This method updates `self.tagged_union_choices` so that all provided (discriminator) `values` map to the - provided `choice`, validating that none of these values already map to another (different) choice. - """ - for discriminator_value in values: - if discriminator_value in self._tagged_union_choices: - # It is okay if `value` is already in tagged_union_choices as long as it maps to the same value. - # Because tagged_union_choices may map values to other values, we need to walk the choices dict - # until we get to a "real" choice, and confirm that is equal to the one assigned. - existing_choice = self._tagged_union_choices[discriminator_value] - if existing_choice != choice: - raise TypeError( - f'Value {discriminator_value!r} for discriminator ' - f'{self.discriminator!r} mapped to multiple choices' - ) - else: - self._tagged_union_choices[discriminator_value] = choice diff --git a/functions/python/data-processor/pydantic/_internal/_docs_extraction.py b/functions/python/data-processor/pydantic/_internal/_docs_extraction.py deleted file mode 100644 index 6df77bf6c..000000000 --- a/functions/python/data-processor/pydantic/_internal/_docs_extraction.py +++ /dev/null @@ -1,113 +0,0 @@ -"""Utilities related to attribute docstring extraction.""" - -from __future__ import annotations - -import ast -import inspect -import sys -import textwrap -from typing import Any - - -class DocstringVisitor(ast.NodeVisitor): - def __init__(self) -> None: - super().__init__() - - self.target: str | None = None - self.attrs: dict[str, str] = {} - self.previous_node_type: type[ast.AST] | None = None - - def visit(self, node: ast.AST) -> Any: - node_result = super().visit(node) - self.previous_node_type = type(node) - return node_result - - def visit_AnnAssign(self, node: ast.AnnAssign) -> Any: - if isinstance(node.target, ast.Name): - self.target = node.target.id - - def visit_Expr(self, node: ast.Expr) -> Any: - if ( - isinstance(node.value, ast.Constant) - and isinstance(node.value.value, str) - and self.previous_node_type is ast.AnnAssign - ): - docstring = inspect.cleandoc(node.value.value) - if self.target: - self.attrs[self.target] = docstring - self.target = None - - -def _dedent_source_lines(source: list[str]) -> str: - # Required for nested class definitions, e.g. in a function block - dedent_source = textwrap.dedent(''.join(source)) - if dedent_source.startswith((' ', '\t')): - # We are in the case where there's a dedented (usually multiline) string - # at a lower indentation level than the class itself. We wrap our class - # in a function as a workaround. - dedent_source = f'def dedent_workaround():\n{dedent_source}' - return dedent_source - - -def _extract_source_from_frame(cls: type[Any]) -> list[str] | None: - frame = inspect.currentframe() - - while frame: - if inspect.getmodule(frame) is inspect.getmodule(cls): - lnum = frame.f_lineno - try: - lines, _ = inspect.findsource(frame) - except OSError: # pragma: no cover - # Source can't be retrieved (maybe because running in an interactive terminal), - # we don't want to error here. - pass - else: - block_lines = inspect.getblock(lines[lnum - 1 :]) - dedent_source = _dedent_source_lines(block_lines) - try: - block_tree = ast.parse(dedent_source) - except SyntaxError: - pass - else: - stmt = block_tree.body[0] - if isinstance(stmt, ast.FunctionDef) and stmt.name == 'dedent_workaround': - # `_dedent_source_lines` wrapped the class around the workaround function - stmt = stmt.body[0] - if isinstance(stmt, ast.ClassDef) and stmt.name == cls.__name__: - return block_lines - - frame = frame.f_back - - -def extract_docstrings_from_cls(cls: type[Any], use_inspect: bool = False) -> dict[str, str]: - """Map model attributes and their corresponding docstring. - - Args: - cls: The class of the Pydantic model to inspect. - use_inspect: Whether to skip usage of frames to find the object and use - the `inspect` module instead. - - Returns: - A mapping containing attribute names and their corresponding docstring. - """ - if use_inspect or sys.version_info >= (3, 13): - # On Python < 3.13, `inspect.getsourcelines()` might not work as expected - # if two classes have the same name in the same source file. - # On Python 3.13+, it will use the new `__firstlineno__` class attribute, - # making it way more robust. - try: - source, _ = inspect.getsourcelines(cls) - except OSError: # pragma: no cover - return {} - else: - # TODO remove this implementation when we drop support for Python 3.12: - source = _extract_source_from_frame(cls) - - if not source: - return {} - - dedent_source = _dedent_source_lines(source) - - visitor = DocstringVisitor() - visitor.visit(ast.parse(dedent_source)) - return visitor.attrs diff --git a/functions/python/data-processor/pydantic/_internal/_fields.py b/functions/python/data-processor/pydantic/_internal/_fields.py deleted file mode 100644 index 9a2a96b92..000000000 --- a/functions/python/data-processor/pydantic/_internal/_fields.py +++ /dev/null @@ -1,729 +0,0 @@ -"""Private logic related to fields (the `Field()` function and `FieldInfo` class), and arguments to `Annotated`.""" - -from __future__ import annotations as _annotations - -import dataclasses -import warnings -from collections.abc import Mapping -from functools import cache -from inspect import Parameter, ismethoddescriptor -from re import Pattern -from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast - -from pydantic_core import PydanticUndefined -from typing_extensions import TypeIs -from typing_inspection.introspection import AnnotationSource - -from pydantic import PydanticDeprecatedSince211 -from pydantic.errors import PydanticUserError - -from ..aliases import AliasGenerator -from . import _generics, _typing_extra -from ._config import ConfigWrapper -from ._docs_extraction import extract_docstrings_from_cls -from ._import_utils import import_cached_base_model, import_cached_field_info -from ._internal_dataclass import slots_true -from ._namespace_utils import NsResolver -from ._repr import Representation -from ._utils import can_be_positional, get_first_not_none - -if TYPE_CHECKING: - from annotated_types import BaseMetadata - - from ..fields import FieldInfo - from ..main import BaseModel - from ._dataclasses import PydanticDataclass, StandardDataclass - from ._decorators import DecoratorInfos - - -class PydanticMetadata(Representation): - """Base class for annotation markers like `Strict`.""" - - __slots__ = () - - -@dataclasses.dataclass(**slots_true) # TODO: make kw_only when we drop support for 3.9. -class PydanticExtraInfo: - # TODO: make use of PEP 747: - annotation: Any - complete: bool - - -def pydantic_general_metadata(**metadata: Any) -> BaseMetadata: - """Create a new `_PydanticGeneralMetadata` class with the given metadata. - - Args: - **metadata: The metadata to add. - - Returns: - The new `_PydanticGeneralMetadata` class. - """ - return _general_metadata_cls()(metadata) # type: ignore - - -@cache -def _general_metadata_cls() -> type[BaseMetadata]: - """Do it this way to avoid importing `annotated_types` at import time.""" - from annotated_types import BaseMetadata - - class _PydanticGeneralMetadata(PydanticMetadata, BaseMetadata): - """Pydantic general metadata like `max_digits`.""" - - def __init__(self, metadata: Any): - self.__dict__ = metadata - - return _PydanticGeneralMetadata # type: ignore - - -def _check_protected_namespaces( - protected_namespaces: tuple[str | Pattern[str], ...], - ann_name: str, - bases: tuple[type[Any], ...], - cls_name: str, -) -> None: - BaseModel = import_cached_base_model() - - for protected_namespace in protected_namespaces: - ns_violation = False - if isinstance(protected_namespace, Pattern): - ns_violation = protected_namespace.match(ann_name) is not None - elif isinstance(protected_namespace, str): - ns_violation = ann_name.startswith(protected_namespace) - - if ns_violation: - for b in bases: - if hasattr(b, ann_name): - if not (issubclass(b, BaseModel) and ann_name in getattr(b, '__pydantic_fields__', {})): - raise ValueError( - f'Field {ann_name!r} conflicts with member {getattr(b, ann_name)}' - f' of protected namespace {protected_namespace!r}.' - ) - else: - valid_namespaces: list[str] = [] - for pn in protected_namespaces: - if isinstance(pn, Pattern): - if not pn.match(ann_name): - valid_namespaces.append(f're.compile({pn.pattern!r})') - else: - if not ann_name.startswith(pn): - valid_namespaces.append(f"'{pn}'") - - valid_namespaces_str = f'({", ".join(valid_namespaces)}{",)" if len(valid_namespaces) == 1 else ")"}' - - warnings.warn( - f'Field {ann_name!r} in {cls_name!r} conflicts with protected namespace {protected_namespace!r}.\n\n' - f"You may be able to solve this by setting the 'protected_namespaces' configuration to {valid_namespaces_str}.", - UserWarning, - stacklevel=5, - ) - - -def _update_fields_from_docstrings(cls: type[Any], fields: dict[str, FieldInfo], use_inspect: bool = False) -> None: - fields_docs = extract_docstrings_from_cls(cls, use_inspect=use_inspect) - for ann_name, field_info in fields.items(): - if field_info.description is None and ann_name in fields_docs: - field_info.description = fields_docs[ann_name] - - -def _apply_field_title_generator_to_field_info( - title_generator: Callable[[str, FieldInfo], str], - field_name: str, - field_info: FieldInfo, -): - if field_info.title is None: - title = title_generator(field_name, field_info) - if not isinstance(title, str): - raise TypeError(f'field_title_generator {title_generator} must return str, not {title.__class__}') - - field_info.title = title - - -def _apply_alias_generator_to_field_info( - alias_generator: Callable[[str], str] | AliasGenerator, field_name: str, field_info: FieldInfo -): - """Apply an alias generator to aliases on a `FieldInfo` instance if appropriate. - - Args: - alias_generator: A callable that takes a string and returns a string, or an `AliasGenerator` instance. - field_name: The name of the field from which to generate the alias. - field_info: The `FieldInfo` instance to which the alias generator is (maybe) applied. - """ - # Apply an alias_generator if - # 1. An alias is not specified - # 2. An alias is specified, but the priority is <= 1 - if ( - field_info.alias_priority is None - or field_info.alias_priority <= 1 - or field_info.alias is None - or field_info.validation_alias is None - or field_info.serialization_alias is None - ): - alias, validation_alias, serialization_alias = None, None, None - - if isinstance(alias_generator, AliasGenerator): - alias, validation_alias, serialization_alias = alias_generator.generate_aliases(field_name) - elif callable(alias_generator): - alias = alias_generator(field_name) - if not isinstance(alias, str): - raise TypeError(f'alias_generator {alias_generator} must return str, not {alias.__class__}') - - # if priority is not set, we set to 1 - # which supports the case where the alias_generator from a child class is used - # to generate an alias for a field in a parent class - if field_info.alias_priority is None or field_info.alias_priority <= 1: - field_info.alias_priority = 1 - - # if the priority is 1, then we set the aliases to the generated alias - if field_info.alias_priority == 1: - field_info.serialization_alias = get_first_not_none(serialization_alias, alias) - field_info.validation_alias = get_first_not_none(validation_alias, alias) - field_info.alias = alias - - # if any of the aliases are not set, then we set them to the corresponding generated alias - if field_info.alias is None: - field_info.alias = alias - if field_info.serialization_alias is None: - field_info.serialization_alias = get_first_not_none(serialization_alias, alias) - if field_info.validation_alias is None: - field_info.validation_alias = get_first_not_none(validation_alias, alias) - - -def update_field_from_config(config_wrapper: ConfigWrapper, field_name: str, field_info: FieldInfo) -> None: - """Update the `FieldInfo` instance from the configuration set on the model it belongs to. - - This will apply the title and alias generators from the configuration. - - Args: - config_wrapper: The configuration from the model. - field_name: The field name the `FieldInfo` instance is attached to. - field_info: The `FieldInfo` instance to update. - """ - field_title_generator = field_info.field_title_generator or config_wrapper.field_title_generator - if field_title_generator is not None: - _apply_field_title_generator_to_field_info(field_title_generator, field_name, field_info) - if config_wrapper.alias_generator is not None: - _apply_alias_generator_to_field_info(config_wrapper.alias_generator, field_name, field_info) - - -_deprecated_method_names = {'dict', 'json', 'copy', '_iter', '_copy_and_set_values', '_calculate_keys'} - -_deprecated_classmethod_names = { - 'parse_obj', - 'parse_raw', - 'parse_file', - 'from_orm', - 'construct', - 'schema', - 'schema_json', - 'validate', - 'update_forward_refs', - '_get_value', -} - - -def collect_model_fields( # noqa: C901 - cls: type[BaseModel], - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver, - *, - typevars_map: Mapping[TypeVar, Any] | None = None, -) -> tuple[dict[str, FieldInfo], PydanticExtraInfo | None, set[str]]: - """Collect the fields and class variables names of a nascent Pydantic model. - - The fields collection process is *lenient*, meaning it won't error if string annotations - fail to evaluate. If this happens, the original annotation (and assigned value, if any) - is stored on the created `FieldInfo` instance. - - The `rebuild_model_fields()` should be called at a later point (e.g. when rebuilding the model), - and will make use of these stored attributes. - - Args: - cls: BaseModel or dataclass. - config_wrapper: The config wrapper instance. - ns_resolver: Namespace resolver to use when getting model annotations. - typevars_map: A dictionary mapping type variables to their concrete types. - - Returns: - A three-tuple containing the model fields, the `PydanticExtraInfo` instance if the `__pydantic_extra__` annotation is set, - and class variables names. - - Raises: - NameError: - - If there is a conflict between a field name and protected namespaces. - - If there is a field other than `root` in `RootModel`. - - If a field shadows an attribute in the parent model. - """ - FieldInfo_ = import_cached_field_info() - BaseModel_ = import_cached_base_model() - - bases = cls.__bases__ - parent_fields_lookup: dict[str, FieldInfo] = {} - for base in reversed(bases): - if model_fields := getattr(base, '__pydantic_fields__', None): - parent_fields_lookup.update(model_fields) - - type_hints = _typing_extra.get_model_type_hints(cls, ns_resolver=ns_resolver) - - # `cls_annotations` is only used to determine if an annotation comes from a parent class - cls_annotations = _typing_extra.safe_get_annotations(cls) - - fields: dict[str, FieldInfo] = {} - - class_vars: set[str] = set() - for ann_name, (ann_type, evaluated) in type_hints.items(): - if ann_name == 'model_config': - # We never want to treat `model_config` as a field - # Note: we may need to change this logic if/when we introduce a `BareModel` class with no - # protected namespaces (where `model_config` might be allowed as a field name) - continue - - _check_protected_namespaces( - protected_namespaces=config_wrapper.protected_namespaces, - ann_name=ann_name, - bases=bases, - cls_name=cls.__name__, - ) - - if _typing_extra.is_classvar_annotation(ann_type): - class_vars.add(ann_name) - continue - - assigned_value = getattr(cls, ann_name, PydanticUndefined) - if assigned_value is not PydanticUndefined and ( - # One of the deprecated instance methods was used as a field name (e.g. `dict()`): - any(getattr(BaseModel_, depr_name, None) is assigned_value for depr_name in _deprecated_method_names) - # One of the deprecated class methods was used as a field name (e.g. `schema()`): - or ( - hasattr(assigned_value, '__func__') - and any( - getattr(getattr(BaseModel_, depr_name, None), '__func__', None) is assigned_value.__func__ # pyright: ignore[reportAttributeAccessIssue] - for depr_name in _deprecated_classmethod_names - ) - ) - ): - # Then `assigned_value` would be the method, even though no default was specified: - assigned_value = PydanticUndefined - - if not is_valid_field_name(ann_name): - continue - if cls.__pydantic_root_model__ and ann_name != 'root': - raise NameError( - f"Unexpected field with name {ann_name!r}; only 'root' is allowed as a field of a `RootModel`" - ) - - for base in bases: - if hasattr(base, ann_name): - if ann_name not in cls_annotations: - # Don't warn when a field exists in a parent class but has not been defined in the current class - continue - - # when building a generic model with `MyModel[int]`, the generic_origin check makes sure we don't get - # "... shadows an attribute" warnings - generic_origin = getattr(cls, '__pydantic_generic_metadata__', {}).get('origin') - if base is generic_origin: - # Don't warn when "shadowing" of attributes in parametrized generics - continue - - dataclass_fields = { - field.name for field in (dataclasses.fields(base) if dataclasses.is_dataclass(base) else ()) - } - if ann_name in dataclass_fields: - # Don't warn when inheriting stdlib dataclasses whose fields are "shadowed" by defaults being set - # on the class instance. - continue - - warnings.warn( - f'Field name "{ann_name}" in "{cls.__qualname__}" shadows an attribute in parent ' - f'"{base.__qualname__}"', - UserWarning, - stacklevel=4, - ) - - if assigned_value is PydanticUndefined: # no assignment, just a plain annotation - if ann_name in cls_annotations or ann_name not in parent_fields_lookup: - # field is either: - # - present in the current model's annotations (and *not* from parent classes) - # - not found on any base classes; this seems to be caused by fields not getting - # generated due to models not being fully defined while initializing recursive models. - # Nothing stops us from just creating a `FieldInfo` for this type hint, so we do this. - field_info = FieldInfo_.from_annotation(ann_type, _source=AnnotationSource.CLASS) - field_info._original_annotation = ann_type - if not evaluated: - field_info._complete = False - # Store the original annotation that should be used to rebuild - # the field info later: - else: - # The field was present on one of the (possibly multiple) base classes, we make a copy directly from it. - parent_field_info = parent_fields_lookup[ann_name]._copy() - - # The only case where substituting the type variables is relevant (i.e. when `typevars_map` is not empty) - # is when a generic class is parameterized (e.g. `MyGenericModel[int, str]`), which creates a new class object - # (unlike the stdlib genercis that create a generic alias). In this case, we are guaranteed to only have to copy - # from the origin/parent model (e.g. `MyGenericModel`). - if typevars_map: - field_info = _recreate_field_info( - parent_field_info, ns_resolver=ns_resolver, typevars_map=typevars_map, lenient=True - ) - else: - field_info = parent_field_info - - else: # An assigned value is present (either the default value, or a `Field()` function) - if isinstance(assigned_value, FieldInfo_) and ismethoddescriptor(assigned_value.default): - # `assigned_value` was fetched using `getattr`, which triggers a call to `__get__` - # for descriptors, so we do the same if the `= field(default=...)` form is used. - # Note that we only do this for method descriptors for now, we might want to - # extend this to any descriptor in the future (by simply checking for - # `hasattr(assigned_value.default, '__get__')`). - default = assigned_value.default.__get__(None, cls) - assigned_value.default = default - assigned_value._attributes_set['default'] = default - - field_info = FieldInfo_.from_annotated_attribute(ann_type, assigned_value, _source=AnnotationSource.CLASS) - - # Store the original annotation and assignment value that could be used to rebuild the field info later. - field_info._original_assignment = assigned_value - field_info._original_annotation = ann_type - if not evaluated: - field_info._complete = False - elif 'final' in field_info._qualifiers and not field_info.is_required(): - warnings.warn( - f'Annotation {ann_name!r} is marked as final and has a default value. Pydantic treats {ann_name!r} as a ' - 'class variable, but it will be considered as a normal field in V3 to be aligned with dataclasses. If you ' - f'still want {ann_name!r} to be considered as a class variable, annotate it as: `ClassVar[] = .`', - category=PydanticDeprecatedSince211, - # Incorrect when `create_model` is used, but the chance that final with a default is used is low in that case: - stacklevel=4, - ) - class_vars.add(ann_name) - continue - - # attributes which are fields are removed from the class namespace: - # 1. To match the behaviour of annotation-only fields - # 2. To avoid false positives in the NameError check above - try: - delattr(cls, ann_name) - except AttributeError: - pass # indicates the attribute was on a parent class - - # Use cls.__dict__['__pydantic_decorators__'] instead of cls.__pydantic_decorators__ - # to make sure the decorators have already been built for this exact class - decorators: DecoratorInfos = cls.__dict__['__pydantic_decorators__'] - if ann_name in decorators.computed_fields: - raise TypeError( - f'Field {ann_name!r} of class {cls.__name__!r} overrides symbol of same name in a parent class. ' - 'This override with a computed_field is incompatible.' - ) - fields[ann_name] = field_info - - if field_info._complete: - # If not complete, this will be called in `rebuild_model_fields()`: - update_field_from_config(config_wrapper, ann_name, field_info) - - if config_wrapper.use_attribute_docstrings: - _update_fields_from_docstrings(cls, fields) - - pydantic_extra_info: PydanticExtraInfo | None = None - if '__pydantic_extra__' in type_hints: - ann, complete = type_hints['__pydantic_extra__'] - pydantic_extra_info = PydanticExtraInfo( - annotation=ann, - complete=complete, - ) - - return fields, pydantic_extra_info, class_vars - - -def rebuild_model_fields( - cls: type[BaseModel], - *, - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver, - typevars_map: Mapping[TypeVar, Any], -) -> tuple[dict[str, FieldInfo], PydanticExtraInfo | None]: - """Rebuild the (already present) model fields by trying to reevaluate annotations. - - This function should be called whenever a model with incomplete fields is encountered. - - Returns: - A two-tuple, the first element being the rebuilt fields, the second element being - the rebuild `PydanticExtraInfo` instance, if available. - - Raises: - NameError: If one of the annotations failed to evaluate. - - Note: - This function *doesn't* mutate the model fields in place, as it can be called during - schema generation, where you don't want to mutate other model's fields. - """ - rebuilt_fields: dict[str, FieldInfo] = {} - with ns_resolver.push(cls): - for f_name, field_info in cls.__pydantic_fields__.items(): - if field_info._complete: - rebuilt_fields[f_name] = field_info - else: - new_field = _recreate_field_info( - field_info, ns_resolver=ns_resolver, typevars_map=typevars_map, lenient=False - ) - update_field_from_config(config_wrapper, f_name, new_field) - rebuilt_fields[f_name] = new_field - - if cls.__pydantic_extra_info__ is not None and not cls.__pydantic_extra_info__.complete: - rebuilt_extra_info = PydanticExtraInfo( - annotation=_typing_extra.eval_type( - cls.__pydantic_extra_info__.annotation, *ns_resolver.types_namespace - ), - complete=True, - ) - else: - rebuilt_extra_info = cls.__pydantic_extra_info__ - - return rebuilt_fields, rebuilt_extra_info - - -def _recreate_field_info( - field_info: FieldInfo, - ns_resolver: NsResolver, - typevars_map: Mapping[TypeVar, Any], - *, - lenient: bool, -) -> FieldInfo: - FieldInfo_ = import_cached_field_info() - - existing_desc = field_info.description - if lenient: - ann = _generics.replace_types(field_info._original_annotation, typevars_map) - ann, evaluated = _typing_extra.try_eval_type( - ann, - *ns_resolver.types_namespace, - ) - else: - # Not the best pattern, maybe we could ship our own `eval_type()`, - # that would replace the type variables on the fly during evaluation. - ann = _typing_extra.eval_type( - field_info._original_annotation, - *ns_resolver.types_namespace, - ) - ann = _generics.replace_types(ann, typevars_map) - ann = _typing_extra.eval_type( - ann, - *ns_resolver.types_namespace, - ) - evaluated = True - - if (assign := field_info._original_assignment) is PydanticUndefined: - new_field = FieldInfo_.from_annotation(ann, _source=AnnotationSource.CLASS) - else: - new_field = FieldInfo_.from_annotated_attribute(ann, assign, _source=AnnotationSource.CLASS) - new_field._original_assignment = assign - new_field._original_annotation = ann - # The description might come from the docstring if `use_attribute_docstrings` was `True`: - new_field.description = new_field.description if new_field.description is not None else existing_desc - if not evaluated: - new_field._complete = False - - return new_field - - -def collect_dataclass_fields( - cls: type[StandardDataclass], - *, - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver | None = None, - typevars_map: dict[Any, Any] | None = None, -) -> dict[str, FieldInfo]: - """Collect the fields of a dataclass. - - Args: - cls: dataclass. - config_wrapper: The config wrapper instance. - ns_resolver: Namespace resolver to use when getting dataclass annotations. - Defaults to an empty instance. - typevars_map: A dictionary mapping type variables to their concrete types. - - Returns: - The dataclass fields. - """ - FieldInfo_ = import_cached_field_info() - - fields: dict[str, FieldInfo] = {} - ns_resolver = ns_resolver or NsResolver() - dataclass_fields = cls.__dataclass_fields__ - - # The logic here is similar to `_typing_extra.get_cls_type_hints`, - # although we do it manually as stdlib dataclasses already have annotations - # collected in each class: - for base in reversed(cls.__mro__): - if not dataclasses.is_dataclass(base): - continue - - with ns_resolver.push(base): - for ann_name, dataclass_field in dataclass_fields.items(): - base_anns = _typing_extra.safe_get_annotations(base) - - if ann_name not in base_anns: - # `__dataclass_fields__`contains every field, even the ones from base classes. - # Only collect the ones defined on `base`. - continue - - globalns, localns = ns_resolver.types_namespace - ann_type, evaluated = _typing_extra.try_eval_type(dataclass_field.type, globalns, localns) - - if _typing_extra.is_classvar_annotation(ann_type): - continue - - if ( - not dataclass_field.init - and dataclass_field.default is dataclasses.MISSING - and dataclass_field.default_factory is dataclasses.MISSING - ): - # TODO: We should probably do something with this so that validate_assignment behaves properly - # Issue: https://github.com/pydantic/pydantic/issues/5470 - continue - - if isinstance(dataclass_field.default, FieldInfo_): - if dataclass_field.default.init_var: - if dataclass_field.default.init is False: - raise PydanticUserError( - f'Dataclass field {ann_name} has init=False and init_var=True, but these are mutually exclusive.', - code='clashing-init-and-init-var', - ) - - # TODO: same note as above re validate_assignment - continue - field_info = FieldInfo_.from_annotated_attribute( - ann_type, dataclass_field.default, _source=AnnotationSource.DATACLASS - ) - field_info._original_assignment = dataclass_field.default - else: - field_info = FieldInfo_.from_annotated_attribute( - ann_type, dataclass_field, _source=AnnotationSource.DATACLASS - ) - field_info._original_assignment = dataclass_field - - if not evaluated: - field_info._complete = False - field_info._original_annotation = ann_type - - fields[ann_name] = field_info - update_field_from_config(config_wrapper, ann_name, field_info) - - if field_info.default is not PydanticUndefined and isinstance( - getattr(cls, ann_name, field_info), FieldInfo_ - ): - # We need this to fix the default when the "default" from __dataclass_fields__ is a pydantic.FieldInfo - setattr(cls, ann_name, field_info.default) - - if typevars_map: - for field in fields.values(): - # We don't pass any ns, as `field.annotation` - # was already evaluated. TODO: is this method relevant? - # Can't we juste use `_generics.replace_types`? - field.apply_typevars_map(typevars_map) - - if config_wrapper.use_attribute_docstrings: - _update_fields_from_docstrings( - cls, - fields, - # We can't rely on the (more reliable) frame inspection method - # for stdlib dataclasses: - use_inspect=not hasattr(cls, '__is_pydantic_dataclass__'), - ) - - return fields - - -def rebuild_dataclass_fields( - cls: type[PydanticDataclass], - *, - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver, - typevars_map: Mapping[TypeVar, Any], -) -> dict[str, FieldInfo]: - """Rebuild the (already present) dataclass fields by trying to reevaluate annotations. - - This function should be called whenever a dataclass with incomplete fields is encountered. - - Raises: - NameError: If one of the annotations failed to evaluate. - - Note: - This function *doesn't* mutate the dataclass fields in place, as it can be called during - schema generation, where you don't want to mutate other dataclass's fields. - """ - FieldInfo_ = import_cached_field_info() - - rebuilt_fields: dict[str, FieldInfo] = {} - with ns_resolver.push(cls): - for f_name, field_info in cls.__pydantic_fields__.items(): - if field_info._complete: - rebuilt_fields[f_name] = field_info - else: - existing_desc = field_info.description - ann = _typing_extra.eval_type( - field_info._original_annotation, - *ns_resolver.types_namespace, - ) - ann = _generics.replace_types(ann, typevars_map) - new_field = FieldInfo_.from_annotated_attribute( - ann, - field_info._original_assignment, - _source=AnnotationSource.DATACLASS, - ) - - # The description might come from the docstring if `use_attribute_docstrings` was `True`: - new_field.description = new_field.description if new_field.description is not None else existing_desc - update_field_from_config(config_wrapper, f_name, new_field) - rebuilt_fields[f_name] = new_field - - return rebuilt_fields - - -def is_valid_field_name(name: str) -> bool: - return not name.startswith('_') - - -def is_valid_privateattr_name(name: str) -> bool: - return name.startswith('_') and not name.startswith('__') - - -def takes_validated_data_argument( - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any], -) -> TypeIs[Callable[[dict[str, Any]], Any]]: - """Whether the provided default factory callable has a validated data parameter.""" - try: - sig = _typing_extra.signature_no_eval(default_factory) - except (ValueError, TypeError): - # `inspect.signature` might not be able to infer a signature, e.g. with C objects. - # In this case, we assume no data argument is present: - return False - - parameters = list(sig.parameters.values()) - - return len(parameters) == 1 and can_be_positional(parameters[0]) and parameters[0].default is Parameter.empty - - -def resolve_default_value( - default: Any, - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None, - *, - validated_data: dict[str, Any] | None = None, - call_default_factory: bool = False, -) -> Any: - """Resolve the default value using either a static default or a default_factory.""" - from ._utils import smart_deepcopy - - if default_factory is None: - return smart_deepcopy(default) - if call_default_factory: - if takes_validated_data_argument(default_factory=default_factory): - fac = cast('Callable[[dict[str, Any]], Any]', default_factory) - if validated_data is None: - raise ValueError( - "The default factory requires the 'validated_data' argument, which was not provided when calling 'get_default()'." - ) - return fac(validated_data) - else: - fac = cast('Callable[[], Any]', default_factory) - return fac() - - return PydanticUndefined diff --git a/functions/python/data-processor/pydantic/_internal/_forward_ref.py b/functions/python/data-processor/pydantic/_internal/_forward_ref.py deleted file mode 100644 index 231f81d11..000000000 --- a/functions/python/data-processor/pydantic/_internal/_forward_ref.py +++ /dev/null @@ -1,23 +0,0 @@ -from __future__ import annotations as _annotations - -from dataclasses import dataclass -from typing import Union - - -@dataclass -class PydanticRecursiveRef: - type_ref: str - - __name__ = 'PydanticRecursiveRef' - __hash__ = object.__hash__ - - def __call__(self) -> None: - """Defining __call__ is necessary for the `typing` module to let you use an instance of - this class as the result of resolving a standard ForwardRef. - """ - - def __or__(self, other): - return Union[self, other] # type: ignore - - def __ror__(self, other): - return Union[other, self] # type: ignore diff --git a/functions/python/data-processor/pydantic/_internal/_generate_schema.py b/functions/python/data-processor/pydantic/_internal/_generate_schema.py deleted file mode 100644 index 98fae40d6..000000000 --- a/functions/python/data-processor/pydantic/_internal/_generate_schema.py +++ /dev/null @@ -1,2934 +0,0 @@ -"""Convert python types to pydantic-core schema.""" - -from __future__ import annotations as _annotations - -import collections.abc -import dataclasses -import datetime -import inspect -import os -import pathlib -import re -import sys -import typing -import warnings -from collections.abc import Generator, Iterable, Iterator, Mapping -from contextlib import contextmanager -from copy import copy -from decimal import Decimal -from enum import Enum -from fractions import Fraction -from functools import partial -from inspect import Parameter, _ParameterKind -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from itertools import chain -from operator import attrgetter -from types import FunctionType, GenericAlias, LambdaType, MethodType -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Final, - ForwardRef, - Literal, - TypeVar, - Union, - cast, - overload, -) -from uuid import UUID -from zoneinfo import ZoneInfo - -import typing_extensions -from pydantic_core import ( - MISSING, - CoreSchema, - MultiHostUrl, - PydanticCustomError, - PydanticSerializationUnexpectedValue, - PydanticUndefined, - Url, - core_schema, - to_jsonable_python, -) -from typing_extensions import TypeAlias, TypeAliasType, get_args, get_origin, is_typeddict -from typing_inspection import typing_objects -from typing_inspection.introspection import AnnotationSource, get_literal_values, is_union_origin - -from ..aliases import AliasChoices, AliasPath -from ..annotated_handlers import GetCoreSchemaHandler, GetJsonSchemaHandler -from ..config import ConfigDict, JsonDict, JsonEncoder, JsonSchemaExtraCallable -from ..errors import ( - PydanticForbiddenQualifier, - PydanticInvalidForJsonSchema, - PydanticSchemaGenerationError, - PydanticUndefinedAnnotation, - PydanticUserError, -) -from ..functional_validators import AfterValidator, BeforeValidator, FieldValidatorModes, PlainValidator, WrapValidator -from ..json_schema import JsonSchemaValue -from ..version import version_short -from ..warnings import ( - ArbitraryTypeWarning, - PydanticDeprecatedSince20, - TypedDictExtraConfigWarning, - UnsupportedFieldAttributeWarning, -) -from . import _decorators, _discriminated_union, _known_annotated_metadata, _repr, _typing_extra -from ._config import ConfigWrapper, ConfigWrapperStack -from ._core_metadata import CoreMetadata, update_core_metadata -from ._core_utils import ( - get_ref, - get_type_ref, - is_list_like_schema_with_items_schema, -) -from ._decorators import ( - Decorator, - DecoratorInfos, - FieldSerializerDecoratorInfo, - FieldValidatorDecoratorInfo, - ModelSerializerDecoratorInfo, - ModelValidatorDecoratorInfo, - RootValidatorDecoratorInfo, - ValidatorDecoratorInfo, - get_attribute_from_bases, - inspect_field_serializer, - inspect_model_serializer, - inspect_validator, -) -from ._docs_extraction import extract_docstrings_from_cls -from ._fields import ( - collect_dataclass_fields, - rebuild_dataclass_fields, - rebuild_model_fields, - takes_validated_data_argument, - update_field_from_config, -) -from ._forward_ref import PydanticRecursiveRef -from ._generics import get_standard_typevars_map, replace_types -from ._import_utils import import_cached_base_model, import_cached_field_info -from ._mock_val_ser import MockCoreSchema -from ._namespace_utils import NamespacesTuple, NsResolver -from ._schema_gather import MissingDefinitionError, gather_schemas_for_cleaning -from ._schema_generation_shared import CallbackGetCoreSchemaHandler -from ._utils import lenient_issubclass, smart_deepcopy - -if TYPE_CHECKING: - from ..fields import ComputedFieldInfo, FieldInfo - from ..main import BaseModel - from ..types import Discriminator - from ._dataclasses import StandardDataclass - from ._schema_generation_shared import GetJsonSchemaFunction - -_SUPPORTS_TYPEDDICT = sys.version_info >= (3, 12) - -FieldDecoratorInfo = Union[ValidatorDecoratorInfo, FieldValidatorDecoratorInfo, FieldSerializerDecoratorInfo] -FieldDecoratorInfoType = TypeVar('FieldDecoratorInfoType', bound=FieldDecoratorInfo) -AnyFieldDecorator = Union[ - Decorator[ValidatorDecoratorInfo], - Decorator[FieldValidatorDecoratorInfo], - Decorator[FieldSerializerDecoratorInfo], -] - -ModifyCoreSchemaWrapHandler: TypeAlias = GetCoreSchemaHandler -GetCoreSchemaFunction: TypeAlias = Callable[[Any, ModifyCoreSchemaWrapHandler], core_schema.CoreSchema] -ParametersCallback: TypeAlias = "Callable[[int, str, Any], Literal['skip'] | None]" - -TUPLE_TYPES: list[type] = [typing.Tuple, tuple] # noqa: UP006 -LIST_TYPES: list[type] = [typing.List, list, collections.abc.MutableSequence] # noqa: UP006 -SET_TYPES: list[type] = [typing.Set, set, collections.abc.MutableSet] # noqa: UP006 -FROZEN_SET_TYPES: list[type] = [typing.FrozenSet, frozenset, collections.abc.Set] # noqa: UP006 -DICT_TYPES: list[type] = [typing.Dict, dict] # noqa: UP006 -IP_TYPES: list[type] = [IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network] -SEQUENCE_TYPES: list[type] = [typing.Sequence, collections.abc.Sequence] -ITERABLE_TYPES: list[type] = [typing.Iterable, collections.abc.Iterable, typing.Generator, collections.abc.Generator] -TYPE_TYPES: list[type] = [typing.Type, type] # noqa: UP006 -PATTERN_TYPES: list[type] = [typing.Pattern, re.Pattern] -PATH_TYPES: list[type] = [ - os.PathLike, - pathlib.Path, - pathlib.PurePath, - pathlib.PosixPath, - pathlib.PurePosixPath, - pathlib.PureWindowsPath, -] -MAPPING_TYPES = [ - typing.Mapping, - typing.MutableMapping, - collections.abc.Mapping, - collections.abc.MutableMapping, - collections.OrderedDict, - typing_extensions.OrderedDict, - typing.DefaultDict, # noqa: UP006 - collections.defaultdict, -] -COUNTER_TYPES = [collections.Counter, typing.Counter] -DEQUE_TYPES: list[type] = [collections.deque, typing.Deque] # noqa: UP006 - -# Note: This does not play very well with type checkers. For example, -# `a: LambdaType = lambda x: x` will raise a type error by Pyright. -ValidateCallSupportedTypes = Union[ - LambdaType, - FunctionType, - MethodType, - partial, -] - -VALIDATE_CALL_SUPPORTED_TYPES = get_args(ValidateCallSupportedTypes) -UNSUPPORTED_STANDALONE_FIELDINFO_ATTRIBUTES: list[tuple[str, Any]] = [ - ('alias', None), - ('validation_alias', None), - ('serialization_alias', None), - # will be set if any alias is set, so disable it to avoid double warnings: - # 'alias_priority', - ('default', PydanticUndefined), - ('default_factory', None), - ('exclude', None), - ('deprecated', None), - ('repr', True), - ('validate_default', None), - ('frozen', None), - ('init', None), - ('init_var', None), - ('kw_only', None), -] -"""`FieldInfo` attributes (and their default value) that can't be used outside of a model (e.g. in a type adapter or a PEP 695 type alias).""" - -_mode_to_validator: dict[ - FieldValidatorModes, type[BeforeValidator | AfterValidator | PlainValidator | WrapValidator] -] = {'before': BeforeValidator, 'after': AfterValidator, 'plain': PlainValidator, 'wrap': WrapValidator} - - -def check_validator_fields_against_field_name( - info: FieldDecoratorInfo, - field: str, -) -> bool: - """Check if field name is in validator fields. - - Args: - info: The field info. - field: The field name to check. - - Returns: - `True` if field name is in validator fields, `False` otherwise. - """ - fields = info.fields - return '*' in fields or field in fields - - -def check_decorator_fields_exist(decorators: Iterable[AnyFieldDecorator], fields: Iterable[str]) -> None: - """Check if the defined fields in decorators exist in `fields` param. - - It ignores the check for a decorator if the decorator has `*` as field or `check_fields=False`. - - Args: - decorators: An iterable of decorators. - fields: An iterable of fields name. - - Raises: - PydanticUserError: If one of the field names does not exist in `fields` param. - """ - fields = set(fields) - for dec in decorators: - if '*' in dec.info.fields: - continue - if dec.info.check_fields is False: - continue - for field in dec.info.fields: - if field not in fields: - raise PydanticUserError( - f'Decorators defined with incorrect fields: {dec.cls_ref}.{dec.cls_var_name}' - " (use check_fields=False if you're inheriting from the model and intended this)", - code='decorator-missing-field', - ) - - -def filter_field_decorator_info_by_field( - validator_functions: Iterable[Decorator[FieldDecoratorInfoType]], field: str -) -> list[Decorator[FieldDecoratorInfoType]]: - return [dec for dec in validator_functions if check_validator_fields_against_field_name(dec.info, field)] - - -def apply_each_item_validators( - schema: core_schema.CoreSchema, - each_item_validators: list[Decorator[ValidatorDecoratorInfo]], -) -> core_schema.CoreSchema: - # This V1 compatibility shim should eventually be removed - - # fail early if each_item_validators is empty - if not each_item_validators: - return schema - - # push down any `each_item=True` validators - # note that this won't work for any Annotated types that get wrapped by a function validator - # but that's okay because that didn't exist in V1 - if schema['type'] == 'nullable': - schema['schema'] = apply_each_item_validators(schema['schema'], each_item_validators) - return schema - elif schema['type'] == 'tuple': - if (variadic_item_index := schema.get('variadic_item_index')) is not None: - schema['items_schema'][variadic_item_index] = apply_validators( - schema['items_schema'][variadic_item_index], - each_item_validators, - ) - elif is_list_like_schema_with_items_schema(schema): - inner_schema = schema.get('items_schema', core_schema.any_schema()) - schema['items_schema'] = apply_validators(inner_schema, each_item_validators) - elif schema['type'] == 'dict': - inner_schema = schema.get('values_schema', core_schema.any_schema()) - schema['values_schema'] = apply_validators(inner_schema, each_item_validators) - else: - raise TypeError( - f'`@validator(..., each_item=True)` cannot be applied to fields with a schema of {schema["type"]}' - ) - return schema - - -def _extract_json_schema_info_from_field_info( - info: FieldInfo | ComputedFieldInfo, -) -> tuple[JsonDict | None, JsonDict | JsonSchemaExtraCallable | None]: - json_schema_updates = { - 'title': info.title, - 'description': info.description, - 'deprecated': bool(info.deprecated) or info.deprecated == '' or None, - 'examples': to_jsonable_python(info.examples), - } - json_schema_updates = {k: v for k, v in json_schema_updates.items() if v is not None} - return (json_schema_updates or None, info.json_schema_extra) - - -JsonEncoders = dict[type[Any], JsonEncoder] - - -def _add_custom_serialization_from_json_encoders( - json_encoders: JsonEncoders | None, tp: Any, schema: CoreSchema -) -> CoreSchema: - """Iterate over the json_encoders and add the first matching encoder to the schema. - - Args: - json_encoders: A dictionary of types and their encoder functions. - tp: The type to check for a matching encoder. - schema: The schema to add the encoder to. - """ - if not json_encoders: - return schema - if 'serialization' in schema: - return schema - # Check the class type and its superclasses for a matching encoder - # Decimal.__class__.__mro__ (and probably other cases) doesn't include Decimal itself - # if the type is a GenericAlias (e.g. from list[int]) we need to use __class__ instead of .__mro__ - for base in (tp, *getattr(tp, '__mro__', tp.__class__.__mro__)[:-1]): - encoder = json_encoders.get(base) - if encoder is None: - continue - - warnings.warn( - f'`json_encoders` is deprecated. See https://docs.pydantic.dev/{version_short()}/concepts/serialization/#custom-serializers for alternatives', - PydanticDeprecatedSince20, - ) - - # TODO: in theory we should check that the schema accepts a serialization key - schema['serialization'] = core_schema.plain_serializer_function_ser_schema(encoder, when_used='json') - return schema - - return schema - - -GENERATE_SCHEMA_ERRORS = ( - PydanticForbiddenQualifier, - PydanticInvalidForJsonSchema, - PydanticSchemaGenerationError, - PydanticUndefinedAnnotation, -) -"""Errors raised during core schema generation. This does *not* include `InvalidSchemaError`, which is raised during schema cleaning.""" - - -class InvalidSchemaError(Exception): - """The core schema is invalid.""" - - -class GenerateSchema: - """Generate core schema for a Pydantic model, dataclass and types like `str`, `datetime`, ... .""" - - __slots__ = ( - '_config_wrapper_stack', - '_ns_resolver', - '_typevars_map', - 'field_name_stack', - 'model_type_stack', - 'defs', - ) - - def __init__( - self, - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver | None = None, - typevars_map: Mapping[TypeVar, Any] | None = None, - ) -> None: - # we need a stack for recursing into nested models - self._config_wrapper_stack = ConfigWrapperStack(config_wrapper) - self._ns_resolver = ns_resolver or NsResolver() - self._typevars_map = typevars_map - self.field_name_stack = _FieldNameStack() - self.model_type_stack = _ModelTypeStack() - self.defs = _Definitions() - - def __init_subclass__(cls) -> None: - super().__init_subclass__() - warnings.warn( - 'Subclassing `GenerateSchema` is not supported. The API is highly subject to change in minor versions.', - UserWarning, - stacklevel=2, - ) - - @property - def _config_wrapper(self) -> ConfigWrapper: - return self._config_wrapper_stack.tail - - @property - def _types_namespace(self) -> NamespacesTuple: - return self._ns_resolver.types_namespace - - @property - def _arbitrary_types(self) -> bool: - return self._config_wrapper.arbitrary_types_allowed - - # the following methods can be overridden but should be considered - # unstable / private APIs - def _list_schema(self, items_type: Any) -> CoreSchema: - return core_schema.list_schema(self.generate_schema(items_type)) - - def _dict_schema(self, keys_type: Any, values_type: Any) -> CoreSchema: - return core_schema.dict_schema(self.generate_schema(keys_type), self.generate_schema(values_type)) - - def _set_schema(self, items_type: Any) -> CoreSchema: - return core_schema.set_schema(self.generate_schema(items_type)) - - def _frozenset_schema(self, items_type: Any) -> CoreSchema: - return core_schema.frozenset_schema(self.generate_schema(items_type)) - - def _enum_schema(self, enum_type: type[Enum]) -> CoreSchema: - cases: list[Any] = list(enum_type.__members__.values()) - - enum_ref = get_type_ref(enum_type) - description = None if not enum_type.__doc__ else inspect.cleandoc(enum_type.__doc__) - if ( - description == 'An enumeration.' - ): # This is the default value provided by enum.EnumMeta.__new__; don't use it - description = None - js_updates = {'title': enum_type.__name__, 'description': description} - js_updates = {k: v for k, v in js_updates.items() if v is not None} - - sub_type: Literal['str', 'int', 'float'] | None = None - if issubclass(enum_type, int): - sub_type = 'int' - value_ser_type: core_schema.SerSchema = core_schema.simple_ser_schema('int') - elif issubclass(enum_type, str): - # this handles `StrEnum` (3.11 only), and also `Foobar(str, Enum)` - sub_type = 'str' - value_ser_type = core_schema.simple_ser_schema('str') - elif issubclass(enum_type, float): - sub_type = 'float' - value_ser_type = core_schema.simple_ser_schema('float') - else: - # TODO this is an ugly hack, how do we trigger an Any schema for serialization? - value_ser_type = core_schema.plain_serializer_function_ser_schema(lambda x: x) - - if cases: - - def get_json_schema(schema: CoreSchema, handler: GetJsonSchemaHandler) -> JsonSchemaValue: - json_schema = handler(schema) - original_schema = handler.resolve_ref_schema(json_schema) - original_schema.update(js_updates) - return json_schema - - # we don't want to add the missing to the schema if it's the default one - default_missing = getattr(enum_type._missing_, '__func__', None) is Enum._missing_.__func__ # pyright: ignore[reportFunctionMemberAccess] - enum_schema = core_schema.enum_schema( - enum_type, - cases, - sub_type=sub_type, - missing=None if default_missing else enum_type._missing_, - ref=enum_ref, - metadata={'pydantic_js_functions': [get_json_schema]}, - ) - - if self._config_wrapper.use_enum_values: - enum_schema = core_schema.no_info_after_validator_function( - attrgetter('value'), enum_schema, serialization=value_ser_type - ) - - return enum_schema - - else: - - def get_json_schema_no_cases(_, handler: GetJsonSchemaHandler) -> JsonSchemaValue: - json_schema = handler(core_schema.enum_schema(enum_type, cases, sub_type=sub_type, ref=enum_ref)) - original_schema = handler.resolve_ref_schema(json_schema) - original_schema.update(js_updates) - return json_schema - - # Use an isinstance check for enums with no cases. - # The most important use case for this is creating TypeVar bounds for generics that should - # be restricted to enums. This is more consistent than it might seem at first, since you can only - # subclass enum.Enum (or subclasses of enum.Enum) if all parent classes have no cases. - # We use the get_json_schema function when an Enum subclass has been declared with no cases - # so that we can still generate a valid json schema. - return core_schema.is_instance_schema( - enum_type, - metadata={'pydantic_js_functions': [get_json_schema_no_cases]}, - ) - - def _ip_schema(self, tp: Any) -> CoreSchema: - from ._validators import IP_VALIDATOR_LOOKUP, IpType - - ip_type_json_schema_format: dict[type[IpType], str] = { - IPv4Address: 'ipv4', - IPv4Network: 'ipv4network', - IPv4Interface: 'ipv4interface', - IPv6Address: 'ipv6', - IPv6Network: 'ipv6network', - IPv6Interface: 'ipv6interface', - } - - def ser_ip(ip: Any, info: core_schema.SerializationInfo) -> str | IpType: - if not isinstance(ip, (tp, str)): - raise PydanticSerializationUnexpectedValue( - f"Expected `{tp}` but got `{type(ip)}` with value `'{ip}'` - serialized value may not be as expected." - ) - if info.mode == 'python': - return ip - return str(ip) - - return core_schema.lax_or_strict_schema( - lax_schema=core_schema.no_info_plain_validator_function(IP_VALIDATOR_LOOKUP[tp]), - strict_schema=core_schema.json_or_python_schema( - json_schema=core_schema.no_info_after_validator_function(tp, core_schema.str_schema()), - python_schema=core_schema.is_instance_schema(tp), - ), - serialization=core_schema.plain_serializer_function_ser_schema(ser_ip, info_arg=True, when_used='always'), - metadata={ - 'pydantic_js_functions': [lambda _1, _2: {'type': 'string', 'format': ip_type_json_schema_format[tp]}] - }, - ) - - def _path_schema(self, tp: Any, path_type: Any) -> CoreSchema: - if tp is os.PathLike and (path_type not in {str, bytes} and not typing_objects.is_any(path_type)): - raise PydanticUserError( - '`os.PathLike` can only be used with `str`, `bytes` or `Any`', code='schema-for-unknown-type' - ) - - path_constructor = pathlib.PurePath if tp is os.PathLike else tp - strict_inner_schema = ( - core_schema.bytes_schema(strict=True) if (path_type is bytes) else core_schema.str_schema(strict=True) - ) - lax_inner_schema = core_schema.bytes_schema() if (path_type is bytes) else core_schema.str_schema() - - def path_validator(input_value: str | bytes) -> os.PathLike[Any]: # type: ignore - try: - if path_type is bytes: - if isinstance(input_value, bytes): - try: - input_value = input_value.decode() - except UnicodeDecodeError as e: - raise PydanticCustomError('bytes_type', 'Input must be valid bytes') from e - else: - raise PydanticCustomError('bytes_type', 'Input must be bytes') - elif not isinstance(input_value, str): - raise PydanticCustomError('path_type', 'Input is not a valid path') - - return path_constructor(input_value) # type: ignore - except TypeError as e: - raise PydanticCustomError('path_type', 'Input is not a valid path') from e - - def ser_path(path: Any, info: core_schema.SerializationInfo) -> str | os.PathLike[Any]: - if not isinstance(path, (tp, str)): - raise PydanticSerializationUnexpectedValue( - f"Expected `{tp}` but got `{type(path)}` with value `'{path}'` - serialized value may not be as expected." - ) - if info.mode == 'python': - return path - return str(path) - - instance_schema = core_schema.json_or_python_schema( - json_schema=core_schema.no_info_after_validator_function(path_validator, lax_inner_schema), - python_schema=core_schema.is_instance_schema(tp), - ) - - schema = core_schema.lax_or_strict_schema( - lax_schema=core_schema.union_schema( - [ - instance_schema, - core_schema.no_info_after_validator_function(path_validator, strict_inner_schema), - ], - custom_error_type='path_type', - custom_error_message=f'Input is not a valid path for {tp}', - ), - strict_schema=instance_schema, - serialization=core_schema.plain_serializer_function_ser_schema(ser_path, info_arg=True, when_used='always'), - metadata={'pydantic_js_functions': [lambda source, handler: {**handler(source), 'format': 'path'}]}, - ) - return schema - - def _deque_schema(self, items_type: Any) -> CoreSchema: - from ._serializers import serialize_sequence_via_list - from ._validators import deque_validator - - item_type_schema = self.generate_schema(items_type) - - # we have to use a lax list schema here, because we need to validate the deque's - # items via a list schema, but it's ok if the deque itself is not a list - list_schema = core_schema.list_schema(item_type_schema, strict=False) - - check_instance = core_schema.json_or_python_schema( - json_schema=list_schema, - python_schema=core_schema.is_instance_schema(collections.deque, cls_repr='Deque'), - ) - - lax_schema = core_schema.no_info_wrap_validator_function(deque_validator, list_schema) - - return core_schema.lax_or_strict_schema( - lax_schema=lax_schema, - strict_schema=core_schema.chain_schema([check_instance, lax_schema]), - serialization=core_schema.wrap_serializer_function_ser_schema( - serialize_sequence_via_list, schema=item_type_schema, info_arg=True - ), - ) - - def _mapping_schema(self, tp: Any, keys_type: Any, values_type: Any) -> CoreSchema: - from ._validators import MAPPING_ORIGIN_MAP, defaultdict_validator, get_defaultdict_default_default_factory - - mapped_origin = MAPPING_ORIGIN_MAP[tp] - keys_schema = self.generate_schema(keys_type) - with warnings.catch_warnings(): - # We kind of abused `Field()` default factories to be able to specify - # the `defaultdict`'s `default_factory`. As a consequence, we get warnings - # as normally `FieldInfo.default_factory` is unsupported in the context where - # `Field()` is used and our only solution is to ignore them (note that this might - # wrongfully ignore valid warnings, e.g. if the `value_type` is a PEP 695 type alias - # with unsupported metadata). - warnings.simplefilter('ignore', category=UnsupportedFieldAttributeWarning) - values_schema = self.generate_schema(values_type) - dict_schema = core_schema.dict_schema(keys_schema, values_schema, strict=False) - - if mapped_origin is dict: - schema = dict_schema - else: - check_instance = core_schema.json_or_python_schema( - json_schema=dict_schema, - python_schema=core_schema.is_instance_schema(mapped_origin), - ) - - if tp is collections.defaultdict: - default_default_factory = get_defaultdict_default_default_factory(values_type) - coerce_instance_wrap = partial( - core_schema.no_info_wrap_validator_function, - partial(defaultdict_validator, default_default_factory=default_default_factory), - ) - else: - coerce_instance_wrap = partial(core_schema.no_info_after_validator_function, mapped_origin) - - lax_schema = coerce_instance_wrap(dict_schema) - strict_schema = core_schema.chain_schema([check_instance, lax_schema]) - - schema = core_schema.lax_or_strict_schema( - lax_schema=lax_schema, - strict_schema=strict_schema, - serialization=core_schema.wrap_serializer_function_ser_schema( - lambda v, h: h(v), schema=dict_schema, info_arg=False - ), - ) - - return schema - - def _fraction_schema(self) -> CoreSchema: - """Support for [`fractions.Fraction`][fractions.Fraction].""" - from ._validators import fraction_validator - - # TODO: note, this is a fairly common pattern, re lax / strict for attempted type coercion, - # can we use a helper function to reduce boilerplate? - return core_schema.lax_or_strict_schema( - lax_schema=core_schema.no_info_plain_validator_function(fraction_validator), - strict_schema=core_schema.json_or_python_schema( - json_schema=core_schema.no_info_plain_validator_function(fraction_validator), - python_schema=core_schema.is_instance_schema(Fraction), - ), - # use str serialization to guarantee round trip behavior - serialization=core_schema.to_string_ser_schema(when_used='always'), - metadata={'pydantic_js_functions': [lambda _1, _2: {'type': 'string', 'format': 'fraction'}]}, - ) - - def _arbitrary_type_schema(self, tp: Any) -> CoreSchema: - if not isinstance(tp, type): - warnings.warn( - f'{tp!r} is not a Python type (it may be an instance of an object),' - ' Pydantic will allow any object with no validation since we cannot even' - ' enforce that the input is an instance of the given type.' - ' To get rid of this error wrap the type with `pydantic.SkipValidation`.', - ArbitraryTypeWarning, - ) - return core_schema.any_schema() - return core_schema.is_instance_schema(tp) - - def _unknown_type_schema(self, obj: Any) -> CoreSchema: - raise PydanticSchemaGenerationError( - f'Unable to generate pydantic-core schema for {obj!r}. ' - 'Set `arbitrary_types_allowed=True` in the model_config to ignore this error' - ' or implement `__get_pydantic_core_schema__` on your type to fully support it.' - '\n\nIf you got this error by calling handler() within' - ' `__get_pydantic_core_schema__` then you likely need to call' - ' `handler.generate_schema()` since we do not call' - ' `__get_pydantic_core_schema__` on `` otherwise to avoid infinite recursion.' - ) - - def _apply_discriminator_to_union( - self, schema: CoreSchema, discriminator: str | Discriminator | None - ) -> CoreSchema: - if discriminator is None: - return schema - try: - return _discriminated_union.apply_discriminator( - schema, - discriminator, - self.defs._definitions, - ) - except _discriminated_union.MissingDefinitionForUnionRef: - # defer until defs are resolved - _discriminated_union.set_discriminator_in_metadata( - schema, - discriminator, - ) - return schema - - def clean_schema(self, schema: CoreSchema) -> CoreSchema: - return self.defs.finalize_schema(schema) - - def _add_js_function(self, metadata_schema: CoreSchema, js_function: Callable[..., Any]) -> None: - metadata = metadata_schema.get('metadata', {}) - pydantic_js_functions = metadata.setdefault('pydantic_js_functions', []) - # because of how we generate core schemas for nested generic models - # we can end up adding `BaseModel.__get_pydantic_json_schema__` multiple times - # this check may fail to catch duplicates if the function is a `functools.partial` - # or something like that, but if it does it'll fail by inserting the duplicate - if js_function not in pydantic_js_functions: - pydantic_js_functions.append(js_function) - metadata_schema['metadata'] = metadata - - def generate_schema( - self, - obj: Any, - ) -> core_schema.CoreSchema: - """Generate core schema. - - Args: - obj: The object to generate core schema for. - - Returns: - The generated core schema. - - Raises: - PydanticUndefinedAnnotation: - If it is not possible to evaluate forward reference. - PydanticSchemaGenerationError: - If it is not possible to generate pydantic-core schema. - TypeError: - - If `alias_generator` returns a disallowed type (must be str, AliasPath or AliasChoices). - - If V1 style validator with `each_item=True` applied on a wrong field. - PydanticUserError: - - If `typing.TypedDict` is used instead of `typing_extensions.TypedDict` on Python < 3.12. - - If `__modify_schema__` method is used instead of `__get_pydantic_json_schema__`. - """ - schema = self._generate_schema_from_get_schema_method(obj, obj) - - if schema is None: - schema = self._generate_schema_inner(obj) - - metadata_js_function = _extract_get_pydantic_json_schema(obj) - if metadata_js_function is not None: - metadata_schema = resolve_original_schema(schema, self.defs) - if metadata_schema: - self._add_js_function(metadata_schema, metadata_js_function) - - schema = _add_custom_serialization_from_json_encoders(self._config_wrapper.json_encoders, obj, schema) - - return schema - - def _model_schema(self, cls: type[BaseModel]) -> core_schema.CoreSchema: - """Generate schema for a Pydantic model.""" - BaseModel_ = import_cached_base_model() - - with self.defs.get_schema_or_ref(cls) as (model_ref, maybe_schema): - if maybe_schema is not None: - return maybe_schema - - schema = cls.__dict__.get('__pydantic_core_schema__') - if schema is not None and not isinstance(schema, MockCoreSchema): - if schema['type'] == 'definitions': - schema = self.defs.unpack_definitions(schema) - ref = get_ref(schema) - if ref: - return self.defs.create_definition_reference_schema(schema) - else: - return schema - - config_wrapper = ConfigWrapper(cls.model_config, check=False) - - with self._config_wrapper_stack.push(config_wrapper), self._ns_resolver.push(cls): - core_config = self._config_wrapper.core_config(title=cls.__name__) - - if cls.__pydantic_fields_complete__ or cls is BaseModel_: - fields = getattr(cls, '__pydantic_fields__', {}) - extra_info = getattr(cls, '__pydantic_extra_info__', None) - else: - if '__pydantic_fields__' not in cls.__dict__: - # This happens when we have a loop in the schema generation: - # class Base[T](BaseModel): - # t: T - # - # class Other(BaseModel): - # b: 'Base[Other]' - # When we build fields for `Other`, we evaluate the forward annotation. - # At this point, `Other` doesn't have the model fields set. We create - # `Base[Other]`; model fields are successfully built, and we try to generate - # a schema for `t: Other`. As `Other.__pydantic_fields__` aren't set, we abort. - raise PydanticUndefinedAnnotation( - name=cls.__name__, - message=f'Class {cls.__name__!r} is not defined', - ) - try: - fields, extra_info = rebuild_model_fields( - cls, - config_wrapper=self._config_wrapper, - ns_resolver=self._ns_resolver, - typevars_map=self._typevars_map or {}, - ) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - decorators = cls.__pydantic_decorators__ - computed_fields = decorators.computed_fields - check_decorator_fields_exist( - chain( - decorators.field_validators.values(), - decorators.field_serializers.values(), - decorators.validators.values(), - ), - {*fields.keys(), *computed_fields.keys()}, - ) - - model_validators = decorators.model_validators.values() - - extras_schema = None - extras_keys_schema = None - if core_config.get('extra_fields_behavior') == 'allow' and extra_info is not None: - tp = get_origin(extra_info.annotation) - if tp not in DICT_TYPES: - raise PydanticSchemaGenerationError( - 'The type annotation for `__pydantic_extra__` must be `dict[str, ...]`' - ) - # See the comments in `_get_args_resolving_forward_refs()` for why we need - # to re-evaluate the annotation: - extra_keys_type, extra_items_type = self._get_args_resolving_forward_refs( - extra_info.annotation, - required=True, - ) - if extra_keys_type is not str: - extras_keys_schema = self.generate_schema(extra_keys_type) - if not typing_objects.is_any(extra_items_type): - extras_schema = self.generate_schema(extra_items_type) - - generic_origin: type[BaseModel] | None = getattr(cls, '__pydantic_generic_metadata__', {}).get('origin') - - if cls.__pydantic_root_model__: - inner_schema, metadata = self._common_field_schema('root', fields['root'], decorators) - if cls.__doc__ and metadata.get('pydantic_js_updates', {}).get('description'): - # This is a bit of a leaky abstraction, but as the model docstring takes priority - # over the root field's description, we need to override it here. This can't be done - # in the JSON Schema generation logic because the metadata's `pydantic_js_updates` are - # applied last, and overrides any value previously set (so the description set from the - # docstring in `GenerateJsonSchema._update_class_schema()` is overridden): - update_core_metadata( - metadata, pydantic_js_updates={'description': inspect.cleandoc(cls.__doc__)} - ) - - inner_schema = apply_model_validators(inner_schema, model_validators, 'inner') - model_schema = core_schema.model_schema( - cls, - inner_schema, - generic_origin=generic_origin, - custom_init=getattr(cls, '__pydantic_custom_init__', None), - root_model=True, - post_init=getattr(cls, '__pydantic_post_init__', None), - config=core_config, - ref=model_ref, - metadata=metadata, - ) - else: - fields_schema: core_schema.CoreSchema = core_schema.model_fields_schema( - {k: self._generate_md_field_schema(k, v, decorators) for k, v in fields.items()}, - computed_fields=[ - self._computed_field_schema(d, decorators.field_serializers) - for d in computed_fields.values() - ], - extras_schema=extras_schema, - extras_keys_schema=extras_keys_schema, - model_name=cls.__name__, - ) - inner_schema = apply_validators(fields_schema, decorators.root_validators.values()) - inner_schema = apply_model_validators(inner_schema, model_validators, 'inner') - - model_schema = core_schema.model_schema( - cls, - inner_schema, - generic_origin=generic_origin, - custom_init=getattr(cls, '__pydantic_custom_init__', None), - root_model=False, - post_init=getattr(cls, '__pydantic_post_init__', None), - config=core_config, - ref=model_ref, - ) - - schema = self._apply_model_serializers(model_schema, decorators.model_serializers.values()) - schema = apply_model_validators(schema, model_validators, 'outer') - return self.defs.create_definition_reference_schema(schema) - - def _resolve_self_type(self, obj: Any) -> Any: - obj = self.model_type_stack.get() - if obj is None: - raise PydanticUserError('`typing.Self` is invalid in this context', code='invalid-self-type') - return obj - - def _generate_schema_from_get_schema_method(self, obj: Any, source: Any) -> core_schema.CoreSchema | None: - BaseModel_ = import_cached_base_model() - - get_schema = getattr(obj, '__get_pydantic_core_schema__', None) - is_base_model_get_schema = ( - getattr(get_schema, '__func__', None) is BaseModel_.__get_pydantic_core_schema__.__func__ # pyright: ignore[reportFunctionMemberAccess] - ) - - if ( - get_schema is not None - # BaseModel.__get_pydantic_core_schema__ is defined for backwards compatibility, - # to allow existing code to call `super().__get_pydantic_core_schema__` in Pydantic - # model that overrides `__get_pydantic_core_schema__`. However, it raises a deprecation - # warning stating that the method will be removed, and during the core schema gen we actually - # don't call the method: - and not is_base_model_get_schema - ): - # Some referenceable types might have a `__get_pydantic_core_schema__` method - # defined on it by users (e.g. on a dataclass). This generally doesn't play well - # as these types are already recognized by the `GenerateSchema` class and isn't ideal - # as we might end up calling `get_schema_or_ref` (expensive) on types that are actually - # not referenceable: - with self.defs.get_schema_or_ref(obj) as (_, maybe_schema): - if maybe_schema is not None: - return maybe_schema - - if obj is source: - ref_mode = 'unpack' - else: - ref_mode = 'to-def' - schema = get_schema( - source, CallbackGetCoreSchemaHandler(self._generate_schema_inner, self, ref_mode=ref_mode) - ) - if schema['type'] == 'definitions': - schema = self.defs.unpack_definitions(schema) - - ref = get_ref(schema) - if ref: - return self.defs.create_definition_reference_schema(schema) - - # Note: if schema is of type `'definition-ref'`, we might want to copy it as a - # safety measure (because these are inlined in place -- i.e. mutated directly) - return schema - - if get_schema is None and (validators := getattr(obj, '__get_validators__', None)) is not None: - from pydantic.v1 import BaseModel as BaseModelV1 - - if issubclass(obj, BaseModelV1): - warnings.warn( - f'Mixing V1 models and V2 models (or constructs, like `TypeAdapter`) is not supported. Please upgrade `{obj.__name__}` to V2.', - UserWarning, - ) - else: - warnings.warn( - '`__get_validators__` is deprecated and will be removed, use `__get_pydantic_core_schema__` instead.', - PydanticDeprecatedSince20, - ) - return core_schema.chain_schema([core_schema.with_info_plain_validator_function(v) for v in validators()]) - - def _resolve_forward_ref(self, obj: Any) -> Any: - # we assume that types_namespace has the target of forward references in its scope, - # but this could fail, for example, if calling Validator on an imported type which contains - # forward references to other types only defined in the module from which it was imported - # `Validator(SomeImportedTypeAliasWithAForwardReference)` - # or the equivalent for BaseModel - # class Model(BaseModel): - # x: SomeImportedTypeAliasWithAForwardReference - try: - obj = _typing_extra.eval_type_backport(obj, *self._types_namespace) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - # if obj is still a ForwardRef, it means we can't evaluate it, raise PydanticUndefinedAnnotation - if isinstance(obj, ForwardRef): - raise PydanticUndefinedAnnotation(obj.__forward_arg__, f'Unable to evaluate forward reference {obj}') - - if self._typevars_map: - obj = replace_types(obj, self._typevars_map) - - return obj - - @overload - def _get_args_resolving_forward_refs(self, obj: Any, required: Literal[True]) -> tuple[Any, ...]: ... - - @overload - def _get_args_resolving_forward_refs(self, obj: Any) -> tuple[Any, ...] | None: ... - - def _get_args_resolving_forward_refs(self, obj: Any, required: bool = False) -> tuple[Any, ...] | None: - args = get_args(obj) - if args: - if isinstance(obj, GenericAlias): - # PEP 585 generic aliases don't convert args to ForwardRefs, unlike `typing.List/Dict` etc. - # This was fixed in https://github.com/python/cpython/pull/30900 (Python 3.11). - # TODO: this shouldn't be necessary (probably even this `_get_args_resolving_forward_refs()` function) - # once we drop support for Python 3.10 *or* if we implement our own `typing._eval_type()` implementation. - args = (_typing_extra._make_forward_ref(a) if isinstance(a, str) else a for a in args) - args = tuple(self._resolve_forward_ref(a) if isinstance(a, ForwardRef) else a for a in args) - elif required: # pragma: no cover - raise TypeError(f'Expected {obj} to have generic parameters but it had none') - return args - - def _get_first_arg_or_any(self, obj: Any) -> Any: - args = self._get_args_resolving_forward_refs(obj) - if not args: - return Any - return args[0] - - def _get_first_two_args_or_any(self, obj: Any) -> tuple[Any, Any]: - args = self._get_args_resolving_forward_refs(obj) - if not args: - return (Any, Any) - if len(args) < 2: - origin = get_origin(obj) - raise TypeError(f'Expected two type arguments for {origin}, got 1') - return args[0], args[1] - - def _generate_schema_inner(self, obj: Any) -> core_schema.CoreSchema: - if typing_objects.is_self(obj): - obj = self._resolve_self_type(obj) - - if typing_objects.is_annotated(get_origin(obj)): - return self._annotated_schema(obj) - - if isinstance(obj, dict): - # we assume this is already a valid schema - return obj # type: ignore[return-value] - - if isinstance(obj, str): - obj = ForwardRef(obj) - - if isinstance(obj, ForwardRef): - return self.generate_schema(self._resolve_forward_ref(obj)) - - BaseModel = import_cached_base_model() - - if lenient_issubclass(obj, BaseModel): - with self.model_type_stack.push(obj): - return self._model_schema(obj) - - if isinstance(obj, PydanticRecursiveRef): - return core_schema.definition_reference_schema(schema_ref=obj.type_ref) - - return self.match_type(obj) - - def match_type(self, obj: Any) -> core_schema.CoreSchema: # noqa: C901 - """Main mapping of types to schemas. - - The general structure is a series of if statements starting with the simple cases - (non-generic primitive types) and then handling generics and other more complex cases. - - Each case either generates a schema directly, calls into a public user-overridable method - (like `GenerateSchema.tuple_variable_schema`) or calls into a private method that handles some - boilerplate before calling into the user-facing method (e.g. `GenerateSchema._tuple_schema`). - - The idea is that we'll evolve this into adding more and more user facing methods over time - as they get requested and we figure out what the right API for them is. - """ - if obj is str: - return core_schema.str_schema() - elif obj is bytes: - return core_schema.bytes_schema() - elif obj is int: - return core_schema.int_schema() - elif obj is float: - return core_schema.float_schema() - elif obj is bool: - return core_schema.bool_schema() - elif obj is complex: - return core_schema.complex_schema() - elif typing_objects.is_any(obj) or obj is object: - return core_schema.any_schema() - elif obj is datetime.date: - return core_schema.date_schema() - elif obj is datetime.datetime: - return core_schema.datetime_schema() - elif obj is datetime.time: - return core_schema.time_schema() - elif obj is datetime.timedelta: - return core_schema.timedelta_schema() - elif obj is Decimal: - return core_schema.decimal_schema() - elif obj is UUID: - return core_schema.uuid_schema() - elif obj is Url: - return core_schema.url_schema() - elif obj is Fraction: - return self._fraction_schema() - elif obj is MultiHostUrl: - return core_schema.multi_host_url_schema() - elif obj is None or obj is _typing_extra.NoneType: - return core_schema.none_schema() - if obj is MISSING: - return core_schema.missing_sentinel_schema() - elif obj in IP_TYPES: - return self._ip_schema(obj) - elif obj in TUPLE_TYPES: - return self._tuple_schema(obj) - elif obj in LIST_TYPES: - return self._list_schema(Any) - elif obj in SET_TYPES: - return self._set_schema(Any) - elif obj in FROZEN_SET_TYPES: - return self._frozenset_schema(Any) - elif obj in SEQUENCE_TYPES: - return self._sequence_schema(Any) - elif obj in ITERABLE_TYPES: - return self._iterable_schema(obj) - elif obj in DICT_TYPES: - return self._dict_schema(Any, Any) - elif obj in PATH_TYPES: - return self._path_schema(obj, Any) - elif obj in DEQUE_TYPES: - return self._deque_schema(Any) - elif obj in MAPPING_TYPES: - return self._mapping_schema(obj, Any, Any) - elif obj in COUNTER_TYPES: - return self._mapping_schema(obj, Any, int) - elif typing_objects.is_typealiastype(obj): - return self._type_alias_type_schema(obj) - elif obj is type: - return self._type_schema() - elif _typing_extra.is_callable(obj): - return core_schema.callable_schema() - elif typing_objects.is_literal(get_origin(obj)): - return self._literal_schema(obj) - elif is_typeddict(obj): - return self._typed_dict_schema(obj, None) - elif inspect.isclass(obj) and issubclass(obj, Enum): - # NOTE: this must come before the `is_namedtuple()` check as enums values - # can be namedtuples: - return self._enum_schema(obj) - elif _typing_extra.is_namedtuple(obj): - return self._namedtuple_schema(obj, None) - elif typing_objects.is_newtype(obj): - # NewType, can't use isinstance because it fails <3.10 - return self.generate_schema(obj.__supertype__) - elif obj in PATTERN_TYPES: - return self._pattern_schema(obj) - elif _typing_extra.is_hashable(obj): - return self._hashable_schema() - elif isinstance(obj, typing.TypeVar): - return self._unsubstituted_typevar_schema(obj) - elif _typing_extra.is_finalvar(obj): - if obj is Final: - return core_schema.any_schema() - return self.generate_schema( - self._get_first_arg_or_any(obj), - ) - elif isinstance(obj, VALIDATE_CALL_SUPPORTED_TYPES): - return self._call_schema(obj) # pyright: ignore[reportArgumentType] - elif obj is ZoneInfo: - return self._zoneinfo_schema() - - # dataclasses.is_dataclass coerces dc instances to types, but we only handle - # the case of a dc type here - if dataclasses.is_dataclass(obj): - return self._dataclass_schema(obj, None) # pyright: ignore[reportArgumentType] - - origin = get_origin(obj) - if origin is not None: - return self._match_generic_type(obj, origin) - - if self._arbitrary_types: - return self._arbitrary_type_schema(obj) - return self._unknown_type_schema(obj) - - def _match_generic_type(self, obj: Any, origin: Any) -> CoreSchema: # noqa: C901 - # Need to handle generic dataclasses before looking for the schema properties because attribute accesses - # on _GenericAlias delegate to the origin type, so lose the information about the concrete parametrization - # As a result, currently, there is no way to cache the schema for generic dataclasses. This may be possible - # to resolve by modifying the value returned by `Generic.__class_getitem__`, but that is a dangerous game. - if dataclasses.is_dataclass(origin): - return self._dataclass_schema(obj, origin) # pyright: ignore[reportArgumentType] - if _typing_extra.is_namedtuple(origin): - return self._namedtuple_schema(obj, origin) - - schema = self._generate_schema_from_get_schema_method(origin, obj) - if schema is not None: - return schema - - if typing_objects.is_typealiastype(origin): - return self._type_alias_type_schema(obj) - elif is_union_origin(origin): - return self._union_schema(obj) - elif origin in TUPLE_TYPES: - return self._tuple_schema(obj) - elif origin in LIST_TYPES: - return self._list_schema(self._get_first_arg_or_any(obj)) - elif origin in SET_TYPES: - return self._set_schema(self._get_first_arg_or_any(obj)) - elif origin in FROZEN_SET_TYPES: - return self._frozenset_schema(self._get_first_arg_or_any(obj)) - elif origin in DICT_TYPES: - return self._dict_schema(*self._get_first_two_args_or_any(obj)) - elif origin in PATH_TYPES: - return self._path_schema(origin, self._get_first_arg_or_any(obj)) - elif origin in DEQUE_TYPES: - return self._deque_schema(self._get_first_arg_or_any(obj)) - elif origin in MAPPING_TYPES: - return self._mapping_schema(origin, *self._get_first_two_args_or_any(obj)) - elif origin in COUNTER_TYPES: - return self._mapping_schema(origin, self._get_first_arg_or_any(obj), int) - elif is_typeddict(origin): - return self._typed_dict_schema(obj, origin) - elif origin in TYPE_TYPES: - return self._subclass_schema(obj) - elif origin in SEQUENCE_TYPES: - return self._sequence_schema(self._get_first_arg_or_any(obj)) - elif origin in ITERABLE_TYPES: - return self._iterable_schema(obj) - elif origin in PATTERN_TYPES: - return self._pattern_schema(obj) - - if self._arbitrary_types: - return self._arbitrary_type_schema(origin) - return self._unknown_type_schema(obj) - - def _generate_td_field_schema( - self, - name: str, - field_info: FieldInfo, - decorators: DecoratorInfos, - *, - required: bool = True, - ) -> core_schema.TypedDictField: - """Prepare a TypedDictField to represent a model or typeddict field.""" - schema, metadata = self._common_field_schema(name, field_info, decorators) - return core_schema.typed_dict_field( - schema, - required=False if not field_info.is_required() else required, - serialization_exclude=field_info.exclude, - validation_alias=_convert_to_aliases(field_info.validation_alias), - serialization_alias=field_info.serialization_alias, - serialization_exclude_if=field_info.exclude_if, - metadata=metadata, - ) - - def _generate_md_field_schema( - self, - name: str, - field_info: FieldInfo, - decorators: DecoratorInfos, - ) -> core_schema.ModelField: - """Prepare a ModelField to represent a model field.""" - schema, metadata = self._common_field_schema(name, field_info, decorators) - return core_schema.model_field( - schema, - serialization_exclude=field_info.exclude, - validation_alias=_convert_to_aliases(field_info.validation_alias), - serialization_alias=field_info.serialization_alias, - serialization_exclude_if=field_info.exclude_if, - frozen=field_info.frozen, - metadata=metadata, - ) - - def _generate_dc_field_schema( - self, - name: str, - field_info: FieldInfo, - decorators: DecoratorInfos, - ) -> core_schema.DataclassField: - """Prepare a DataclassField to represent the parameter/field, of a dataclass.""" - schema, metadata = self._common_field_schema(name, field_info, decorators) - return core_schema.dataclass_field( - name, - schema, - init=field_info.init, - init_only=field_info.init_var or None, - kw_only=None if field_info.kw_only else False, - serialization_exclude=field_info.exclude, - validation_alias=_convert_to_aliases(field_info.validation_alias), - serialization_alias=field_info.serialization_alias, - serialization_exclude_if=field_info.exclude_if, - frozen=field_info.frozen, - metadata=metadata, - ) - - def _common_field_schema( # C901 - self, name: str, field_info: FieldInfo, decorators: DecoratorInfos - ) -> tuple[CoreSchema, dict[str, Any]]: - source_type, annotations = field_info.annotation, field_info.metadata - - def set_discriminator(schema: CoreSchema) -> CoreSchema: - schema = self._apply_discriminator_to_union(schema, field_info.discriminator) - return schema - - # Convert `@field_validator` decorators to `Before/After/Plain/WrapValidator` instances: - validators_from_decorators = [ - _mode_to_validator[decorator.info.mode]._from_decorator(decorator) - for decorator in filter_field_decorator_info_by_field(decorators.field_validators.values(), name) - ] - - with self.field_name_stack.push(name): - if field_info.discriminator is not None: - schema = self._apply_annotations( - source_type, annotations + validators_from_decorators, transform_inner_schema=set_discriminator - ) - else: - schema = self._apply_annotations( - source_type, - annotations + validators_from_decorators, - ) - - # This V1 compatibility shim should eventually be removed - # push down any `each_item=True` validators - # note that this won't work for any Annotated types that get wrapped by a function validator - # but that's okay because that didn't exist in V1 - this_field_validators = filter_field_decorator_info_by_field(decorators.validators.values(), name) - if _validators_require_validate_default(this_field_validators): - field_info.validate_default = True - each_item_validators = [v for v in this_field_validators if v.info.each_item is True] - this_field_validators = [v for v in this_field_validators if v not in each_item_validators] - schema = apply_each_item_validators(schema, each_item_validators) - - schema = apply_validators(schema, this_field_validators) - - # the default validator needs to go outside of any other validators - # so that it is the topmost validator for the field validator - # which uses it to check if the field has a default value or not - if not field_info.is_required(): - schema = wrap_default(field_info, schema) - - schema = self._apply_field_serializers( - schema, filter_field_decorator_info_by_field(decorators.field_serializers.values(), name) - ) - - pydantic_js_updates, pydantic_js_extra = _extract_json_schema_info_from_field_info(field_info) - core_metadata: dict[str, Any] = {} - update_core_metadata( - core_metadata, pydantic_js_updates=pydantic_js_updates, pydantic_js_extra=pydantic_js_extra - ) - - return schema, core_metadata - - def _union_schema(self, union_type: Any) -> core_schema.CoreSchema: - """Generate schema for a Union.""" - args = self._get_args_resolving_forward_refs(union_type, required=True) - choices: list[CoreSchema] = [] - nullable = False - for arg in args: - if arg is None or arg is _typing_extra.NoneType: - nullable = True - else: - choices.append(self.generate_schema(arg)) - - if len(choices) == 1: - s = choices[0] - else: - choices_with_tags: list[CoreSchema | tuple[CoreSchema, str]] = [] - for choice in choices: - tag = cast(CoreMetadata, choice.get('metadata', {})).get('pydantic_internal_union_tag_key') - if tag is not None: - choices_with_tags.append((choice, tag)) - else: - choices_with_tags.append(choice) - s = core_schema.union_schema(choices_with_tags) - - if nullable: - s = core_schema.nullable_schema(s) - return s - - def _type_alias_type_schema(self, obj: TypeAliasType) -> CoreSchema: - with self.defs.get_schema_or_ref(obj) as (ref, maybe_schema): - if maybe_schema is not None: - return maybe_schema - - origin: TypeAliasType = get_origin(obj) or obj - typevars_map = get_standard_typevars_map(obj) - - with self._ns_resolver.push(origin): - try: - annotation = _typing_extra.eval_type(origin.__value__, *self._types_namespace) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - annotation = replace_types(annotation, typevars_map) - schema = self.generate_schema(annotation) - assert schema['type'] != 'definitions' - schema['ref'] = ref # type: ignore - return self.defs.create_definition_reference_schema(schema) - - def _literal_schema(self, literal_type: Any) -> CoreSchema: - """Generate schema for a Literal.""" - expected = list(get_literal_values(literal_type, type_check=False, unpack_type_aliases='eager')) - assert expected, f'literal "expected" cannot be empty, obj={literal_type}' - schema = core_schema.literal_schema(expected) - - if self._config_wrapper.use_enum_values and any(isinstance(v, Enum) for v in expected): - schema = core_schema.no_info_after_validator_function( - lambda v: v.value if isinstance(v, Enum) else v, schema - ) - - return schema - - def _typed_dict_schema(self, typed_dict_cls: Any, origin: Any) -> core_schema.CoreSchema: - """Generate a core schema for a `TypedDict` class. - - To be able to build a `DecoratorInfos` instance for the `TypedDict` class (which will include - validators, serializers, etc.), we need to have access to the original bases of the class - (see https://docs.python.org/3/library/types.html#types.get_original_bases). - However, the `__orig_bases__` attribute was only added in 3.12 (https://github.com/python/cpython/pull/103698). - - For this reason, we require Python 3.12 (or using the `typing_extensions` backport). - """ - FieldInfo = import_cached_field_info() - - with ( - self.model_type_stack.push(typed_dict_cls), - self.defs.get_schema_or_ref(typed_dict_cls) as ( - typed_dict_ref, - maybe_schema, - ), - ): - if maybe_schema is not None: - return maybe_schema - - typevars_map = get_standard_typevars_map(typed_dict_cls) - if origin is not None: - typed_dict_cls = origin - - if not _SUPPORTS_TYPEDDICT and type(typed_dict_cls).__module__ == 'typing': - raise PydanticUserError( - 'Please use `typing_extensions.TypedDict` instead of `typing.TypedDict` on Python < 3.12.', - code='typed-dict-version', - ) - - try: - # if a typed dictionary class doesn't have config, we use the parent's config, hence a default of `None` - # see https://github.com/pydantic/pydantic/issues/10917 - config: ConfigDict | None = get_attribute_from_bases(typed_dict_cls, '__pydantic_config__') - except AttributeError: - config = None - - with self._config_wrapper_stack.push(config): - core_config = self._config_wrapper.core_config(title=typed_dict_cls.__name__) - - required_keys: frozenset[str] = typed_dict_cls.__required_keys__ - - fields: dict[str, core_schema.TypedDictField] = {} - - decorators = DecoratorInfos.build(typed_dict_cls, replace_wrapped_methods=False) - decorators.update_from_config(self._config_wrapper) - - if self._config_wrapper.use_attribute_docstrings: - field_docstrings = extract_docstrings_from_cls(typed_dict_cls, use_inspect=True) - else: - field_docstrings = None - - try: - annotations = _typing_extra.get_cls_type_hints(typed_dict_cls, ns_resolver=self._ns_resolver) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - readonly_fields: list[str] = [] - - for field_name, annotation in annotations.items(): - field_info = FieldInfo.from_annotation(annotation, _source=AnnotationSource.TYPED_DICT) - field_info.annotation = replace_types(field_info.annotation, typevars_map) - - required = ( - field_name in required_keys or 'required' in field_info._qualifiers - ) and 'not_required' not in field_info._qualifiers - if 'read_only' in field_info._qualifiers: - readonly_fields.append(field_name) - - if ( - field_docstrings is not None - and field_info.description is None - and field_name in field_docstrings - ): - field_info.description = field_docstrings[field_name] - update_field_from_config(self._config_wrapper, field_name, field_info) - - fields[field_name] = self._generate_td_field_schema( - field_name, field_info, decorators, required=required - ) - - if readonly_fields: - fields_repr = ', '.join(repr(f) for f in readonly_fields) - plural = len(readonly_fields) >= 2 - warnings.warn( - f'Item{"s" if plural else ""} {fields_repr} on TypedDict class {typed_dict_cls.__name__!r} ' - f'{"are" if plural else "is"} using the `ReadOnly` qualifier. Pydantic will not protect items ' - 'from any mutation on dictionary instances.', - UserWarning, - ) - - extra_behavior: core_schema.ExtraBehavior = 'ignore' - extras_schema: CoreSchema | None = None # For 'allow', equivalent to `Any` - no validation performed. - - # `__closed__` is `None` when not specified (equivalent to `False`): - is_closed = bool(getattr(typed_dict_cls, '__closed__', False)) - extra_items = getattr(typed_dict_cls, '__extra_items__', typing_extensions.NoExtraItems) - if is_closed: - extra_behavior = 'forbid' - extras_schema = None - elif not typing_objects.is_noextraitems(extra_items): - extra_behavior = 'allow' - extras_schema = self.generate_schema(replace_types(extra_items, typevars_map)) - - if (config_extra := self._config_wrapper.extra) in ('allow', 'forbid'): - if is_closed and config_extra == 'allow': - warnings.warn( - f"TypedDict class {typed_dict_cls.__qualname__!r} is closed, but 'extra' configuration " - "is set to `'allow'`. The 'extra' configuration value will be ignored.", - category=TypedDictExtraConfigWarning, - ) - elif not typing_objects.is_noextraitems(extra_items) and config_extra == 'forbid': - warnings.warn( - f"TypedDict class {typed_dict_cls.__qualname__!r} allows extra items, but 'extra' configuration " - "is set to `'forbid'`. The 'extra' configuration value will be ignored.", - category=TypedDictExtraConfigWarning, - ) - else: - extra_behavior = config_extra - - td_schema = core_schema.typed_dict_schema( - fields, - cls=typed_dict_cls, - computed_fields=[ - self._computed_field_schema(d, decorators.field_serializers) - for d in decorators.computed_fields.values() - ], - extra_behavior=extra_behavior, - extras_schema=extras_schema, - ref=typed_dict_ref, - config=core_config, - ) - - schema = self._apply_model_serializers(td_schema, decorators.model_serializers.values()) - schema = apply_model_validators(schema, decorators.model_validators.values(), 'all') - return self.defs.create_definition_reference_schema(schema) - - def _namedtuple_schema(self, namedtuple_cls: Any, origin: Any) -> core_schema.CoreSchema: - """Generate schema for a NamedTuple.""" - with ( - self.model_type_stack.push(namedtuple_cls), - self.defs.get_schema_or_ref(namedtuple_cls) as ( - namedtuple_ref, - maybe_schema, - ), - ): - if maybe_schema is not None: - return maybe_schema - typevars_map = get_standard_typevars_map(namedtuple_cls) - if origin is not None: - namedtuple_cls = origin - - try: - annotations = _typing_extra.get_cls_type_hints(namedtuple_cls, ns_resolver=self._ns_resolver) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - # Filter annotations to only include fields that are actually in the NamedTuple - # (as subclassing an existing NamedTuple is not supported yet - see https://github.com/python/typing/issues/427) - # and use `Any` if no annotation exist (i.e. when using `collections.namedtuple()`). - annotations = {field_name: annotations.get(field_name, Any) for field_name in namedtuple_cls._fields} - - if typevars_map: - annotations = { - field_name: replace_types(annotation, typevars_map) - for field_name, annotation in annotations.items() - } - - arguments_schema = core_schema.arguments_schema( - [ - self._generate_parameter_schema( - field_name, - annotation, - source=AnnotationSource.NAMED_TUPLE, - default=namedtuple_cls._field_defaults.get(field_name, Parameter.empty), - ) - for field_name, annotation in annotations.items() - ], - metadata={'pydantic_js_prefer_positional_arguments': True}, - ) - schema = core_schema.call_schema(arguments_schema, namedtuple_cls, ref=namedtuple_ref) - return self.defs.create_definition_reference_schema(schema) - - def _generate_parameter_schema( - self, - name: str, - annotation: type[Any], - source: AnnotationSource, - default: Any = Parameter.empty, - mode: Literal['positional_only', 'positional_or_keyword', 'keyword_only'] | None = None, - ) -> core_schema.ArgumentsParameter: - """Generate the definition of a field in a namedtuple or a parameter in a function signature. - - This definition is meant to be used for the `'arguments'` core schema, which will be replaced - in V3 by the `'arguments-v3`'. - """ - FieldInfo = import_cached_field_info() - - if default is Parameter.empty: - field = FieldInfo.from_annotation(annotation, _source=source) - else: - field = FieldInfo.from_annotated_attribute(annotation, default, _source=source) - - assert field.annotation is not None, 'field.annotation should not be None when generating a schema' - update_field_from_config(self._config_wrapper, name, field) - - with self.field_name_stack.push(name): - schema = self._apply_annotations( - field.annotation, - [field], - # Because we pass `field` as metadata above (required for attributes relevant for - # JSON Scheme generation), we need to ignore the potential warnings about `FieldInfo` - # attributes that will not be used: - check_unsupported_field_info_attributes=False, - ) - - if not field.is_required(): - schema = wrap_default(field, schema) - - parameter_schema = core_schema.arguments_parameter( - name, - schema, - mode=mode, - alias=_convert_to_aliases(field.validation_alias), - ) - - return parameter_schema - - def _generate_parameter_v3_schema( - self, - name: str, - annotation: Any, - source: AnnotationSource, - mode: Literal[ - 'positional_only', - 'positional_or_keyword', - 'keyword_only', - 'var_args', - 'var_kwargs_uniform', - 'var_kwargs_unpacked_typed_dict', - ], - default: Any = Parameter.empty, - ) -> core_schema.ArgumentsV3Parameter: - """Generate the definition of a parameter in a function signature. - - This definition is meant to be used for the `'arguments-v3'` core schema, which will replace - the `'arguments`' schema in V3. - """ - FieldInfo = import_cached_field_info() - - if default is Parameter.empty: - field = FieldInfo.from_annotation(annotation, _source=source) - else: - field = FieldInfo.from_annotated_attribute(annotation, default, _source=source) - update_field_from_config(self._config_wrapper, name, field) - - with self.field_name_stack.push(name): - schema = self._apply_annotations( - field.annotation, - [field], - # Because we pass `field` as metadata above (required for attributes relevant for - # JSON Scheme generation), we need to ignore the potential warnings about `FieldInfo` - # attributes that will not be used: - check_unsupported_field_info_attributes=False, - ) - - if not field.is_required(): - schema = wrap_default(field, schema) - - parameter_schema = core_schema.arguments_v3_parameter( - name=name, - schema=schema, - mode=mode, - alias=_convert_to_aliases(field.validation_alias), - ) - - return parameter_schema - - def _tuple_schema(self, tuple_type: Any) -> core_schema.CoreSchema: - """Generate schema for a Tuple, e.g. `tuple[int, str]` or `tuple[int, ...]`.""" - # TODO: do we really need to resolve type vars here? - typevars_map = get_standard_typevars_map(tuple_type) - params = self._get_args_resolving_forward_refs(tuple_type) - - if typevars_map and params: - params = tuple(replace_types(param, typevars_map) for param in params) - - # NOTE: subtle difference: `tuple[()]` gives `params=()`, whereas `typing.Tuple[()]` gives `params=((),)` - # This is only true for <3.11, on Python 3.11+ `typing.Tuple[()]` gives `params=()` - if not params: - if tuple_type in TUPLE_TYPES: - return core_schema.tuple_schema([core_schema.any_schema()], variadic_item_index=0) - else: - # special case for `tuple[()]` which means `tuple[]` - an empty tuple - return core_schema.tuple_schema([]) - elif params[-1] is Ellipsis: - if len(params) == 2: - return core_schema.tuple_schema([self.generate_schema(params[0])], variadic_item_index=0) - else: - # TODO: something like https://github.com/pydantic/pydantic/issues/5952 - raise ValueError('Variable tuples can only have one type') - elif len(params) == 1 and params[0] == (): - # special case for `tuple[()]` which means `tuple[]` - an empty tuple - # NOTE: This conditional can be removed when we drop support for Python 3.10. - return core_schema.tuple_schema([]) - else: - return core_schema.tuple_schema([self.generate_schema(param) for param in params]) - - def _type_schema(self) -> core_schema.CoreSchema: - return core_schema.custom_error_schema( - core_schema.is_instance_schema(type), - custom_error_type='is_type', - custom_error_message='Input should be a type', - ) - - def _zoneinfo_schema(self) -> core_schema.CoreSchema: - """Generate schema for a zone_info.ZoneInfo object""" - from ._validators import validate_str_is_valid_iana_tz - - metadata = {'pydantic_js_functions': [lambda _1, _2: {'type': 'string', 'format': 'zoneinfo'}]} - return core_schema.no_info_plain_validator_function( - validate_str_is_valid_iana_tz, - serialization=core_schema.to_string_ser_schema(), - metadata=metadata, - ) - - def _union_is_subclass_schema(self, union_type: Any) -> core_schema.CoreSchema: - """Generate schema for `type[Union[X, ...]]`.""" - args = self._get_args_resolving_forward_refs(union_type, required=True) - return core_schema.union_schema([self.generate_schema(type[args]) for args in args]) - - def _subclass_schema(self, type_: Any) -> core_schema.CoreSchema: - """Generate schema for a type, e.g. `type[int]`.""" - type_param = self._get_first_arg_or_any(type_) - - # Assume `type[Annotated[, ...]]` is equivalent to `type[]`: - type_param = _typing_extra.annotated_type(type_param) or type_param - - if typing_objects.is_any(type_param): - return self._type_schema() - elif typing_objects.is_typealiastype(type_param): - return self.generate_schema(type[type_param.__value__]) - elif typing_objects.is_typevar(type_param): - if type_param.__bound__: - if is_union_origin(get_origin(type_param.__bound__)): - return self._union_is_subclass_schema(type_param.__bound__) - return core_schema.is_subclass_schema(type_param.__bound__) - elif type_param.__constraints__: - return core_schema.union_schema([self.generate_schema(type[c]) for c in type_param.__constraints__]) - else: - return self._type_schema() - elif is_union_origin(get_origin(type_param)): - return self._union_is_subclass_schema(type_param) - else: - if typing_objects.is_self(type_param): - type_param = self._resolve_self_type(type_param) - if _typing_extra.is_generic_alias(type_param): - raise PydanticUserError( - 'Subscripting `type[]` with an already parametrized type is not supported. ' - f'Instead of using type[{type_param!r}], use type[{_repr.display_as_type(get_origin(type_param))}].', - code=None, - ) - if not inspect.isclass(type_param): - # when using type[None], this doesn't type convert to type[NoneType], and None isn't a class - # so we handle it manually here - if type_param is None: - return core_schema.is_subclass_schema(_typing_extra.NoneType) - raise TypeError(f'Expected a class, got {type_param!r}') - return core_schema.is_subclass_schema(type_param) - - def _sequence_schema(self, items_type: Any) -> core_schema.CoreSchema: - """Generate schema for a Sequence, e.g. `Sequence[int]`.""" - from ._serializers import serialize_sequence_via_list - - item_type_schema = self.generate_schema(items_type) - list_schema = core_schema.list_schema(item_type_schema) - - json_schema = smart_deepcopy(list_schema) - python_schema = core_schema.is_instance_schema(typing.Sequence, cls_repr='Sequence') - if not typing_objects.is_any(items_type): - from ._validators import sequence_validator - - python_schema = core_schema.chain_schema( - [python_schema, core_schema.no_info_wrap_validator_function(sequence_validator, list_schema)], - ) - - serialization = core_schema.wrap_serializer_function_ser_schema( - serialize_sequence_via_list, schema=item_type_schema, info_arg=True - ) - return core_schema.json_or_python_schema( - json_schema=json_schema, python_schema=python_schema, serialization=serialization - ) - - def _iterable_schema(self, type_: Any) -> core_schema.GeneratorSchema: - """Generate a schema for an `Iterable`.""" - item_type = self._get_first_arg_or_any(type_) - - return core_schema.generator_schema(self.generate_schema(item_type)) - - def _pattern_schema(self, pattern_type: Any) -> core_schema.CoreSchema: - from . import _validators - - metadata = {'pydantic_js_functions': [lambda _1, _2: {'type': 'string', 'format': 'regex'}]} - ser = core_schema.plain_serializer_function_ser_schema( - attrgetter('pattern'), when_used='json', return_schema=core_schema.str_schema() - ) - if pattern_type is typing.Pattern or pattern_type is re.Pattern: - # bare type - return core_schema.no_info_plain_validator_function( - _validators.pattern_either_validator, serialization=ser, metadata=metadata - ) - - param = self._get_args_resolving_forward_refs( - pattern_type, - required=True, - )[0] - if param is str: - return core_schema.no_info_plain_validator_function( - _validators.pattern_str_validator, serialization=ser, metadata=metadata - ) - elif param is bytes: - return core_schema.no_info_plain_validator_function( - _validators.pattern_bytes_validator, serialization=ser, metadata=metadata - ) - else: - raise PydanticSchemaGenerationError(f'Unable to generate pydantic-core schema for {pattern_type!r}.') - - def _hashable_schema(self) -> core_schema.CoreSchema: - return core_schema.custom_error_schema( - schema=core_schema.json_or_python_schema( - json_schema=core_schema.chain_schema( - [core_schema.any_schema(), core_schema.is_instance_schema(collections.abc.Hashable)] - ), - python_schema=core_schema.is_instance_schema(collections.abc.Hashable), - ), - custom_error_type='is_hashable', - custom_error_message='Input should be hashable', - ) - - def _dataclass_schema( - self, dataclass: type[StandardDataclass], origin: type[StandardDataclass] | None - ) -> core_schema.CoreSchema: - """Generate schema for a dataclass.""" - with ( - self.model_type_stack.push(dataclass), - self.defs.get_schema_or_ref(dataclass) as ( - dataclass_ref, - maybe_schema, - ), - ): - if maybe_schema is not None: - return maybe_schema - - schema = dataclass.__dict__.get('__pydantic_core_schema__') - if schema is not None and not isinstance(schema, MockCoreSchema): - if schema['type'] == 'definitions': - schema = self.defs.unpack_definitions(schema) - ref = get_ref(schema) - if ref: - return self.defs.create_definition_reference_schema(schema) - else: - return schema - - typevars_map = get_standard_typevars_map(dataclass) - if origin is not None: - dataclass = origin - - # if (plain) dataclass doesn't have config, we use the parent's config, hence a default of `None` - # (Pydantic dataclasses have an empty dict config by default). - # see https://github.com/pydantic/pydantic/issues/10917 - config = getattr(dataclass, '__pydantic_config__', None) - - from ..dataclasses import is_pydantic_dataclass - - with self._ns_resolver.push(dataclass), self._config_wrapper_stack.push(config): - if is_pydantic_dataclass(dataclass): - if dataclass.__pydantic_fields_complete__(): - # Copy the field info instances to avoid mutating the `FieldInfo` instances - # of the generic dataclass generic origin (e.g. `apply_typevars_map` below). - # Note that we don't apply `deepcopy` on `__pydantic_fields__` because we - # don't want to copy the `FieldInfo` attributes: - fields = { - f_name: copy(field_info) for f_name, field_info in dataclass.__pydantic_fields__.items() - } - if typevars_map: - for field in fields.values(): - field.apply_typevars_map(typevars_map, *self._types_namespace) - else: - try: - fields = rebuild_dataclass_fields( - dataclass, - config_wrapper=self._config_wrapper, - ns_resolver=self._ns_resolver, - typevars_map=typevars_map or {}, - ) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - else: - fields = collect_dataclass_fields( - dataclass, - typevars_map=typevars_map, - config_wrapper=self._config_wrapper, - ) - - if self._config_wrapper.extra == 'allow': - # disallow combination of init=False on a dataclass field and extra='allow' on a dataclass - for field_name, field in fields.items(): - if field.init is False: - raise PydanticUserError( - f'Field {field_name} has `init=False` and dataclass has config setting `extra="allow"`. ' - f'This combination is not allowed.', - code='dataclass-init-false-extra-allow', - ) - - decorators = dataclass.__dict__.get('__pydantic_decorators__') - if decorators is None: - decorators = DecoratorInfos.build(dataclass, replace_wrapped_methods=False) - decorators.update_from_config(self._config_wrapper) - # Move kw_only=False args to the start of the list, as this is how vanilla dataclasses work. - # Note that when kw_only is missing or None, it is treated as equivalent to kw_only=True - args = sorted( - (self._generate_dc_field_schema(k, v, decorators) for k, v in fields.items()), - key=lambda a: a.get('kw_only') is not False, - ) - has_post_init = hasattr(dataclass, '__post_init__') - has_slots = hasattr(dataclass, '__slots__') - - args_schema = core_schema.dataclass_args_schema( - dataclass.__name__, - args, - computed_fields=[ - self._computed_field_schema(d, decorators.field_serializers) - for d in decorators.computed_fields.values() - ], - collect_init_only=has_post_init, - ) - - inner_schema = apply_validators(args_schema, decorators.root_validators.values()) - - model_validators = decorators.model_validators.values() - inner_schema = apply_model_validators(inner_schema, model_validators, 'inner') - - core_config = self._config_wrapper.core_config(title=dataclass.__name__) - - dc_schema = core_schema.dataclass_schema( - dataclass, - inner_schema, - generic_origin=origin, - post_init=has_post_init, - ref=dataclass_ref, - fields=[field.name for field in dataclasses.fields(dataclass)], - slots=has_slots, - config=core_config, - # we don't use a custom __setattr__ for dataclasses, so we must - # pass along the frozen config setting to the pydantic-core schema - frozen=self._config_wrapper_stack.tail.frozen, - ) - schema = self._apply_model_serializers(dc_schema, decorators.model_serializers.values()) - schema = apply_model_validators(schema, model_validators, 'outer') - return self.defs.create_definition_reference_schema(schema) - - def _call_schema(self, function: ValidateCallSupportedTypes) -> core_schema.CallSchema: - """Generate schema for a Callable. - - TODO support functional validators once we support them in Config - """ - arguments_schema = self._arguments_schema(function) - - return_schema: core_schema.CoreSchema | None = None - config_wrapper = self._config_wrapper - if config_wrapper.validate_return: - sig = _typing_extra.signature_no_eval(function) - return_hint = sig.return_annotation - if return_hint is not sig.empty: - globalns, localns = self._types_namespace - type_hints = _typing_extra.get_function_type_hints( - function, globalns=globalns, localns=localns, include_keys={'return'} - ) - return_schema = self.generate_schema(type_hints['return']) - - return core_schema.call_schema( - arguments_schema, - function, - return_schema=return_schema, - ) - - def _arguments_schema( - self, function: ValidateCallSupportedTypes, parameters_callback: ParametersCallback | None = None - ) -> core_schema.ArgumentsSchema: - """Generate schema for a Signature.""" - mode_lookup: dict[_ParameterKind, Literal['positional_only', 'positional_or_keyword', 'keyword_only']] = { - Parameter.POSITIONAL_ONLY: 'positional_only', - Parameter.POSITIONAL_OR_KEYWORD: 'positional_or_keyword', - Parameter.KEYWORD_ONLY: 'keyword_only', - } - - sig = _typing_extra.signature_no_eval(function) - globalns, localns = self._types_namespace - type_hints = _typing_extra.get_function_type_hints(function, globalns=globalns, localns=localns) - - arguments_list: list[core_schema.ArgumentsParameter] = [] - var_args_schema: core_schema.CoreSchema | None = None - var_kwargs_schema: core_schema.CoreSchema | None = None - var_kwargs_mode: core_schema.VarKwargsMode | None = None - - for i, (name, p) in enumerate(sig.parameters.items()): - if p.annotation is sig.empty: - annotation = typing.cast(Any, Any) - else: - annotation = type_hints[name] - - if parameters_callback is not None: - result = parameters_callback(i, name, annotation) - if result == 'skip': - continue - - parameter_mode = mode_lookup.get(p.kind) - if parameter_mode is not None: - arg_schema = self._generate_parameter_schema( - name, annotation, AnnotationSource.FUNCTION, p.default, parameter_mode - ) - arguments_list.append(arg_schema) - elif p.kind == Parameter.VAR_POSITIONAL: - var_args_schema = self.generate_schema(annotation) - else: - assert p.kind == Parameter.VAR_KEYWORD, p.kind - - unpack_type = _typing_extra.unpack_type(annotation) - if unpack_type is not None: - origin = get_origin(unpack_type) or unpack_type - if not is_typeddict(origin): - raise PydanticUserError( - f'Expected a `TypedDict` class inside `Unpack[...]`, got {unpack_type!r}', - code='unpack-typed-dict', - ) - non_pos_only_param_names = { - name for name, p in sig.parameters.items() if p.kind != Parameter.POSITIONAL_ONLY - } - overlapping_params = non_pos_only_param_names.intersection(origin.__annotations__) - if overlapping_params: - raise PydanticUserError( - f'Typed dictionary {origin.__name__!r} overlaps with parameter' - f'{"s" if len(overlapping_params) >= 2 else ""} ' - f'{", ".join(repr(p) for p in sorted(overlapping_params))}', - code='overlapping-unpack-typed-dict', - ) - - var_kwargs_mode = 'unpacked-typed-dict' - var_kwargs_schema = self._typed_dict_schema(unpack_type, get_origin(unpack_type)) - else: - var_kwargs_mode = 'uniform' - var_kwargs_schema = self.generate_schema(annotation) - - return core_schema.arguments_schema( - arguments_list, - var_args_schema=var_args_schema, - var_kwargs_mode=var_kwargs_mode, - var_kwargs_schema=var_kwargs_schema, - validate_by_name=self._config_wrapper.validate_by_name, - ) - - def _arguments_v3_schema( - self, function: ValidateCallSupportedTypes, parameters_callback: ParametersCallback | None = None - ) -> core_schema.ArgumentsV3Schema: - mode_lookup: dict[ - _ParameterKind, Literal['positional_only', 'positional_or_keyword', 'var_args', 'keyword_only'] - ] = { - Parameter.POSITIONAL_ONLY: 'positional_only', - Parameter.POSITIONAL_OR_KEYWORD: 'positional_or_keyword', - Parameter.VAR_POSITIONAL: 'var_args', - Parameter.KEYWORD_ONLY: 'keyword_only', - } - - sig = _typing_extra.signature_no_eval(function) - globalns, localns = self._types_namespace - type_hints = _typing_extra.get_function_type_hints(function, globalns=globalns, localns=localns) - - parameters_list: list[core_schema.ArgumentsV3Parameter] = [] - - for i, (name, p) in enumerate(sig.parameters.items()): - if parameters_callback is not None: - result = parameters_callback(i, name, p.annotation) - if result == 'skip': - continue - - if p.annotation is Parameter.empty: - annotation = typing.cast(Any, Any) - else: - annotation = type_hints[name] - - parameter_mode = mode_lookup.get(p.kind) - if parameter_mode is None: - assert p.kind == Parameter.VAR_KEYWORD, p.kind - - unpack_type = _typing_extra.unpack_type(annotation) - if unpack_type is not None: - origin = get_origin(unpack_type) or unpack_type - if not is_typeddict(origin): - raise PydanticUserError( - f'Expected a `TypedDict` class inside `Unpack[...]`, got {unpack_type!r}', - code='unpack-typed-dict', - ) - non_pos_only_param_names = { - name for name, p in sig.parameters.items() if p.kind != Parameter.POSITIONAL_ONLY - } - overlapping_params = non_pos_only_param_names.intersection(origin.__annotations__) - if overlapping_params: - raise PydanticUserError( - f'Typed dictionary {origin.__name__!r} overlaps with parameter' - f'{"s" if len(overlapping_params) >= 2 else ""} ' - f'{", ".join(repr(p) for p in sorted(overlapping_params))}', - code='overlapping-unpack-typed-dict', - ) - parameter_mode = 'var_kwargs_unpacked_typed_dict' - annotation = unpack_type - else: - parameter_mode = 'var_kwargs_uniform' - - parameters_list.append( - self._generate_parameter_v3_schema( - name, annotation, AnnotationSource.FUNCTION, parameter_mode, default=p.default - ) - ) - - return core_schema.arguments_v3_schema( - parameters_list, - validate_by_name=self._config_wrapper.validate_by_name, - ) - - def _unsubstituted_typevar_schema(self, typevar: typing.TypeVar) -> core_schema.CoreSchema: - try: - has_default = typevar.has_default() # pyright: ignore[reportAttributeAccessIssue] - except AttributeError: - # Happens if using `typing.TypeVar` (and not `typing_extensions`) on Python < 3.13 - pass - else: - if has_default: - return self.generate_schema(typevar.__default__) # pyright: ignore[reportAttributeAccessIssue] - - if constraints := typevar.__constraints__: - return self._union_schema(typing.Union[constraints]) - - if bound := typevar.__bound__: - schema = self.generate_schema(bound) - schema['serialization'] = core_schema.simple_ser_schema('any') - return schema - - return core_schema.any_schema() - - def _computed_field_schema( - self, - d: Decorator[ComputedFieldInfo], - field_serializers: dict[str, Decorator[FieldSerializerDecoratorInfo]], - ) -> core_schema.ComputedField: - if d.info.return_type is not PydanticUndefined: - return_type = d.info.return_type - else: - try: - # Do not pass in globals as the function could be defined in a different module. - # Instead, let `get_callable_return_type` infer the globals to use, but still pass - # in locals that may contain a parent/rebuild namespace: - return_type = _decorators.get_callable_return_type(d.func, localns=self._types_namespace.locals) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - if return_type is PydanticUndefined: - raise PydanticUserError( - 'Computed field is missing return type annotation or specifying `return_type`' - ' to the `@computed_field` decorator (e.g. `@computed_field(return_type=int | str)`)', - code='model-field-missing-annotation', - ) - - return_type = replace_types(return_type, self._typevars_map) - # Create a new ComputedFieldInfo so that different type parametrizations of the same - # generic model's computed field can have different return types. - d.info = dataclasses.replace(d.info, return_type=return_type) - return_type_schema = self.generate_schema(return_type) - # Apply serializers to computed field if there exist - return_type_schema = self._apply_field_serializers( - return_type_schema, - filter_field_decorator_info_by_field(field_serializers.values(), d.cls_var_name), - ) - - pydantic_js_updates, pydantic_js_extra = _extract_json_schema_info_from_field_info(d.info) - core_metadata: dict[str, Any] = {} - update_core_metadata( - core_metadata, - pydantic_js_updates={'readOnly': True, **(pydantic_js_updates if pydantic_js_updates else {})}, - pydantic_js_extra=pydantic_js_extra, - ) - exclude_if = d.info.exclude_if - # TODO: Should we support exclude_if from annotations? - return core_schema.computed_field( - d.cls_var_name, - return_schema=return_type_schema, - alias=d.info.alias, - serialization_exclude_if=exclude_if, - metadata=core_metadata, - ) - - def _annotated_schema(self, annotated_type: Any) -> core_schema.CoreSchema: - """Generate schema for an Annotated type, e.g. `Annotated[int, Field(...)]` or `Annotated[int, Gt(0)]`.""" - FieldInfo = import_cached_field_info() - source_type, *annotations = self._get_args_resolving_forward_refs( - annotated_type, - required=True, - ) - schema = self._apply_annotations(source_type, annotations) - # put the default validator last so that TypeAdapter.get_default_value() works - # even if there are function validators involved - for annotation in annotations: - if isinstance(annotation, FieldInfo): - schema = wrap_default(annotation, schema) - return schema - - def _apply_annotations( - self, - source_type: Any, - annotations: list[Any], - transform_inner_schema: Callable[[CoreSchema], CoreSchema] = lambda x: x, - check_unsupported_field_info_attributes: bool = True, - ) -> CoreSchema: - """Apply arguments from `Annotated` or from `FieldInfo` to a schema. - - This gets called by `GenerateSchema._annotated_schema` but differs from it in that it does - not expect `source_type` to be an `Annotated` object, it expects it to be the first argument of that - (in other words, `GenerateSchema._annotated_schema` just unpacks `Annotated`, this process it). - """ - annotations = list(_known_annotated_metadata.expand_grouped_metadata(annotations)) - - pydantic_js_annotation_functions: list[GetJsonSchemaFunction] = [] - - def inner_handler(obj: Any) -> CoreSchema: - schema = self._generate_schema_from_get_schema_method(obj, source_type) - - if schema is None: - schema = self._generate_schema_inner(obj) - - metadata_js_function = _extract_get_pydantic_json_schema(obj) - if metadata_js_function is not None: - metadata_schema = resolve_original_schema(schema, self.defs) - if metadata_schema is not None: - self._add_js_function(metadata_schema, metadata_js_function) - return transform_inner_schema(schema) - - get_inner_schema = CallbackGetCoreSchemaHandler(inner_handler, self) - - for annotation in annotations: - if annotation is None: - continue - get_inner_schema = self._get_wrapped_inner_schema( - get_inner_schema, - annotation, - pydantic_js_annotation_functions, - check_unsupported_field_info_attributes=check_unsupported_field_info_attributes, - ) - - schema = get_inner_schema(source_type) - if pydantic_js_annotation_functions: - core_metadata = schema.setdefault('metadata', {}) - update_core_metadata(core_metadata, pydantic_js_annotation_functions=pydantic_js_annotation_functions) - return _add_custom_serialization_from_json_encoders(self._config_wrapper.json_encoders, source_type, schema) - - def _apply_single_annotation( - self, - schema: core_schema.CoreSchema, - metadata: Any, - check_unsupported_field_info_attributes: bool = True, - ) -> core_schema.CoreSchema: - FieldInfo = import_cached_field_info() - - if isinstance(metadata, FieldInfo): - if ( - check_unsupported_field_info_attributes - # HACK: we don't want to emit the warning for `FieldInfo` subclasses, because FastAPI does weird manipulations - # with its subclasses and their annotations: - and type(metadata) is FieldInfo - ): - for attr, value in (unsupported_attributes := self._get_unsupported_field_info_attributes(metadata)): - warnings.warn( - f'The {attr!r} attribute with value {value!r} was provided to the `Field()` function, ' - f'which has no effect in the context it was used. {attr!r} is field-specific metadata, ' - 'and can only be attached to a model field using `Annotated` metadata or by assignment. ' - 'This may have happened because an `Annotated` type alias using the `type` statement was ' - 'used, or if the `Field()` function was attached to a single member of a union type.', - category=UnsupportedFieldAttributeWarning, - ) - - if ( - metadata.default_factory_takes_validated_data - and self.model_type_stack.get() is None - and 'defaut_factory' not in unsupported_attributes - ): - warnings.warn( - "A 'default_factory' taking validated data as an argument was provided to the `Field()` function, " - 'but no validated data is available in the context it was used.', - category=UnsupportedFieldAttributeWarning, - ) - - for field_metadata in metadata.metadata: - schema = self._apply_single_annotation(schema, field_metadata) - - if metadata.discriminator is not None: - schema = self._apply_discriminator_to_union(schema, metadata.discriminator) - return schema - - if schema['type'] == 'nullable': - # for nullable schemas, metadata is automatically applied to the inner schema - inner = schema.get('schema', core_schema.any_schema()) - inner = self._apply_single_annotation(inner, metadata) - if inner: - schema['schema'] = inner - return schema - - if schema['type'] == 'union' and any( - choice['type'] == 'missing-sentinel' for choice in core_schema.iter_union_choices(schema) - ): - # Same behavior as for nullable schemas. This is a bit gross, but we have to support the same pattern - filtered_choices = [ - choice - for choice in schema['choices'] - if (choice[0] if isinstance(choice, tuple) else choice)['type'] != 'missing-sentinel' - ] - if len(filtered_choices) >= 2: - # e.g. `Annotated[int | str | MISSING, Constraint(...)]`. We apply `Constraint(...)` to `int | str`, - # and create a new union semantically equivalent to `Annotated[int | str, Constraint(...)] | MISSING`: - filtered_union = core_schema.union_schema(filtered_choices) - filtered_union = self._apply_single_annotation(filtered_union, metadata) - new_union = schema.copy() - new_union['choices'] = [ - filtered_union, - next( - choice - for choice in schema['choices'] - if (choice[0] if isinstance(choice, tuple) else choice)['type'] == 'missing-sentinel' - ), - ] - return new_union - elif len(filtered_choices) == 1: - # e.g. `Annotated[int | MISSING, Constraint(...)]`. We apply `Constraint(...)` to `int`, and reconstruct - # a new union preserving the order. - inner = filtered_choices[0][0] if isinstance(filtered_choices[0], tuple) else filtered_choices[0] - inner = self._apply_single_annotation(inner, metadata) - - # Create a new union schema, preserving the order of the union: - new_union = schema.copy() - new_union['choices'] = [ - (inner, choice[1]) - if isinstance(choice, tuple) and choice[0]['type'] != 'missing-sentinel' - else inner - if not isinstance(choice, tuple) and choice['type'] != 'missing-sentinel' - else choice - for choice in schema['choices'] - ] - return new_union - - original_schema = schema - ref = schema.get('ref') - if ref is not None: - schema = schema.copy() - new_ref = ref + f'_{repr(metadata)}' - if (existing := self.defs.get_schema_from_ref(new_ref)) is not None: - return existing - schema['ref'] = new_ref # pyright: ignore[reportGeneralTypeIssues] - elif schema['type'] == 'definition-ref': - ref = schema['schema_ref'] - if (referenced_schema := self.defs.get_schema_from_ref(ref)) is not None: - schema = referenced_schema.copy() - new_ref = ref + f'_{repr(metadata)}' - if (existing := self.defs.get_schema_from_ref(new_ref)) is not None: - return existing - schema['ref'] = new_ref # pyright: ignore[reportGeneralTypeIssues] - - maybe_updated_schema = _known_annotated_metadata.apply_known_metadata(metadata, schema) - - if maybe_updated_schema is not None: - return maybe_updated_schema - return original_schema - - def _apply_single_annotation_json_schema( - self, schema: core_schema.CoreSchema, metadata: Any - ) -> core_schema.CoreSchema: - FieldInfo = import_cached_field_info() - - if isinstance(metadata, FieldInfo): - for field_metadata in metadata.metadata: - schema = self._apply_single_annotation_json_schema(schema, field_metadata) - - pydantic_js_updates, pydantic_js_extra = _extract_json_schema_info_from_field_info(metadata) - core_metadata = schema.setdefault('metadata', {}) - update_core_metadata( - core_metadata, pydantic_js_updates=pydantic_js_updates, pydantic_js_extra=pydantic_js_extra - ) - return schema - - def _get_unsupported_field_info_attributes(self, field_info: FieldInfo) -> list[tuple[str, Any]]: - """Get the list of unsupported `FieldInfo` attributes when not directly used in `Annotated` for field annotations.""" - unused_metadata: list[tuple[str, Any]] = [] - for unused_metadata_name, unset_value in UNSUPPORTED_STANDALONE_FIELDINFO_ATTRIBUTES: - if ( - (unused_metadata_value := getattr(field_info, unused_metadata_name)) is not unset_value - # `default` and `default_factory` can still be used with a type adapter, so only include them - # if used with a model-like class: - and ( - unused_metadata_name not in ('default', 'default_factory') - or self.model_type_stack.get() is not None - ) - # Setting `alias` will set `validation/serialization_alias` as well, so we want to avoid duplicate warnings: - and ( - unused_metadata_name not in ('validation_alias', 'serialization_alias') - or 'alias' not in field_info._attributes_set - ) - ): - unused_metadata.append((unused_metadata_name, unused_metadata_value)) - - return unused_metadata - - def _get_wrapped_inner_schema( - self, - get_inner_schema: GetCoreSchemaHandler, - annotation: Any, - pydantic_js_annotation_functions: list[GetJsonSchemaFunction], - check_unsupported_field_info_attributes: bool = False, - ) -> CallbackGetCoreSchemaHandler: - annotation_get_schema: GetCoreSchemaFunction | None = getattr(annotation, '__get_pydantic_core_schema__', None) - - def new_handler(source: Any) -> core_schema.CoreSchema: - if annotation_get_schema is not None: - schema = annotation_get_schema(source, get_inner_schema) - else: - schema = get_inner_schema(source) - schema = self._apply_single_annotation( - schema, - annotation, - check_unsupported_field_info_attributes=check_unsupported_field_info_attributes, - ) - schema = self._apply_single_annotation_json_schema(schema, annotation) - - metadata_js_function = _extract_get_pydantic_json_schema(annotation) - if metadata_js_function is not None: - pydantic_js_annotation_functions.append(metadata_js_function) - return schema - - return CallbackGetCoreSchemaHandler(new_handler, self) - - def _apply_field_serializers( - self, - schema: core_schema.CoreSchema, - serializers: list[Decorator[FieldSerializerDecoratorInfo]], - ) -> core_schema.CoreSchema: - """Apply field serializers to a schema.""" - if serializers: - schema = copy(schema) - if schema['type'] == 'definitions': - inner_schema = schema['schema'] - schema['schema'] = self._apply_field_serializers(inner_schema, serializers) - return schema - elif 'ref' in schema: - schema = self.defs.create_definition_reference_schema(schema) - - # use the last serializer to make it easy to override a serializer set on a parent model - serializer = serializers[-1] - is_field_serializer, info_arg = inspect_field_serializer(serializer.func, serializer.info.mode) - - if serializer.info.return_type is not PydanticUndefined: - return_type = serializer.info.return_type - else: - try: - # Do not pass in globals as the function could be defined in a different module. - # Instead, let `get_callable_return_type` infer the globals to use, but still pass - # in locals that may contain a parent/rebuild namespace: - return_type = _decorators.get_callable_return_type( - serializer.func, localns=self._types_namespace.locals - ) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - if return_type is PydanticUndefined: - return_schema = None - else: - return_schema = self.generate_schema(return_type) - - if serializer.info.mode == 'wrap': - schema['serialization'] = core_schema.wrap_serializer_function_ser_schema( - serializer.func, - is_field_serializer=is_field_serializer, - info_arg=info_arg, - return_schema=return_schema, - when_used=serializer.info.when_used, - ) - else: - assert serializer.info.mode == 'plain' - schema['serialization'] = core_schema.plain_serializer_function_ser_schema( - serializer.func, - is_field_serializer=is_field_serializer, - info_arg=info_arg, - return_schema=return_schema, - when_used=serializer.info.when_used, - ) - return schema - - def _apply_model_serializers( - self, schema: core_schema.CoreSchema, serializers: Iterable[Decorator[ModelSerializerDecoratorInfo]] - ) -> core_schema.CoreSchema: - """Apply model serializers to a schema.""" - ref: str | None = schema.pop('ref', None) # type: ignore - if serializers: - serializer = list(serializers)[-1] - info_arg = inspect_model_serializer(serializer.func, serializer.info.mode) - - if serializer.info.return_type is not PydanticUndefined: - return_type = serializer.info.return_type - else: - try: - # Do not pass in globals as the function could be defined in a different module. - # Instead, let `get_callable_return_type` infer the globals to use, but still pass - # in locals that may contain a parent/rebuild namespace: - return_type = _decorators.get_callable_return_type( - serializer.func, localns=self._types_namespace.locals - ) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - if return_type is PydanticUndefined: - return_schema = None - else: - return_schema = self.generate_schema(return_type) - - if serializer.info.mode == 'wrap': - ser_schema: core_schema.SerSchema = core_schema.wrap_serializer_function_ser_schema( - serializer.func, - info_arg=info_arg, - return_schema=return_schema, - when_used=serializer.info.when_used, - ) - else: - # plain - ser_schema = core_schema.plain_serializer_function_ser_schema( - serializer.func, - info_arg=info_arg, - return_schema=return_schema, - when_used=serializer.info.when_used, - ) - schema['serialization'] = ser_schema - if ref: - schema['ref'] = ref # type: ignore - return schema - - -_VALIDATOR_F_MATCH: Mapping[ - tuple[FieldValidatorModes, Literal['no-info', 'with-info']], - Callable[[Callable[..., Any], core_schema.CoreSchema], core_schema.CoreSchema], -] = { - ('before', 'no-info'): lambda f, schema: core_schema.no_info_before_validator_function(f, schema), - ('after', 'no-info'): lambda f, schema: core_schema.no_info_after_validator_function(f, schema), - ('plain', 'no-info'): lambda f, _: core_schema.no_info_plain_validator_function(f), - ('wrap', 'no-info'): lambda f, schema: core_schema.no_info_wrap_validator_function(f, schema), - ('before', 'with-info'): lambda f, schema: core_schema.with_info_before_validator_function(f, schema), - ('after', 'with-info'): lambda f, schema: core_schema.with_info_after_validator_function(f, schema), - ('plain', 'with-info'): lambda f, _: core_schema.with_info_plain_validator_function(f), - ('wrap', 'with-info'): lambda f, schema: core_schema.with_info_wrap_validator_function(f, schema), -} - - -# TODO V3: this function is only used for deprecated decorators. It should -# be removed once we drop support for those. -def apply_validators( - schema: core_schema.CoreSchema, - validators: Iterable[Decorator[RootValidatorDecoratorInfo]] - | Iterable[Decorator[ValidatorDecoratorInfo]] - | Iterable[Decorator[FieldValidatorDecoratorInfo]], -) -> core_schema.CoreSchema: - """Apply validators to a schema. - - Args: - schema: The schema to apply validators on. - validators: An iterable of validators. - field_name: The name of the field if validators are being applied to a model field. - - Returns: - The updated schema. - """ - for validator in validators: - # Actually, type could be 'field' or 'model', but this is only used for deprecated - # decorators, so let's not worry about it. - info_arg = inspect_validator(validator.func, mode=validator.info.mode, type='field') - val_type = 'with-info' if info_arg else 'no-info' - - schema = _VALIDATOR_F_MATCH[(validator.info.mode, val_type)](validator.func, schema) - return schema - - -def _validators_require_validate_default(validators: Iterable[Decorator[ValidatorDecoratorInfo]]) -> bool: - """In v1, if any of the validators for a field had `always=True`, the default value would be validated. - - This serves as an auxiliary function for re-implementing that logic, by looping over a provided - collection of (v1-style) ValidatorDecoratorInfo's and checking if any of them have `always=True`. - - We should be able to drop this function and the associated logic calling it once we drop support - for v1-style validator decorators. (Or we can extend it and keep it if we add something equivalent - to the v1-validator `always` kwarg to `field_validator`.) - """ - for validator in validators: - if validator.info.always: - return True - return False - - -def _convert_to_aliases( - alias: str | AliasChoices | AliasPath | None, -) -> str | list[str | int] | list[list[str | int]] | None: - if isinstance(alias, (AliasChoices, AliasPath)): - return alias.convert_to_aliases() - else: - return alias - - -def apply_model_validators( - schema: core_schema.CoreSchema, - validators: Iterable[Decorator[ModelValidatorDecoratorInfo]], - mode: Literal['inner', 'outer', 'all'], -) -> core_schema.CoreSchema: - """Apply model validators to a schema. - - If mode == 'inner', only "before" validators are applied - If mode == 'outer', validators other than "before" are applied - If mode == 'all', all validators are applied - - Args: - schema: The schema to apply validators on. - validators: An iterable of validators. - mode: The validator mode. - - Returns: - The updated schema. - """ - ref: str | None = schema.pop('ref', None) # type: ignore - for validator in validators: - if mode == 'inner' and validator.info.mode != 'before': - continue - if mode == 'outer' and validator.info.mode == 'before': - continue - info_arg = inspect_validator(validator.func, mode=validator.info.mode, type='model') - if validator.info.mode == 'wrap': - if info_arg: - schema = core_schema.with_info_wrap_validator_function(function=validator.func, schema=schema) - else: - schema = core_schema.no_info_wrap_validator_function(function=validator.func, schema=schema) - elif validator.info.mode == 'before': - if info_arg: - schema = core_schema.with_info_before_validator_function(function=validator.func, schema=schema) - else: - schema = core_schema.no_info_before_validator_function(function=validator.func, schema=schema) - else: - assert validator.info.mode == 'after' - if info_arg: - schema = core_schema.with_info_after_validator_function(function=validator.func, schema=schema) - else: - schema = core_schema.no_info_after_validator_function(function=validator.func, schema=schema) - if ref: - schema['ref'] = ref # type: ignore - return schema - - -def wrap_default(field_info: FieldInfo, schema: core_schema.CoreSchema) -> core_schema.CoreSchema: - """Wrap schema with default schema if default value or `default_factory` are available. - - Args: - field_info: The field info object. - schema: The schema to apply default on. - - Returns: - Updated schema by default value or `default_factory`. - """ - if field_info.default_factory: - return core_schema.with_default_schema( - schema, - default_factory=field_info.default_factory, - default_factory_takes_data=takes_validated_data_argument(field_info.default_factory), - validate_default=field_info.validate_default, - ) - elif field_info.default is not PydanticUndefined: - return core_schema.with_default_schema( - schema, default=field_info.default, validate_default=field_info.validate_default - ) - else: - return schema - - -def _extract_get_pydantic_json_schema(tp: Any) -> GetJsonSchemaFunction | None: - """Extract `__get_pydantic_json_schema__` from a type, handling the deprecated `__modify_schema__`.""" - js_modify_function = getattr(tp, '__get_pydantic_json_schema__', None) - - if hasattr(tp, '__modify_schema__'): - BaseModel = import_cached_base_model() - - has_custom_v2_modify_js_func = ( - js_modify_function is not None - and BaseModel.__get_pydantic_json_schema__.__func__ # type: ignore - not in (js_modify_function, getattr(js_modify_function, '__func__', None)) - ) - - if not has_custom_v2_modify_js_func: - cls_name = getattr(tp, '__name__', None) - raise PydanticUserError( - f'The `__modify_schema__` method is not supported in Pydantic v2. ' - f'Use `__get_pydantic_json_schema__` instead{f" in class `{cls_name}`" if cls_name else ""}.', - code='custom-json-schema', - ) - - if (origin := get_origin(tp)) is not None: - # Generic aliases proxy attribute access to the origin, *except* dunder attributes, - # such as `__get_pydantic_json_schema__`, hence the explicit check. - return _extract_get_pydantic_json_schema(origin) - - if js_modify_function is None: - return None - - return js_modify_function - - -def resolve_original_schema(schema: CoreSchema, definitions: _Definitions) -> CoreSchema | None: - if schema['type'] == 'definition-ref': - return definitions.get_schema_from_ref(schema['schema_ref']) - elif schema['type'] == 'definitions': - return schema['schema'] - else: - return schema - - -def _inlining_behavior( - def_ref: core_schema.DefinitionReferenceSchema, -) -> Literal['inline', 'keep', 'preserve_metadata']: - """Determine the inlining behavior of the `'definition-ref'` schema. - - - If no `'serialization'` schema and no metadata is attached, the schema can safely be inlined. - - If it has metadata but only related to the deferred discriminator application, it can be inlined - provided that such metadata is kept. - - Otherwise, the schema should not be inlined. Doing so would remove the `'serialization'` schema or metadata. - """ - if 'serialization' in def_ref: - return 'keep' - metadata = def_ref.get('metadata') - if not metadata: - return 'inline' - if len(metadata) == 1 and 'pydantic_internal_union_discriminator' in metadata: - return 'preserve_metadata' - return 'keep' - - -class _Definitions: - """Keeps track of references and definitions.""" - - _recursively_seen: set[str] - """A set of recursively seen references. - - When a referenceable type is encountered, the `get_schema_or_ref` context manager is - entered to compute the reference. If the type references itself by some way (e.g. for - a dataclass a Pydantic model, the class can be referenced as a field annotation), - entering the context manager again will yield a `'definition-ref'` schema that should - short-circuit the normal generation process, as the reference was already in this set. - """ - - _definitions: dict[str, core_schema.CoreSchema] - """A mapping of references to their corresponding schema. - - When a schema for a referenceable type is generated, it is stored in this mapping. If the - same type is encountered again, the reference is yielded by the `get_schema_or_ref` context - manager. - """ - - def __init__(self) -> None: - self._recursively_seen = set() - self._definitions = {} - - @contextmanager - def get_schema_or_ref(self, tp: Any, /) -> Generator[tuple[str, core_schema.DefinitionReferenceSchema | None]]: - """Get a definition for `tp` if one exists. - - If a definition exists, a tuple of `(ref_string, CoreSchema)` is returned. - If no definition exists yet, a tuple of `(ref_string, None)` is returned. - - Note that the returned `CoreSchema` will always be a `DefinitionReferenceSchema`, - not the actual definition itself. - - This should be called for any type that can be identified by reference. - This includes any recursive types. - - At present the following types can be named/recursive: - - - Pydantic model - - Pydantic and stdlib dataclasses - - Typed dictionaries - - Named tuples - - `TypeAliasType` instances - - Enums - """ - ref = get_type_ref(tp) - # return the reference if we're either (1) in a cycle or (2) it the reference was already encountered: - if ref in self._recursively_seen or ref in self._definitions: - yield (ref, core_schema.definition_reference_schema(ref)) - else: - self._recursively_seen.add(ref) - try: - yield (ref, None) - finally: - self._recursively_seen.discard(ref) - - def get_schema_from_ref(self, ref: str) -> CoreSchema | None: - """Resolve the schema from the given reference.""" - return self._definitions.get(ref) - - def create_definition_reference_schema(self, schema: CoreSchema) -> core_schema.DefinitionReferenceSchema: - """Store the schema as a definition and return a `'definition-reference'` schema pointing to it. - - The schema must have a reference attached to it. - """ - ref = schema['ref'] # pyright: ignore - self._definitions[ref] = schema - return core_schema.definition_reference_schema(ref) - - def unpack_definitions(self, schema: core_schema.DefinitionsSchema) -> CoreSchema: - """Store the definitions of the `'definitions'` core schema and return the inner core schema.""" - for def_schema in schema['definitions']: - self._definitions[def_schema['ref']] = def_schema # pyright: ignore - return schema['schema'] - - def finalize_schema(self, schema: CoreSchema) -> CoreSchema: - """Finalize the core schema. - - This traverses the core schema and referenced definitions, replaces `'definition-ref'` schemas - by the referenced definition if possible, and applies deferred discriminators. - """ - definitions = self._definitions - try: - gather_result = gather_schemas_for_cleaning( - schema, - definitions=definitions, - ) - except MissingDefinitionError as e: - raise InvalidSchemaError from e - - remaining_defs: dict[str, CoreSchema] = {} - - # Note: this logic doesn't play well when core schemas with deferred discriminator metadata - # and references are encountered. See the `test_deferred_discriminated_union_and_references()` test. - for ref, inlinable_def_ref in gather_result['collected_references'].items(): - if inlinable_def_ref is not None and (inlining_behavior := _inlining_behavior(inlinable_def_ref)) != 'keep': - if inlining_behavior == 'inline': - # `ref` was encountered, and only once: - # - `inlinable_def_ref` is a `'definition-ref'` schema and is guaranteed to be - # the only one. Transform it into the definition it points to. - # - Do not store the definition in the `remaining_defs`. - inlinable_def_ref.clear() # pyright: ignore[reportAttributeAccessIssue] - inlinable_def_ref.update(self._resolve_definition(ref, definitions)) # pyright: ignore - elif inlining_behavior == 'preserve_metadata': - # `ref` was encountered, and only once, but contains discriminator metadata. - # We will do the same thing as if `inlining_behavior` was `'inline'`, but make - # sure to keep the metadata for the deferred discriminator application logic below. - meta = inlinable_def_ref.pop('metadata') - inlinable_def_ref.clear() # pyright: ignore[reportAttributeAccessIssue] - inlinable_def_ref.update(self._resolve_definition(ref, definitions)) # pyright: ignore - inlinable_def_ref['metadata'] = meta - else: - # `ref` was encountered, at least two times (or only once, but with metadata or a serialization schema): - # - Do not inline the `'definition-ref'` schemas (they are not provided in the gather result anyway). - # - Store the definition in the `remaining_defs` - remaining_defs[ref] = self._resolve_definition(ref, definitions) - - for cs in gather_result['deferred_discriminator_schemas']: - discriminator: str | None = cs['metadata'].pop('pydantic_internal_union_discriminator', None) # pyright: ignore[reportTypedDictNotRequiredAccess] - if discriminator is None: - # This can happen in rare scenarios, when a deferred schema is present multiple times in the - # gather result (e.g. when using the `Sequence` type -- see `test_sequence_discriminated_union()`). - # In this case, a previous loop iteration applied the discriminator and so we can just skip it here. - continue - applied = _discriminated_union.apply_discriminator(cs.copy(), discriminator, remaining_defs) - # Mutate the schema directly to have the discriminator applied - cs.clear() # pyright: ignore[reportAttributeAccessIssue] - cs.update(applied) # pyright: ignore - - if remaining_defs: - schema = core_schema.definitions_schema(schema=schema, definitions=[*remaining_defs.values()]) - return schema - - def _resolve_definition(self, ref: str, definitions: dict[str, CoreSchema]) -> CoreSchema: - definition = definitions[ref] - if definition['type'] != 'definition-ref': - return definition - - # Some `'definition-ref'` schemas might act as "intermediate" references (e.g. when using - # a PEP 695 type alias (which is referenceable) that references another PEP 695 type alias): - visited: set[str] = set() - while definition['type'] == 'definition-ref' and _inlining_behavior(definition) == 'inline': - schema_ref = definition['schema_ref'] - if schema_ref in visited: - raise PydanticUserError( - f'{ref} contains a circular reference to itself.', code='circular-reference-schema' - ) - visited.add(schema_ref) - definition = definitions[schema_ref] - return {**definition, 'ref': ref} # pyright: ignore[reportReturnType] - - -class _FieldNameStack: - __slots__ = ('_stack',) - - def __init__(self) -> None: - self._stack: list[str] = [] - - @contextmanager - def push(self, field_name: str) -> Iterator[None]: - self._stack.append(field_name) - yield - self._stack.pop() - - def get(self) -> str | None: - if self._stack: - return self._stack[-1] - else: - return None - - -class _ModelTypeStack: - __slots__ = ('_stack',) - - def __init__(self) -> None: - self._stack: list[type] = [] - - @contextmanager - def push(self, type_obj: type) -> Iterator[None]: - self._stack.append(type_obj) - yield - self._stack.pop() - - def get(self) -> type | None: - if self._stack: - return self._stack[-1] - else: - return None diff --git a/functions/python/data-processor/pydantic/_internal/_generics.py b/functions/python/data-processor/pydantic/_internal/_generics.py deleted file mode 100644 index f3e927cb0..000000000 --- a/functions/python/data-processor/pydantic/_internal/_generics.py +++ /dev/null @@ -1,530 +0,0 @@ -from __future__ import annotations - -import operator -import sys -import types -import typing -from collections import ChainMap -from collections.abc import Iterator, Mapping -from contextlib import contextmanager -from contextvars import ContextVar -from functools import reduce -from itertools import zip_longest -from types import prepare_class -from typing import TYPE_CHECKING, Annotated, Any, TypedDict, TypeVar, cast -from weakref import WeakValueDictionary - -import typing_extensions -from typing_inspection import typing_objects -from typing_inspection.introspection import is_union_origin - -from . import _typing_extra -from ._core_utils import get_type_ref -from ._forward_ref import PydanticRecursiveRef -from ._utils import all_identical, is_model_class - -if TYPE_CHECKING: - from ..main import BaseModel - -GenericTypesCacheKey = tuple[Any, Any, tuple[Any, ...]] - -# Note: We want to remove LimitedDict, but to do this, we'd need to improve the handling of generics caching. -# Right now, to handle recursive generics, we some types must remain cached for brief periods without references. -# By chaining the WeakValuesDict with a LimitedDict, we have a way to retain caching for all types with references, -# while also retaining a limited number of types even without references. This is generally enough to build -# specific recursive generic models without losing required items out of the cache. - -KT = TypeVar('KT') -VT = TypeVar('VT') -_LIMITED_DICT_SIZE = 100 - - -class LimitedDict(dict[KT, VT]): - def __init__(self, size_limit: int = _LIMITED_DICT_SIZE) -> None: - self.size_limit = size_limit - super().__init__() - - def __setitem__(self, key: KT, value: VT, /) -> None: - super().__setitem__(key, value) - if len(self) > self.size_limit: - excess = len(self) - self.size_limit + self.size_limit // 10 - to_remove = list(self.keys())[:excess] - for k in to_remove: - del self[k] - - -# weak dictionaries allow the dynamically created parametrized versions of generic models to get collected -# once they are no longer referenced by the caller. -GenericTypesCache = WeakValueDictionary[GenericTypesCacheKey, 'type[BaseModel]'] - -if TYPE_CHECKING: - - class DeepChainMap(ChainMap[KT, VT]): # type: ignore - ... - -else: - - class DeepChainMap(ChainMap): - """Variant of ChainMap that allows direct updates to inner scopes. - - Taken from https://docs.python.org/3/library/collections.html#collections.ChainMap, - with some light modifications for this use case. - """ - - def clear(self) -> None: - for mapping in self.maps: - mapping.clear() - - def __setitem__(self, key: KT, value: VT) -> None: - for mapping in self.maps: - mapping[key] = value - - def __delitem__(self, key: KT) -> None: - hit = False - for mapping in self.maps: - if key in mapping: - del mapping[key] - hit = True - if not hit: - raise KeyError(key) - - -# Despite the fact that LimitedDict _seems_ no longer necessary, I'm very nervous to actually remove it -# and discover later on that we need to re-add all this infrastructure... -# _GENERIC_TYPES_CACHE = DeepChainMap(GenericTypesCache(), LimitedDict()) - -_GENERIC_TYPES_CACHE = GenericTypesCache() - - -class PydanticGenericMetadata(TypedDict): - origin: type[BaseModel] | None # analogous to typing._GenericAlias.__origin__ - args: tuple[Any, ...] # analogous to typing._GenericAlias.__args__ - parameters: tuple[TypeVar, ...] # analogous to typing.Generic.__parameters__ - - -def create_generic_submodel( - model_name: str, origin: type[BaseModel], args: tuple[Any, ...], params: tuple[Any, ...] -) -> type[BaseModel]: - """Dynamically create a submodel of a provided (generic) BaseModel. - - This is used when producing concrete parametrizations of generic models. This function - only *creates* the new subclass; the schema/validators/serialization must be updated to - reflect a concrete parametrization elsewhere. - - Args: - model_name: The name of the newly created model. - origin: The base class for the new model to inherit from. - args: A tuple of generic metadata arguments. - params: A tuple of generic metadata parameters. - - Returns: - The created submodel. - """ - namespace: dict[str, Any] = {'__module__': origin.__module__} - bases = (origin,) - meta, ns, kwds = prepare_class(model_name, bases) - namespace.update(ns) - created_model = meta( - model_name, - bases, - namespace, - __pydantic_generic_metadata__={ - 'origin': origin, - 'args': args, - 'parameters': params, - }, - __pydantic_reset_parent_namespace__=False, - **kwds, - ) - - model_module, called_globally = _get_caller_frame_info(depth=3) - if called_globally: # create global reference and therefore allow pickling - object_by_reference = None - reference_name = model_name - reference_module_globals = sys.modules[created_model.__module__].__dict__ - while object_by_reference is not created_model: - object_by_reference = reference_module_globals.setdefault(reference_name, created_model) - reference_name += '_' - - return created_model - - -def _get_caller_frame_info(depth: int = 2) -> tuple[str | None, bool]: - """Used inside a function to check whether it was called globally. - - Args: - depth: The depth to get the frame. - - Returns: - A tuple contains `module_name` and `called_globally`. - - Raises: - RuntimeError: If the function is not called inside a function. - """ - try: - previous_caller_frame = sys._getframe(depth) - except ValueError as e: - raise RuntimeError('This function must be used inside another function') from e - except AttributeError: # sys module does not have _getframe function, so there's nothing we can do about it - return None, False - frame_globals = previous_caller_frame.f_globals - return frame_globals.get('__name__'), previous_caller_frame.f_locals is frame_globals - - -DictValues: type[Any] = {}.values().__class__ - - -def iter_contained_typevars(v: Any) -> Iterator[TypeVar]: - """Recursively iterate through all subtypes and type args of `v` and yield any typevars that are found. - - This is inspired as an alternative to directly accessing the `__parameters__` attribute of a GenericAlias, - since __parameters__ of (nested) generic BaseModel subclasses won't show up in that list. - """ - if isinstance(v, TypeVar): - yield v - elif is_model_class(v): - yield from v.__pydantic_generic_metadata__['parameters'] - elif isinstance(v, (DictValues, list)): - for var in v: - yield from iter_contained_typevars(var) - else: - args = get_args(v) - for arg in args: - yield from iter_contained_typevars(arg) - - -def get_args(v: Any) -> Any: - pydantic_generic_metadata: PydanticGenericMetadata | None = getattr(v, '__pydantic_generic_metadata__', None) - if pydantic_generic_metadata: - return pydantic_generic_metadata.get('args') - return typing_extensions.get_args(v) - - -def get_origin(v: Any) -> Any: - pydantic_generic_metadata: PydanticGenericMetadata | None = getattr(v, '__pydantic_generic_metadata__', None) - if pydantic_generic_metadata: - return pydantic_generic_metadata.get('origin') - return typing_extensions.get_origin(v) - - -def get_standard_typevars_map(cls: Any) -> dict[TypeVar, Any] | None: - """Package a generic type's typevars and parametrization (if present) into a dictionary compatible with the - `replace_types` function. Specifically, this works with standard typing generics and typing._GenericAlias. - """ - origin = get_origin(cls) - if origin is None: - return None - if not hasattr(origin, '__parameters__'): - return None - - # In this case, we know that cls is a _GenericAlias, and origin is the generic type - # So it is safe to access cls.__args__ and origin.__parameters__ - args: tuple[Any, ...] = cls.__args__ # type: ignore - parameters: tuple[TypeVar, ...] = origin.__parameters__ - return dict(zip(parameters, args)) - - -def get_model_typevars_map(cls: type[BaseModel]) -> dict[TypeVar, Any]: - """Package a generic BaseModel's typevars and concrete parametrization (if present) into a dictionary compatible - with the `replace_types` function. - - Since BaseModel.__class_getitem__ does not produce a typing._GenericAlias, and the BaseModel generic info is - stored in the __pydantic_generic_metadata__ attribute, we need special handling here. - """ - # TODO: This could be unified with `get_standard_typevars_map` if we stored the generic metadata - # in the __origin__, __args__, and __parameters__ attributes of the model. - generic_metadata = cls.__pydantic_generic_metadata__ - origin = generic_metadata['origin'] - args = generic_metadata['args'] - if not args: - # No need to go into `iter_contained_typevars`: - return {} - return dict(zip(iter_contained_typevars(origin), args)) - - -def replace_types(type_: Any, type_map: Mapping[TypeVar, Any] | None) -> Any: - """Return type with all occurrences of `type_map` keys recursively replaced with their values. - - Args: - type_: The class or generic alias. - type_map: Mapping from `TypeVar` instance to concrete types. - - Returns: - A new type representing the basic structure of `type_` with all - `typevar_map` keys recursively replaced. - - Example: - ```python - from typing import Union - - from pydantic._internal._generics import replace_types - - replace_types(tuple[str, Union[list[str], float]], {str: int}) - #> tuple[int, Union[list[int], float]] - ``` - """ - if not type_map: - return type_ - - type_args = get_args(type_) - origin_type = get_origin(type_) - - if typing_objects.is_annotated(origin_type): - annotated_type, *annotations = type_args - annotated_type = replace_types(annotated_type, type_map) - # TODO remove parentheses when we drop support for Python 3.10: - return Annotated[(annotated_type, *annotations)] - - # Having type args is a good indicator that this is a typing special form - # instance or a generic alias of some sort. - if type_args: - resolved_type_args = tuple(replace_types(arg, type_map) for arg in type_args) - if all_identical(type_args, resolved_type_args): - # If all arguments are the same, there is no need to modify the - # type or create a new object at all - return type_ - - if ( - origin_type is not None - and isinstance(type_, _typing_extra.typing_base) - and not isinstance(origin_type, _typing_extra.typing_base) - and getattr(type_, '_name', None) is not None - ): - # In python < 3.9 generic aliases don't exist so any of these like `list`, - # `type` or `collections.abc.Callable` need to be translated. - # See: https://www.python.org/dev/peps/pep-0585 - origin_type = getattr(typing, type_._name) - assert origin_type is not None - - if is_union_origin(origin_type): - if any(typing_objects.is_any(arg) for arg in resolved_type_args): - # `Any | T` ~ `Any`: - resolved_type_args = (Any,) - # `Never | T` ~ `T`: - resolved_type_args = tuple( - arg - for arg in resolved_type_args - if not (typing_objects.is_noreturn(arg) or typing_objects.is_never(arg)) - ) - - # PEP-604 syntax (e.g. `list | str`) is represented with a types.UnionType object that does not - # implement `__getitem__()`. In Python 3.14+, `typing.Union` and `types.UnionType` are the same, - # and we instead rely on `typing.Union` as it implicitly converts string annotations to `ForwardRef` - # instances (this is to avoid type errors as per https://github.com/python/cpython/pull/105366). - # TODO remove type ignore comment when we drop support for Python 3.9 (https://github.com/microsoft/pyright/issues/11241): - if (3, 10) <= sys.version_info < (3, 14) and origin_type is types.UnionType: # pyright: ignore[reportAttributeAccessIssue] - return reduce(operator.or_, resolved_type_args) - # NotRequired[T] and Required[T] don't support tuple type resolved_type_args, hence the condition below - return origin_type[resolved_type_args[0] if len(resolved_type_args) == 1 else resolved_type_args] - - # We handle pydantic generic models separately as they don't have the same - # semantics as "typing" classes or generic aliases - - if not origin_type and is_model_class(type_): - parameters = type_.__pydantic_generic_metadata__['parameters'] - if not parameters: - return type_ - resolved_type_args = tuple(replace_types(t, type_map) for t in parameters) - if all_identical(parameters, resolved_type_args): - return type_ - return type_[resolved_type_args] - - # Handle special case for typehints that can have lists as arguments. - # `typing.Callable[[int, str], int]` is an example for this. - if isinstance(type_, list): - resolved_list = [replace_types(element, type_map) for element in type_] - if all_identical(type_, resolved_list): - return type_ - return resolved_list - - # If all else fails, we try to resolve the type directly and otherwise just - # return the input with no modifications. - return type_map.get(type_, type_) - - -def map_generic_model_arguments(cls: type[BaseModel], args: tuple[Any, ...]) -> dict[TypeVar, Any]: - """Return a mapping between the parameters of a generic model and the provided arguments during parameterization. - - Raises: - TypeError: If the number of arguments does not match the parameters (i.e. if providing too few or too many arguments). - - Example: - ```python {test="skip" lint="skip"} - class Model[T, U, V = int](BaseModel): ... - - map_generic_model_arguments(Model, (str, bytes)) - #> {T: str, U: bytes, V: int} - - map_generic_model_arguments(Model, (str,)) - #> TypeError: Too few arguments for ; actual 1, expected at least 2 - - map_generic_model_arguments(Model, (str, bytes, int, complex)) - #> TypeError: Too many arguments for ; actual 4, expected 3 - ``` - - Note: - This function is analogous to the private `typing._check_generic_specialization` function. - """ - parameters = cls.__pydantic_generic_metadata__['parameters'] - expected_len = len(parameters) - typevars_map: dict[TypeVar, Any] = {} - - _missing = object() - for parameter, argument in zip_longest(parameters, args, fillvalue=_missing): - if parameter is _missing: - raise TypeError(f'Too many arguments for {cls}; actual {len(args)}, expected {expected_len}') - - if argument is _missing: - param = cast(TypeVar, parameter) - try: - has_default = param.has_default() # pyright: ignore[reportAttributeAccessIssue] - except AttributeError: - # Happens if using `typing.TypeVar` (and not `typing_extensions`) on Python < 3.13. - has_default = False - if has_default: - # The default might refer to other type parameters. For an example, see: - # https://typing.python.org/en/latest/spec/generics.html#type-parameters-as-parameters-to-generics - typevars_map[param] = replace_types(param.__default__, typevars_map) # pyright: ignore[reportAttributeAccessIssue] - else: - expected_len -= sum(hasattr(p, 'has_default') and p.has_default() for p in parameters) # pyright: ignore[reportAttributeAccessIssue] - raise TypeError(f'Too few arguments for {cls}; actual {len(args)}, expected at least {expected_len}') - else: - param = cast(TypeVar, parameter) - typevars_map[param] = argument - - return typevars_map - - -_generic_recursion_cache: ContextVar[set[str] | None] = ContextVar('_generic_recursion_cache', default=None) - - -@contextmanager -def generic_recursion_self_type( - origin: type[BaseModel], args: tuple[Any, ...] -) -> Iterator[PydanticRecursiveRef | None]: - """This contextmanager should be placed around the recursive calls used to build a generic type, - and accept as arguments the generic origin type and the type arguments being passed to it. - - If the same origin and arguments are observed twice, it implies that a self-reference placeholder - can be used while building the core schema, and will produce a schema_ref that will be valid in the - final parent schema. - """ - previously_seen_type_refs = _generic_recursion_cache.get() - if previously_seen_type_refs is None: - previously_seen_type_refs = set() - token = _generic_recursion_cache.set(previously_seen_type_refs) - else: - token = None - - try: - type_ref = get_type_ref(origin, args_override=args) - if type_ref in previously_seen_type_refs: - self_type = PydanticRecursiveRef(type_ref=type_ref) - yield self_type - else: - previously_seen_type_refs.add(type_ref) - yield - previously_seen_type_refs.remove(type_ref) - finally: - if token: - _generic_recursion_cache.reset(token) - - -def recursively_defined_type_refs() -> set[str]: - visited = _generic_recursion_cache.get() - if not visited: - return set() # not in a generic recursion, so there are no types - - return visited.copy() # don't allow modifications - - -def get_cached_generic_type_early(parent: type[BaseModel], typevar_values: Any) -> type[BaseModel] | None: - """The use of a two-stage cache lookup approach was necessary to have the highest performance possible for - repeated calls to `__class_getitem__` on generic types (which may happen in tighter loops during runtime), - while still ensuring that certain alternative parametrizations ultimately resolve to the same type. - - As a concrete example, this approach was necessary to make Model[List[T]][int] equal to Model[List[int]]. - The approach could be modified to not use two different cache keys at different points, but the - _early_cache_key is optimized to be as quick to compute as possible (for repeated-access speed), and the - _late_cache_key is optimized to be as "correct" as possible, so that two types that will ultimately be the - same after resolving the type arguments will always produce cache hits. - - If we wanted to move to only using a single cache key per type, we would either need to always use the - slower/more computationally intensive logic associated with _late_cache_key, or would need to accept - that Model[List[T]][int] is a different type than Model[List[T]][int]. Because we rely on subclass relationships - during validation, I think it is worthwhile to ensure that types that are functionally equivalent are actually - equal. - """ - return _GENERIC_TYPES_CACHE.get(_early_cache_key(parent, typevar_values)) - - -def get_cached_generic_type_late( - parent: type[BaseModel], typevar_values: Any, origin: type[BaseModel], args: tuple[Any, ...] -) -> type[BaseModel] | None: - """See the docstring of `get_cached_generic_type_early` for more information about the two-stage cache lookup.""" - cached = _GENERIC_TYPES_CACHE.get(_late_cache_key(origin, args, typevar_values)) - if cached is not None: - set_cached_generic_type(parent, typevar_values, cached, origin, args) - return cached - - -def set_cached_generic_type( - parent: type[BaseModel], - typevar_values: tuple[Any, ...], - type_: type[BaseModel], - origin: type[BaseModel] | None = None, - args: tuple[Any, ...] | None = None, -) -> None: - """See the docstring of `get_cached_generic_type_early` for more information about why items are cached with - two different keys. - """ - _GENERIC_TYPES_CACHE[_early_cache_key(parent, typevar_values)] = type_ - if len(typevar_values) == 1: - _GENERIC_TYPES_CACHE[_early_cache_key(parent, typevar_values[0])] = type_ - if origin and args: - _GENERIC_TYPES_CACHE[_late_cache_key(origin, args, typevar_values)] = type_ - - -def _union_orderings_key(typevar_values: Any) -> Any: - """This is intended to help differentiate between Union types with the same arguments in different order. - - Thanks to caching internal to the `typing` module, it is not possible to distinguish between - List[Union[int, float]] and List[Union[float, int]] (and similarly for other "parent" origins besides List) - because `typing` considers Union[int, float] to be equal to Union[float, int]. - - However, you _can_ distinguish between (top-level) Union[int, float] vs. Union[float, int]. - Because we parse items as the first Union type that is successful, we get slightly more consistent behavior - if we make an effort to distinguish the ordering of items in a union. It would be best if we could _always_ - get the exact-correct order of items in the union, but that would require a change to the `typing` module itself. - (See https://github.com/python/cpython/issues/86483 for reference.) - """ - if isinstance(typevar_values, tuple): - return tuple(_union_orderings_key(value) for value in typevar_values) - elif typing_objects.is_union(typing_extensions.get_origin(typevar_values)): - return get_args(typevar_values) - else: - return () - - -def _early_cache_key(cls: type[BaseModel], typevar_values: Any) -> GenericTypesCacheKey: - """This is intended for minimal computational overhead during lookups of cached types. - - Note that this is overly simplistic, and it's possible that two different cls/typevar_values - inputs would ultimately result in the same type being created in BaseModel.__class_getitem__. - To handle this, we have a fallback _late_cache_key that is checked later if the _early_cache_key - lookup fails, and should result in a cache hit _precisely_ when the inputs to __class_getitem__ - would result in the same type. - """ - return cls, typevar_values, _union_orderings_key(typevar_values) - - -def _late_cache_key(origin: type[BaseModel], args: tuple[Any, ...], typevar_values: Any) -> GenericTypesCacheKey: - """This is intended for use later in the process of creating a new type, when we have more information - about the exact args that will be passed. If it turns out that a different set of inputs to - __class_getitem__ resulted in the same inputs to the generic type creation process, we can still - return the cached type, and update the cache with the _early_cache_key as well. - """ - # The _union_orderings_key is placed at the start here to ensure there cannot be a collision with an - # _early_cache_key, as that function will always produce a BaseModel subclass as the first item in the key, - # whereas this function will always produce a tuple as the first item in the key. - return _union_orderings_key(typevar_values), origin, args diff --git a/functions/python/data-processor/pydantic/_internal/_git.py b/functions/python/data-processor/pydantic/_internal/_git.py deleted file mode 100644 index 96dcda28f..000000000 --- a/functions/python/data-processor/pydantic/_internal/_git.py +++ /dev/null @@ -1,27 +0,0 @@ -"""Git utilities, adopted from mypy's git utilities (https://github.com/python/mypy/blob/master/mypy/git.py).""" - -from __future__ import annotations - -import subprocess -from pathlib import Path - - -def is_git_repo(dir: Path) -> bool: - """Is the given directory version-controlled with git?""" - return dir.joinpath('.git').exists() - - -def have_git() -> bool: # pragma: no cover - """Can we run the git executable?""" - try: - subprocess.check_output(['git', '--help']) - return True - except subprocess.CalledProcessError: - return False - except OSError: - return False - - -def git_revision(dir: Path) -> str: - """Get the SHA-1 of the HEAD of a git repository.""" - return subprocess.check_output(['git', 'rev-parse', '--short', 'HEAD'], cwd=dir).decode('utf-8').strip() diff --git a/functions/python/data-processor/pydantic/_internal/_import_utils.py b/functions/python/data-processor/pydantic/_internal/_import_utils.py deleted file mode 100644 index 638102f77..000000000 --- a/functions/python/data-processor/pydantic/_internal/_import_utils.py +++ /dev/null @@ -1,20 +0,0 @@ -from functools import cache -from typing import TYPE_CHECKING - -if TYPE_CHECKING: - from pydantic import BaseModel - from pydantic.fields import FieldInfo - - -@cache -def import_cached_base_model() -> type['BaseModel']: - from pydantic import BaseModel - - return BaseModel - - -@cache -def import_cached_field_info() -> type['FieldInfo']: - from pydantic.fields import FieldInfo - - return FieldInfo diff --git a/functions/python/data-processor/pydantic/_internal/_internal_dataclass.py b/functions/python/data-processor/pydantic/_internal/_internal_dataclass.py deleted file mode 100644 index 33e152cc8..000000000 --- a/functions/python/data-processor/pydantic/_internal/_internal_dataclass.py +++ /dev/null @@ -1,7 +0,0 @@ -import sys - -# `slots` is available on Python >= 3.10 -if sys.version_info >= (3, 10): - slots_true = {'slots': True} -else: - slots_true = {} diff --git a/functions/python/data-processor/pydantic/_internal/_known_annotated_metadata.py b/functions/python/data-processor/pydantic/_internal/_known_annotated_metadata.py deleted file mode 100644 index 5954dba70..000000000 --- a/functions/python/data-processor/pydantic/_internal/_known_annotated_metadata.py +++ /dev/null @@ -1,403 +0,0 @@ -from __future__ import annotations - -from collections import defaultdict -from collections.abc import Iterable -from copy import copy -from functools import lru_cache, partial -from typing import TYPE_CHECKING, Any - -from pydantic_core import CoreSchema, PydanticCustomError, ValidationError, to_jsonable_python -from pydantic_core import core_schema as cs - -from ._fields import PydanticMetadata -from ._import_utils import import_cached_field_info - -if TYPE_CHECKING: - pass - -STRICT = {'strict'} -FAIL_FAST = {'fail_fast'} -LENGTH_CONSTRAINTS = {'min_length', 'max_length'} -INEQUALITY = {'le', 'ge', 'lt', 'gt'} -NUMERIC_CONSTRAINTS = {'multiple_of', *INEQUALITY} -ALLOW_INF_NAN = {'allow_inf_nan'} - -STR_CONSTRAINTS = { - *LENGTH_CONSTRAINTS, - *STRICT, - 'strip_whitespace', - 'to_lower', - 'to_upper', - 'pattern', - 'coerce_numbers_to_str', - 'ascii_only', -} -BYTES_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT} - -LIST_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST} -TUPLE_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST} -SET_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT, *FAIL_FAST} -DICT_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT} -GENERATOR_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *STRICT} -SEQUENCE_CONSTRAINTS = {*LENGTH_CONSTRAINTS, *FAIL_FAST} - -FLOAT_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *ALLOW_INF_NAN, *STRICT} -DECIMAL_CONSTRAINTS = {'max_digits', 'decimal_places', *FLOAT_CONSTRAINTS} -INT_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *ALLOW_INF_NAN, *STRICT} -BOOL_CONSTRAINTS = STRICT -UUID_CONSTRAINTS = STRICT - -DATE_TIME_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT} -TIMEDELTA_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT} -TIME_CONSTRAINTS = {*NUMERIC_CONSTRAINTS, *STRICT} -LAX_OR_STRICT_CONSTRAINTS = STRICT -ENUM_CONSTRAINTS = STRICT -COMPLEX_CONSTRAINTS = STRICT - -UNION_CONSTRAINTS = {'union_mode'} -URL_CONSTRAINTS = { - 'max_length', - 'allowed_schemes', - 'host_required', - 'default_host', - 'default_port', - 'default_path', -} - -TEXT_SCHEMA_TYPES = ('str', 'bytes', 'url', 'multi-host-url') -SEQUENCE_SCHEMA_TYPES = ('list', 'tuple', 'set', 'frozenset', 'generator', *TEXT_SCHEMA_TYPES) -NUMERIC_SCHEMA_TYPES = ('float', 'int', 'date', 'time', 'timedelta', 'datetime') - -CONSTRAINTS_TO_ALLOWED_SCHEMAS: dict[str, set[str]] = defaultdict(set) - -constraint_schema_pairings: list[tuple[set[str], tuple[str, ...]]] = [ - (STR_CONSTRAINTS, TEXT_SCHEMA_TYPES), - (BYTES_CONSTRAINTS, ('bytes',)), - (LIST_CONSTRAINTS, ('list',)), - (TUPLE_CONSTRAINTS, ('tuple',)), - (SET_CONSTRAINTS, ('set', 'frozenset')), - (DICT_CONSTRAINTS, ('dict',)), - (GENERATOR_CONSTRAINTS, ('generator',)), - (FLOAT_CONSTRAINTS, ('float',)), - (INT_CONSTRAINTS, ('int',)), - (DATE_TIME_CONSTRAINTS, ('date', 'time', 'datetime', 'timedelta')), - # TODO: this is a bit redundant, we could probably avoid some of these - (STRICT, (*TEXT_SCHEMA_TYPES, *SEQUENCE_SCHEMA_TYPES, *NUMERIC_SCHEMA_TYPES, 'typed-dict', 'model')), - (UNION_CONSTRAINTS, ('union',)), - (URL_CONSTRAINTS, ('url', 'multi-host-url')), - (BOOL_CONSTRAINTS, ('bool',)), - (UUID_CONSTRAINTS, ('uuid',)), - (LAX_OR_STRICT_CONSTRAINTS, ('lax-or-strict',)), - (ENUM_CONSTRAINTS, ('enum',)), - (DECIMAL_CONSTRAINTS, ('decimal',)), - (COMPLEX_CONSTRAINTS, ('complex',)), -] - -for constraints, schemas in constraint_schema_pairings: - for c in constraints: - CONSTRAINTS_TO_ALLOWED_SCHEMAS[c].update(schemas) - - -def as_jsonable_value(v: Any) -> Any: - if type(v) not in (int, str, float, bytes, bool, type(None)): - return to_jsonable_python(v) - return v - - -def expand_grouped_metadata(annotations: Iterable[Any]) -> Iterable[Any]: - """Expand the annotations. - - Args: - annotations: An iterable of annotations. - - Returns: - An iterable of expanded annotations. - - Example: - ```python - from annotated_types import Ge, Len - - from pydantic._internal._known_annotated_metadata import expand_grouped_metadata - - print(list(expand_grouped_metadata([Ge(4), Len(5)]))) - #> [Ge(ge=4), MinLen(min_length=5)] - ``` - """ - import annotated_types as at - - FieldInfo = import_cached_field_info() - - for annotation in annotations: - if isinstance(annotation, at.GroupedMetadata): - yield from annotation - elif isinstance(annotation, FieldInfo): - yield from annotation.metadata - # this is a bit problematic in that it results in duplicate metadata - # all of our "consumers" can handle it, but it is not ideal - # we probably should split up FieldInfo into: - # - annotated types metadata - # - individual metadata known only to Pydantic - annotation = copy(annotation) - annotation.metadata = [] - yield annotation - else: - yield annotation - - -@lru_cache -def _get_at_to_constraint_map() -> dict[type, str]: - """Return a mapping of annotated types to constraints. - - Normally, we would define a mapping like this in the module scope, but we can't do that - because we don't permit module level imports of `annotated_types`, in an attempt to speed up - the import time of `pydantic`. We still only want to have this dictionary defined in one place, - so we use this function to cache the result. - """ - import annotated_types as at - - return { - at.Gt: 'gt', - at.Ge: 'ge', - at.Lt: 'lt', - at.Le: 'le', - at.MultipleOf: 'multiple_of', - at.MinLen: 'min_length', - at.MaxLen: 'max_length', - } - - -def apply_known_metadata(annotation: Any, schema: CoreSchema) -> CoreSchema | None: # noqa: C901 - """Apply `annotation` to `schema` if it is an annotation we know about (Gt, Le, etc.). - Otherwise return `None`. - - This does not handle all known annotations. If / when it does, it can always - return a CoreSchema and return the unmodified schema if the annotation should be ignored. - - Assumes that GroupedMetadata has already been expanded via `expand_grouped_metadata`. - - Args: - annotation: The annotation. - schema: The schema. - - Returns: - An updated schema with annotation if it is an annotation we know about, `None` otherwise. - - Raises: - RuntimeError: If a constraint can't be applied to a specific schema type. - ValueError: If an unknown constraint is encountered. - """ - import annotated_types as at - - from ._validators import NUMERIC_VALIDATOR_LOOKUP, forbid_inf_nan_check - - schema = schema.copy() - schema_update, other_metadata = collect_known_metadata([annotation]) - schema_type = schema['type'] - - chain_schema_constraints: set[str] = { - 'pattern', - 'strip_whitespace', - 'to_lower', - 'to_upper', - 'coerce_numbers_to_str', - 'ascii_only', - } - chain_schema_steps: list[CoreSchema] = [] - - for constraint, value in schema_update.items(): - if constraint not in CONSTRAINTS_TO_ALLOWED_SCHEMAS: - raise ValueError(f'Unknown constraint {constraint}') - allowed_schemas = CONSTRAINTS_TO_ALLOWED_SCHEMAS[constraint] - - # if it becomes necessary to handle more than one constraint - # in this recursive case with function-after or function-wrap, we should refactor - # this is a bit challenging because we sometimes want to apply constraints to the inner schema, - # whereas other times we want to wrap the existing schema with a new one that enforces a new constraint. - if schema_type in {'function-before', 'function-wrap', 'function-after'} and constraint == 'strict': - schema['schema'] = apply_known_metadata(annotation, schema['schema']) # type: ignore # schema is function schema - return schema - - # if we're allowed to apply constraint directly to the schema, like le to int, do that - if schema_type in allowed_schemas: - if constraint == 'union_mode' and schema_type == 'union': - schema['mode'] = value # type: ignore # schema is UnionSchema - else: - schema[constraint] = value - continue - - # else, apply a function after validator to the schema to enforce the corresponding constraint - if constraint in chain_schema_constraints: - - def _apply_constraint_with_incompatibility_info( - value: Any, handler: cs.ValidatorFunctionWrapHandler - ) -> Any: - try: - x = handler(value) - except ValidationError as ve: - # if the error is about the type, it's likely that the constraint is incompatible the type of the field - # for example, the following invalid schema wouldn't be caught during schema build, but rather at this point - # with a cryptic 'string_type' error coming from the string validator, - # that we'd rather express as a constraint incompatibility error (TypeError) - # Annotated[list[int], Field(pattern='abc')] - if 'type' in ve.errors()[0]['type']: - raise TypeError( - f"Unable to apply constraint '{constraint}' to supplied value {value} for schema of type '{schema_type}'" # noqa: B023 - ) - raise ve - return x - - chain_schema_steps.append( - cs.no_info_wrap_validator_function( - _apply_constraint_with_incompatibility_info, cs.str_schema(**{constraint: value}) - ) - ) - elif constraint in NUMERIC_VALIDATOR_LOOKUP: - if constraint in LENGTH_CONSTRAINTS: - inner_schema = schema - while inner_schema['type'] in {'function-before', 'function-wrap', 'function-after'}: - inner_schema = inner_schema['schema'] # type: ignore - inner_schema_type = inner_schema['type'] - if inner_schema_type == 'list' or ( - inner_schema_type == 'json-or-python' and inner_schema['json_schema']['type'] == 'list' # type: ignore - ): - js_constraint_key = 'minItems' if constraint == 'min_length' else 'maxItems' - else: - js_constraint_key = 'minLength' if constraint == 'min_length' else 'maxLength' - else: - js_constraint_key = constraint - - schema = cs.no_info_after_validator_function( - partial(NUMERIC_VALIDATOR_LOOKUP[constraint], **{constraint: value}), schema - ) - metadata = schema.get('metadata', {}) - if (existing_json_schema_updates := metadata.get('pydantic_js_updates')) is not None: - metadata['pydantic_js_updates'] = { - **existing_json_schema_updates, - **{js_constraint_key: as_jsonable_value(value)}, - } - else: - metadata['pydantic_js_updates'] = {js_constraint_key: as_jsonable_value(value)} - schema['metadata'] = metadata - elif constraint == 'allow_inf_nan' and value is False: - schema = cs.no_info_after_validator_function( - forbid_inf_nan_check, - schema, - ) - else: - # It's rare that we'd get here, but it's possible if we add a new constraint and forget to handle it - # Most constraint errors are caught at runtime during attempted application - raise RuntimeError(f"Unable to apply constraint '{constraint}' to schema of type '{schema_type}'") - - for annotation in other_metadata: - if (annotation_type := type(annotation)) in (at_to_constraint_map := _get_at_to_constraint_map()): - constraint = at_to_constraint_map[annotation_type] - validator = NUMERIC_VALIDATOR_LOOKUP.get(constraint) - if validator is None: - raise ValueError(f'Unknown constraint {constraint}') - schema = cs.no_info_after_validator_function( - partial(validator, {constraint: getattr(annotation, constraint)}), schema - ) - continue - elif isinstance(annotation, (at.Predicate, at.Not)): - predicate_name = f'{annotation.func.__qualname__!r} ' if hasattr(annotation.func, '__qualname__') else '' - - # Note: B023 is ignored because even though we iterate over `other_metadata`, it is guaranteed - # to be of length 1. `apply_known_metadata()` is called from `GenerateSchema`, where annotations - # were already expanded via `expand_grouped_metadata()`. Confusing, but this falls into the annotations - # refactor. - if isinstance(annotation, at.Predicate): - - def val_func(v: Any) -> Any: - predicate_satisfied = annotation.func(v) # noqa: B023 - if not predicate_satisfied: - raise PydanticCustomError( - 'predicate_failed', - f'Predicate {predicate_name}failed', # pyright: ignore[reportArgumentType] # noqa: B023 - ) - return v - - else: - - def val_func(v: Any) -> Any: - predicate_satisfied = annotation.func(v) # noqa: B023 - if predicate_satisfied: - raise PydanticCustomError( - 'not_operation_failed', - f'Not of {predicate_name}failed', # pyright: ignore[reportArgumentType] # noqa: B023 - ) - return v - - schema = cs.no_info_after_validator_function(val_func, schema) - else: - # ignore any other unknown metadata - return None - - if chain_schema_steps: - chain_schema_steps = [schema] + chain_schema_steps - return cs.chain_schema(chain_schema_steps) - - return schema - - -def collect_known_metadata(annotations: Iterable[Any]) -> tuple[dict[str, Any], list[Any]]: - """Split `annotations` into known metadata and unknown annotations. - - Args: - annotations: An iterable of annotations. - - Returns: - A tuple contains a dict of known metadata and a list of unknown annotations. - - Example: - ```python - from annotated_types import Gt, Len - - from pydantic._internal._known_annotated_metadata import collect_known_metadata - - print(collect_known_metadata([Gt(1), Len(42), ...])) - #> ({'gt': 1, 'min_length': 42}, [Ellipsis]) - ``` - """ - annotations = expand_grouped_metadata(annotations) - - res: dict[str, Any] = {} - remaining: list[Any] = [] - - for annotation in annotations: - # isinstance(annotation, PydanticMetadata) also covers ._fields:_PydanticGeneralMetadata - if isinstance(annotation, PydanticMetadata): - res.update(annotation.__dict__) - # we don't use dataclasses.asdict because that recursively calls asdict on the field values - elif (annotation_type := type(annotation)) in (at_to_constraint_map := _get_at_to_constraint_map()): - constraint = at_to_constraint_map[annotation_type] - res[constraint] = getattr(annotation, constraint) - elif isinstance(annotation, type) and issubclass(annotation, PydanticMetadata): - # also support PydanticMetadata classes being used without initialisation, - # e.g. `Annotated[int, Strict]` as well as `Annotated[int, Strict()]` - res.update({k: v for k, v in vars(annotation).items() if not k.startswith('_')}) - else: - remaining.append(annotation) - # Nones can sneak in but pydantic-core will reject them - # it'd be nice to clean things up so we don't put in None (we probably don't _need_ to, it was just easier) - # but this is simple enough to kick that can down the road - res = {k: v for k, v in res.items() if v is not None} - return res, remaining - - -def check_metadata(metadata: dict[str, Any], allowed: Iterable[str], source_type: Any) -> None: - """A small utility function to validate that the given metadata can be applied to the target. - More than saving lines of code, this gives us a consistent error message for all of our internal implementations. - - Args: - metadata: A dict of metadata. - allowed: An iterable of allowed metadata. - source_type: The source type. - - Raises: - TypeError: If there is metadatas that can't be applied on source type. - """ - unknown = metadata.keys() - set(allowed) - if unknown: - raise TypeError( - f'The following constraints cannot be applied to {source_type!r}: {", ".join([f"{k!r}" for k in unknown])}' - ) diff --git a/functions/python/data-processor/pydantic/_internal/_mock_val_ser.py b/functions/python/data-processor/pydantic/_internal/_mock_val_ser.py deleted file mode 100644 index 9125ab32e..000000000 --- a/functions/python/data-processor/pydantic/_internal/_mock_val_ser.py +++ /dev/null @@ -1,228 +0,0 @@ -from __future__ import annotations - -from collections.abc import Iterator, Mapping -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, TypeVar, Union - -from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator - -from ..errors import PydanticErrorCodes, PydanticUserError -from ..plugin._schema_validator import PluggableSchemaValidator - -if TYPE_CHECKING: - from ..dataclasses import PydanticDataclass - from ..main import BaseModel - from ..type_adapter import TypeAdapter - - -ValSer = TypeVar('ValSer', bound=Union[SchemaValidator, PluggableSchemaValidator, SchemaSerializer]) -T = TypeVar('T') - - -class MockCoreSchema(Mapping[str, Any]): - """Mocker for `pydantic_core.CoreSchema` which optionally attempts to - rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. - """ - - __slots__ = '_error_message', '_code', '_attempt_rebuild', '_built_memo' - - def __init__( - self, - error_message: str, - *, - code: PydanticErrorCodes, - attempt_rebuild: Callable[[], CoreSchema | None] | None = None, - ) -> None: - self._error_message = error_message - self._code: PydanticErrorCodes = code - self._attempt_rebuild = attempt_rebuild - self._built_memo: CoreSchema | None = None - - def __getitem__(self, key: str) -> Any: - return self._get_built().__getitem__(key) - - def __len__(self) -> int: - return self._get_built().__len__() - - def __iter__(self) -> Iterator[str]: - return self._get_built().__iter__() - - def _get_built(self) -> CoreSchema: - if self._built_memo is not None: - return self._built_memo - - if self._attempt_rebuild: - schema = self._attempt_rebuild() - if schema is not None: - self._built_memo = schema - return schema - raise PydanticUserError(self._error_message, code=self._code) - - def rebuild(self) -> CoreSchema | None: - self._built_memo = None - if self._attempt_rebuild: - schema = self._attempt_rebuild() - if schema is not None: - return schema - else: - raise PydanticUserError(self._error_message, code=self._code) - return None - - -class MockValSer(Generic[ValSer]): - """Mocker for `pydantic_core.SchemaValidator` or `pydantic_core.SchemaSerializer` which optionally attempts to - rebuild the thing it's mocking when one of its methods is accessed and raises an error if that fails. - """ - - __slots__ = '_error_message', '_code', '_val_or_ser', '_attempt_rebuild' - - def __init__( - self, - error_message: str, - *, - code: PydanticErrorCodes, - val_or_ser: Literal['validator', 'serializer'], - attempt_rebuild: Callable[[], ValSer | None] | None = None, - ) -> None: - self._error_message = error_message - self._val_or_ser = SchemaValidator if val_or_ser == 'validator' else SchemaSerializer - self._code: PydanticErrorCodes = code - self._attempt_rebuild = attempt_rebuild - - def __getattr__(self, item: str) -> None: - __tracebackhide__ = True - if self._attempt_rebuild: - val_ser = self._attempt_rebuild() - if val_ser is not None: - return getattr(val_ser, item) - - # raise an AttributeError if `item` doesn't exist - getattr(self._val_or_ser, item) - raise PydanticUserError(self._error_message, code=self._code) - - def rebuild(self) -> ValSer | None: - if self._attempt_rebuild: - val_ser = self._attempt_rebuild() - if val_ser is not None: - return val_ser - else: - raise PydanticUserError(self._error_message, code=self._code) - return None - - -def set_type_adapter_mocks(adapter: TypeAdapter) -> None: - """Set `core_schema`, `validator` and `serializer` to mock core types on a type adapter instance. - - Args: - adapter: The type adapter instance to set the mocks on - """ - type_repr = str(adapter._type) - undefined_type_error_message = ( - f'`TypeAdapter[{type_repr}]` is not fully defined; you should define `{type_repr}` and all referenced types,' - f' then call `.rebuild()` on the instance.' - ) - - def attempt_rebuild_fn(attr_fn: Callable[[TypeAdapter], T]) -> Callable[[], T | None]: - def handler() -> T | None: - if adapter.rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: - return attr_fn(adapter) - return None - - return handler - - adapter.core_schema = MockCoreSchema( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild_fn(lambda ta: ta.core_schema), - ) - adapter.validator = MockValSer( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - val_or_ser='validator', - attempt_rebuild=attempt_rebuild_fn(lambda ta: ta.validator), - ) - adapter.serializer = MockValSer( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - val_or_ser='serializer', - attempt_rebuild=attempt_rebuild_fn(lambda ta: ta.serializer), - ) - - -def set_model_mocks(cls: type[BaseModel], undefined_name: str = 'all referenced types') -> None: - """Set `__pydantic_core_schema__`, `__pydantic_validator__` and `__pydantic_serializer__` to mock core types on a model. - - Args: - cls: The model class to set the mocks on - undefined_name: Name of the undefined thing, used in error messages - """ - undefined_type_error_message = ( - f'`{cls.__name__}` is not fully defined; you should define {undefined_name},' - f' then call `{cls.__name__}.model_rebuild()`.' - ) - - def attempt_rebuild_fn(attr_fn: Callable[[type[BaseModel]], T]) -> Callable[[], T | None]: - def handler() -> T | None: - if cls.model_rebuild(raise_errors=False, _parent_namespace_depth=5) is not False: - return attr_fn(cls) - return None - - return handler - - cls.__pydantic_core_schema__ = MockCoreSchema( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_core_schema__), - ) - cls.__pydantic_validator__ = MockValSer( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - val_or_ser='validator', - attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_validator__), - ) - cls.__pydantic_serializer__ = MockValSer( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - val_or_ser='serializer', - attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_serializer__), - ) - - -def set_dataclass_mocks(cls: type[PydanticDataclass], undefined_name: str = 'all referenced types') -> None: - """Set `__pydantic_validator__` and `__pydantic_serializer__` to `MockValSer`s on a dataclass. - - Args: - cls: The model class to set the mocks on - undefined_name: Name of the undefined thing, used in error messages - """ - from ..dataclasses import rebuild_dataclass - - undefined_type_error_message = ( - f'`{cls.__name__}` is not fully defined; you should define {undefined_name},' - f' then call `pydantic.dataclasses.rebuild_dataclass({cls.__name__})`.' - ) - - def attempt_rebuild_fn(attr_fn: Callable[[type[PydanticDataclass]], T]) -> Callable[[], T | None]: - def handler() -> T | None: - if rebuild_dataclass(cls, raise_errors=False, _parent_namespace_depth=5) is not False: - return attr_fn(cls) - return None - - return handler - - cls.__pydantic_core_schema__ = MockCoreSchema( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_core_schema__), - ) - cls.__pydantic_validator__ = MockValSer( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - val_or_ser='validator', - attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_validator__), - ) - cls.__pydantic_serializer__ = MockValSer( # pyright: ignore[reportAttributeAccessIssue] - undefined_type_error_message, - code='class-not-fully-defined', - val_or_ser='serializer', - attempt_rebuild=attempt_rebuild_fn(lambda c: c.__pydantic_serializer__), - ) diff --git a/functions/python/data-processor/pydantic/_internal/_model_construction.py b/functions/python/data-processor/pydantic/_internal/_model_construction.py deleted file mode 100644 index b7945442f..000000000 --- a/functions/python/data-processor/pydantic/_internal/_model_construction.py +++ /dev/null @@ -1,868 +0,0 @@ -"""Private logic for creating models.""" - -from __future__ import annotations as _annotations - -import operator -import sys -import typing -import warnings -import weakref -from abc import ABCMeta -from functools import cache, partial, wraps -from types import FunctionType -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, NoReturn, TypeVar, cast - -from pydantic_core import PydanticUndefined, SchemaSerializer -from typing_extensions import TypeAliasType, dataclass_transform, deprecated, get_args, get_origin -from typing_inspection import typing_objects - -from ..errors import PydanticUndefinedAnnotation, PydanticUserError -from ..plugin._schema_validator import create_schema_validator -from ..warnings import GenericBeforeBaseModelWarning, PydanticDeprecatedSince20 -from ._config import ConfigWrapper -from ._decorators import DecoratorInfos, PydanticDescriptorProxy, get_attribute_from_bases, unwrap_wrapped_function -from ._fields import collect_model_fields, is_valid_field_name, is_valid_privateattr_name, rebuild_model_fields -from ._generate_schema import GenerateSchema, InvalidSchemaError -from ._generics import PydanticGenericMetadata, get_model_typevars_map -from ._import_utils import import_cached_base_model, import_cached_field_info -from ._mock_val_ser import set_model_mocks -from ._namespace_utils import NsResolver -from ._signature import generate_pydantic_signature -from ._typing_extra import ( - _make_forward_ref, - eval_type_backport, - is_classvar_annotation, - parent_frame_namespace, -) -from ._utils import LazyClassAttribute, SafeGetItemProxy - -if TYPE_CHECKING: - from ..fields import Field as PydanticModelField - from ..fields import FieldInfo, ModelPrivateAttr - from ..fields import PrivateAttr as PydanticModelPrivateAttr - from ..main import BaseModel - from ._fields import PydanticExtraInfo -else: - PydanticModelField = object() - PydanticModelPrivateAttr = object() - -object_setattr = object.__setattr__ - - -class _ModelNamespaceDict(dict): - """A dictionary subclass that intercepts attribute setting on model classes and - warns about overriding of decorators. - """ - - def __setitem__(self, k: str, v: object) -> None: - existing: Any = self.get(k, None) - if existing and v is not existing and isinstance(existing, PydanticDescriptorProxy): - warnings.warn( - f'`{k}` overrides an existing Pydantic `{existing.decorator_info.decorator_repr}` decorator', - stacklevel=2, - ) - - return super().__setitem__(k, v) - - -def NoInitField( - *, - init: Literal[False] = False, -) -> Any: - """Only for typing purposes. Used as default value of `__pydantic_fields_set__`, - `__pydantic_extra__`, `__pydantic_private__`, so they could be ignored when - synthesizing the `__init__` signature. - """ - - -# For ModelMetaclass.register(): -_T = TypeVar('_T') - - -@dataclass_transform(kw_only_default=True, field_specifiers=(PydanticModelField, PydanticModelPrivateAttr, NoInitField)) -class ModelMetaclass(ABCMeta): - def __new__( - mcs, - cls_name: str, - bases: tuple[type[Any], ...], - namespace: dict[str, Any], - __pydantic_generic_metadata__: PydanticGenericMetadata | None = None, - __pydantic_reset_parent_namespace__: bool = True, - _create_model_module: str | None = None, - **kwargs: Any, - ) -> type: - """Metaclass for creating Pydantic models. - - Args: - cls_name: The name of the class to be created. - bases: The base classes of the class to be created. - namespace: The attribute dictionary of the class to be created. - __pydantic_generic_metadata__: Metadata for generic models. - __pydantic_reset_parent_namespace__: Reset parent namespace. - _create_model_module: The module of the class to be created, if created by `create_model`. - **kwargs: Catch-all for any other keyword arguments. - - Returns: - The new class created by the metaclass. - """ - # Note `ModelMetaclass` refers to `BaseModel`, but is also used to *create* `BaseModel`, so we rely on the fact - # that `BaseModel` itself won't have any bases, but any subclass of it will, to determine whether the `__new__` - # call we're in the middle of is for the `BaseModel` class. - if bases: - raw_annotations: dict[str, Any] - if sys.version_info >= (3, 14): - if ( - '__annotations__' in namespace - ): # `from __future__ import annotations` was used in the model's module - raw_annotations = namespace['__annotations__'] - else: - # See https://docs.python.org/3.14/library/annotationlib.html#using-annotations-in-a-metaclass: - from annotationlib import Format, call_annotate_function, get_annotate_from_class_namespace - - if annotate := get_annotate_from_class_namespace(namespace): - raw_annotations = call_annotate_function(annotate, format=Format.FORWARDREF) - else: - raw_annotations = {} - else: - raw_annotations = namespace.get('__annotations__', {}) - - base_field_names, class_vars, base_private_attributes = mcs._collect_bases_data(bases) - - config_wrapper = ConfigWrapper.for_model(bases, namespace, raw_annotations, kwargs) - namespace['model_config'] = config_wrapper.config_dict - private_attributes = inspect_namespace( - namespace, raw_annotations, config_wrapper.ignored_types, class_vars, base_field_names - ) - if private_attributes or base_private_attributes: - original_model_post_init = get_model_post_init(namespace, bases) - if original_model_post_init is not None: - # if there are private attributes and a model_post_init function, we handle both - - @wraps(original_model_post_init) - def wrapped_model_post_init(self: BaseModel, context: Any, /) -> None: - """We need to both initialize private attributes and call the user-defined model_post_init - method. - """ - init_private_attributes(self, context) - original_model_post_init(self, context) - - namespace['model_post_init'] = wrapped_model_post_init - else: - namespace['model_post_init'] = init_private_attributes - - namespace['__class_vars__'] = class_vars - namespace['__private_attributes__'] = {**base_private_attributes, **private_attributes} - - cls = cast('type[BaseModel]', super().__new__(mcs, cls_name, bases, namespace, **kwargs)) - BaseModel_ = import_cached_base_model() - - mro = cls.__mro__ - if Generic in mro and mro.index(Generic) < mro.index(BaseModel_): - warnings.warn( - GenericBeforeBaseModelWarning( - 'Classes should inherit from `BaseModel` before generic classes (e.g. `typing.Generic[T]`) ' - 'for pydantic generics to work properly.' - ), - stacklevel=2, - ) - - cls.__pydantic_custom_init__ = not getattr(cls.__init__, '__pydantic_base_init__', False) - cls.__pydantic_post_init__ = ( - None if cls.model_post_init is BaseModel_.model_post_init else 'model_post_init' - ) - - cls.__pydantic_setattr_handlers__ = {} - - cls.__pydantic_decorators__ = DecoratorInfos.build(cls, replace_wrapped_methods=True) - cls.__pydantic_decorators__.update_from_config(config_wrapper) - - # Use the getattr below to grab the __parameters__ from the `typing.Generic` parent class - if __pydantic_generic_metadata__: - cls.__pydantic_generic_metadata__ = __pydantic_generic_metadata__ - else: - parent_parameters = getattr(cls, '__pydantic_generic_metadata__', {}).get('parameters', ()) - parameters = getattr(cls, '__parameters__', None) or parent_parameters - if parameters and parent_parameters and not all(x in parameters for x in parent_parameters): - from ..root_model import RootModelRootType - - missing_parameters = tuple(x for x in parameters if x not in parent_parameters) - if RootModelRootType in parent_parameters and RootModelRootType not in parameters: - # This is a special case where the user has subclassed RootModel, but has not parameterized - # RootModel with the generic type identifiers being used. Ex: - # class MyModel(RootModel, Generic[T]): - # root: T - # Should instead just be: - # class MyModel(RootModel[T]): - # root: T - parameters_str = ', '.join([x.__name__ for x in missing_parameters]) - error_message = ( - f'{cls.__name__} is a subclass of `RootModel`, but does not include the generic type identifier(s) ' - f'{parameters_str} in its parameters. ' - f'You should parametrize RootModel directly, e.g., `class {cls.__name__}(RootModel[{parameters_str}]): ...`.' - ) - else: - combined_parameters = parent_parameters + missing_parameters - parameters_str = ', '.join([str(x) for x in combined_parameters]) - generic_type_label = f'typing.Generic[{parameters_str}]' - error_message = ( - f'All parameters must be present on typing.Generic;' - f' you should inherit from {generic_type_label}.' - ) - if Generic not in bases: # pragma: no cover - # We raise an error here not because it is desirable, but because some cases are mishandled. - # It would be nice to remove this error and still have things behave as expected, it's just - # challenging because we are using a custom `__class_getitem__` to parametrize generic models, - # and not returning a typing._GenericAlias from it. - bases_str = ', '.join([x.__name__ for x in bases] + [generic_type_label]) - error_message += ( - f' Note: `typing.Generic` must go last: `class {cls.__name__}({bases_str}): ...`)' - ) - raise TypeError(error_message) - - cls.__pydantic_generic_metadata__ = { - 'origin': None, - 'args': (), - 'parameters': parameters, - } - - cls.__pydantic_complete__ = False # Ensure this specific class gets completed - - # preserve `__set_name__` protocol defined in https://peps.python.org/pep-0487 - # for attributes not in `new_namespace` (e.g. private attributes) - for name, obj in private_attributes.items(): - obj.__set_name__(cls, name) - - if __pydantic_reset_parent_namespace__: - cls.__pydantic_parent_namespace__ = build_lenient_weakvaluedict(parent_frame_namespace()) - parent_namespace: dict[str, Any] | None = getattr(cls, '__pydantic_parent_namespace__', None) - if isinstance(parent_namespace, dict): - parent_namespace = unpack_lenient_weakvaluedict(parent_namespace) - - ns_resolver = NsResolver(parent_namespace=parent_namespace) - - set_model_fields(cls, config_wrapper=config_wrapper, ns_resolver=ns_resolver) - - # This is also set in `complete_model_class()`, after schema gen because they are recreated. - # We set them here as well for backwards compatibility: - cls.__pydantic_computed_fields__ = { - k: v.info for k, v in cls.__pydantic_decorators__.computed_fields.items() - } - - if config_wrapper.defer_build: - set_model_mocks(cls) - else: - # Any operation that requires accessing the field infos instances should be put inside - # `complete_model_class()`: - complete_model_class( - cls, - config_wrapper, - ns_resolver, - raise_errors=False, - create_model_module=_create_model_module, - ) - - if config_wrapper.frozen and '__hash__' not in namespace: - set_default_hash_func(cls, bases) - - # using super(cls, cls) on the next line ensures we only call the parent class's __pydantic_init_subclass__ - # I believe the `type: ignore` is only necessary because mypy doesn't realize that this code branch is - # only hit for _proper_ subclasses of BaseModel - super(cls, cls).__pydantic_init_subclass__(**kwargs) # type: ignore[misc] - return cls - else: - # These are instance variables, but have been assigned to `NoInitField` to trick the type checker. - for instance_slot in '__pydantic_fields_set__', '__pydantic_extra__', '__pydantic_private__': - namespace.pop( - instance_slot, - None, # In case the metaclass is used with a class other than `BaseModel`. - ) - namespace.get('__annotations__', {}).clear() - return super().__new__(mcs, cls_name, bases, namespace, **kwargs) - - if not TYPE_CHECKING: # pragma: no branch - # We put `__getattr__` in a non-TYPE_CHECKING block because otherwise, mypy allows arbitrary attribute access - - def __getattr__(self, item: str) -> Any: - """This is necessary to keep attribute access working for class attribute access.""" - private_attributes = self.__dict__.get('__private_attributes__') - if private_attributes and item in private_attributes: - return private_attributes[item] - raise AttributeError(item) - - @classmethod - def __prepare__(cls, *args: Any, **kwargs: Any) -> dict[str, object]: - return _ModelNamespaceDict() - - # Due to performance and memory issues, in the ABCMeta.__subclasscheck__ implementation, we don't support - # registered virtual subclasses. See https://github.com/python/cpython/issues/92810#issuecomment-2762454345. - # This may change once CPython is fixed (possibly in 3.15), in which case we should conditionally - # define `register()`. - def register(self, subclass: type[_T]) -> type[_T]: - warnings.warn( - f"For performance reasons, virtual subclasses registered using '{self.__qualname__}.register()' " - "are not supported in 'isinstance()' and 'issubclass()' checks.", - stacklevel=2, - ) - return super().register(subclass) - - __instancecheck__ = type.__instancecheck__ # pyright: ignore[reportAssignmentType] - __subclasscheck__ = type.__subclasscheck__ # pyright: ignore[reportAssignmentType] - - @staticmethod - def _collect_bases_data(bases: tuple[type[Any], ...]) -> tuple[set[str], set[str], dict[str, ModelPrivateAttr]]: - BaseModel = import_cached_base_model() - - field_names: set[str] = set() - class_vars: set[str] = set() - private_attributes: dict[str, ModelPrivateAttr] = {} - for base in bases: - if issubclass(base, BaseModel) and base is not BaseModel: - # model_fields might not be defined yet in the case of generics, so we use getattr here: - field_names.update(getattr(base, '__pydantic_fields__', {}).keys()) - class_vars.update(base.__class_vars__) - private_attributes.update(base.__private_attributes__) - return field_names, class_vars, private_attributes - - @property - @deprecated( - 'The `__fields__` attribute is deprecated, use the `model_fields` class property instead.', category=None - ) - def __fields__(self) -> dict[str, FieldInfo]: - warnings.warn( - 'The `__fields__` attribute is deprecated, use the `model_fields` class property instead.', - PydanticDeprecatedSince20, - stacklevel=2, - ) - return getattr(self, '__pydantic_fields__', {}) - - @property - def __pydantic_fields_complete__(self) -> bool: - """Whether the fields were successfully collected (i.e. type hints were successfully resolved). - - This is a private attribute, not meant to be used outside Pydantic. - """ - if '__pydantic_fields__' not in self.__dict__: - return False - - field_infos = cast('dict[str, FieldInfo]', self.__pydantic_fields__) # pyright: ignore[reportAttributeAccessIssue] - - pydantic_extra_info = cast('PydanticExtraInfo | None', self.__pydantic_extra_info__) # pyright: ignore[reportAttributeAccessIssue] - if pydantic_extra_info is not None: - extra_complete = pydantic_extra_info.complete - else: - extra_complete = True - - return all(field_info._complete for field_info in field_infos.values()) and extra_complete - - def __dir__(self) -> list[str]: - attributes = list(super().__dir__()) - if '__fields__' in attributes: - attributes.remove('__fields__') - return attributes - - -def init_private_attributes(self: BaseModel, context: Any, /) -> None: - """This function is meant to behave like a BaseModel method to initialize private attributes. - - It takes context as an argument since that's what pydantic-core passes when calling it. - - Args: - self: The BaseModel instance. - context: The context. - """ - if getattr(self, '__pydantic_private__', None) is None: - pydantic_private = {} - for name, private_attr in self.__private_attributes__.items(): - # Avoid needlessly creating a new dict for the validated data: - if private_attr.default_factory_takes_validated_data: - default = private_attr.get_default( - call_default_factory=True, validated_data={**self.__dict__, **pydantic_private} - ) - else: - default = private_attr.get_default(call_default_factory=True) - if default is not PydanticUndefined: - pydantic_private[name] = default - object_setattr(self, '__pydantic_private__', pydantic_private) - - -def get_model_post_init(namespace: dict[str, Any], bases: tuple[type[Any], ...]) -> Callable[..., Any] | None: - """Get the `model_post_init` method from the namespace or the class bases, or `None` if not defined.""" - if 'model_post_init' in namespace: - return namespace['model_post_init'] - - BaseModel = import_cached_base_model() - - model_post_init = get_attribute_from_bases(bases, 'model_post_init') - if model_post_init is not BaseModel.model_post_init: - return model_post_init - - -def inspect_namespace( # noqa C901 - namespace: dict[str, Any], - raw_annotations: dict[str, Any], - ignored_types: tuple[type[Any], ...], - base_class_vars: set[str], - base_class_fields: set[str], -) -> dict[str, ModelPrivateAttr]: - """Iterate over the namespace and: - * gather private attributes - * check for items which look like fields but are not (e.g. have no annotation) and warn. - - Args: - namespace: The attribute dictionary of the class to be created. - raw_annotations: The (non-evaluated) annotations of the model. - ignored_types: A tuple of ignore types. - base_class_vars: A set of base class class variables. - base_class_fields: A set of base class fields. - - Returns: - A dict containing private attributes info. - - Raises: - TypeError: If there is a `__root__` field in model. - NameError: If private attribute name is invalid. - PydanticUserError: - - If a field does not have a type annotation. - - If a field on base class was overridden by a non-annotated attribute. - """ - from ..fields import ModelPrivateAttr, PrivateAttr - - FieldInfo = import_cached_field_info() - - all_ignored_types = ignored_types + default_ignored_types() - - private_attributes: dict[str, ModelPrivateAttr] = {} - - if '__root__' in raw_annotations or '__root__' in namespace: - raise TypeError("To define root models, use `pydantic.RootModel` rather than a field called '__root__'") - - ignored_names: set[str] = set() - for var_name, value in list(namespace.items()): - if var_name == 'model_config' or var_name == '__pydantic_extra__': - continue - elif ( - isinstance(value, type) - and value.__module__ == namespace['__module__'] - and '__qualname__' in namespace - and value.__qualname__.startswith(f'{namespace["__qualname__"]}.') - ): - # `value` is a nested type defined in this namespace; don't error - continue - elif isinstance(value, all_ignored_types) or value.__class__.__module__ == 'functools': - ignored_names.add(var_name) - continue - elif isinstance(value, ModelPrivateAttr): - if var_name.startswith('__'): - raise NameError( - 'Private attributes must not use dunder names;' - f' use a single underscore prefix instead of {var_name!r}.' - ) - elif is_valid_field_name(var_name): - raise NameError( - 'Private attributes must not use valid field names;' - f' use sunder names, e.g. {"_" + var_name!r} instead of {var_name!r}.' - ) - private_attributes[var_name] = value - del namespace[var_name] - elif isinstance(value, FieldInfo) and not is_valid_field_name(var_name): - suggested_name = var_name.lstrip('_') or 'my_field' # don't suggest '' for all-underscore name - raise NameError( - f'Fields must not use names with leading underscores;' - f' e.g., use {suggested_name!r} instead of {var_name!r}.' - ) - - elif var_name.startswith('__'): - continue - elif is_valid_privateattr_name(var_name): - if var_name not in raw_annotations or not is_classvar_annotation(raw_annotations[var_name]): - private_attributes[var_name] = cast(ModelPrivateAttr, PrivateAttr(default=value)) - del namespace[var_name] - elif var_name in base_class_vars: - continue - elif var_name not in raw_annotations: - if var_name in base_class_fields: - raise PydanticUserError( - f'Field {var_name!r} defined on a base class was overridden by a non-annotated attribute. ' - f'All field definitions, including overrides, require a type annotation.', - code='model-field-overridden', - ) - elif isinstance(value, FieldInfo): - raise PydanticUserError( - f'Field {var_name!r} requires a type annotation', code='model-field-missing-annotation' - ) - else: - raise PydanticUserError( - f'A non-annotated attribute was detected: `{var_name} = {value!r}`. All model fields require a ' - f'type annotation; if `{var_name}` is not meant to be a field, you may be able to resolve this ' - f"error by annotating it as a `ClassVar` or updating `model_config['ignored_types']`.", - code='model-field-missing-annotation', - ) - - for ann_name, ann_type in raw_annotations.items(): - if ( - is_valid_privateattr_name(ann_name) - and ann_name not in private_attributes - and ann_name not in ignored_names - # This condition can be a false negative when `ann_type` is stringified, - # but it is handled in most cases in `set_model_fields`: - and not is_classvar_annotation(ann_type) - and ann_type not in all_ignored_types - and getattr(ann_type, '__module__', None) != 'functools' - ): - if isinstance(ann_type, str): - # Walking up the frames to get the module namespace where the model is defined - # (as the model class wasn't created yet, we unfortunately can't use `cls.__module__`): - frame = sys._getframe(2) - if frame is not None: - try: - ann_type = eval_type_backport( - _make_forward_ref(ann_type, is_argument=False, is_class=True), - globalns=frame.f_globals, - localns=frame.f_locals, - ) - except (NameError, TypeError): - pass - - if typing_objects.is_annotated(get_origin(ann_type)): - _, *metadata = get_args(ann_type) - private_attr = next((v for v in metadata if isinstance(v, ModelPrivateAttr)), None) - if private_attr is not None: - private_attributes[ann_name] = private_attr - continue - private_attributes[ann_name] = PrivateAttr() - - return private_attributes - - -def set_default_hash_func(cls: type[BaseModel], bases: tuple[type[Any], ...]) -> None: - base_hash_func = get_attribute_from_bases(bases, '__hash__') - new_hash_func = make_hash_func(cls) - if base_hash_func in {None, object.__hash__} or getattr(base_hash_func, '__code__', None) == new_hash_func.__code__: - # If `__hash__` is some default, we generate a hash function. - # It will be `None` if not overridden from BaseModel. - # It may be `object.__hash__` if there is another - # parent class earlier in the bases which doesn't override `__hash__` (e.g. `typing.Generic`). - # It may be a value set by `set_default_hash_func` if `cls` is a subclass of another frozen model. - # In the last case we still need a new hash function to account for new `model_fields`. - cls.__hash__ = new_hash_func - - -def make_hash_func(cls: type[BaseModel]) -> Any: - getter = operator.itemgetter(*cls.__pydantic_fields__.keys()) if cls.__pydantic_fields__ else lambda _: 0 - - def hash_func(self: Any) -> int: - try: - return hash(getter(self.__dict__)) - except KeyError: - # In rare cases (such as when using the deprecated copy method), the __dict__ may not contain - # all model fields, which is how we can get here. - # getter(self.__dict__) is much faster than any 'safe' method that accounts for missing keys, - # and wrapping it in a `try` doesn't slow things down much in the common case. - return hash(getter(SafeGetItemProxy(self.__dict__))) - - return hash_func - - -def set_model_fields( - cls: type[BaseModel], - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver, -) -> None: - """Collect and set `cls.__pydantic_fields__` and `cls.__class_vars__`. - - Args: - cls: BaseModel or dataclass. - config_wrapper: The config wrapper instance. - ns_resolver: Namespace resolver to use when getting model annotations. - """ - typevars_map = get_model_typevars_map(cls) - fields, pydantic_extra_info, class_vars = collect_model_fields( - cls, config_wrapper, ns_resolver, typevars_map=typevars_map - ) - - cls.__pydantic_fields__ = fields - cls.__pydantic_extra_info__ = pydantic_extra_info - cls.__class_vars__.update(class_vars) - - for k in class_vars: - # Class vars should not be private attributes - # We remove them _here_ and not earlier because we rely on inspecting the class to determine its classvars, - # but private attributes are determined by inspecting the namespace _prior_ to class creation. - # In the case that a classvar with a leading-'_' is defined via a ForwardRef (e.g., when using - # `__future__.annotations`), we want to remove the private attribute which was detected _before_ we knew it - # evaluated to a classvar - - value = cls.__private_attributes__.pop(k, None) - if value is not None and value.default is not PydanticUndefined: - setattr(cls, k, value.default) - - -def complete_model_class( - cls: type[BaseModel], - config_wrapper: ConfigWrapper, - ns_resolver: NsResolver, - *, - raise_errors: bool = True, - call_on_complete_hook: bool = True, - create_model_module: str | None = None, - is_force_rebuild: bool = False, -) -> bool: - """Finish building a model class. - - This logic must be called after class has been created since validation functions must be bound - and `get_type_hints` requires a class object. - - Args: - cls: BaseModel or dataclass. - config_wrapper: The config wrapper instance. - ns_resolver: The namespace resolver instance to use during schema building. - raise_errors: Whether to raise errors. - call_on_complete_hook: Whether to call the `__pydantic_on_complete__` hook. - create_model_module: The module of the class to be created, if created by `create_model`. - is_force_rebuild: Whether the model is being force-rebuilt (if True, pre-built serializers and - validators are not used, to avoid stale references). - - Returns: - `True` if the model is successfully completed, else `False`. - - Raises: - PydanticUndefinedAnnotation: If PydanticUndefinedAnnotation occurs in __get_pydantic_core_schema__ - and `raise_errors=True`. - """ - typevars_map = get_model_typevars_map(cls) - - if not cls.__pydantic_fields_complete__: - # Note: when coming from `ModelMetaclass.__new__()`, this results in fields being built twice. - # We do so a second time here so that we can get the ``NameError`` for the specific undefined annotation. - # Alternatively, we could let `GenerateSchema()` raise the error, but there are cases where incomplete - # fields are inherited in `collect_model_fields()` and can actually have their annotation resolved in the - # generate schema process. As we want to avoid having `__pydantic_fields_complete__` set to `False` - # when `__pydantic_complete__` is `True`, we rebuild here: - try: - cls.__pydantic_fields__, cls.__pydantic_extra_info__ = rebuild_model_fields( - cls, - config_wrapper=config_wrapper, - ns_resolver=ns_resolver, - typevars_map=typevars_map, - ) - except NameError as e: - exc = PydanticUndefinedAnnotation.from_name_error(e) - set_model_mocks(cls, f'`{exc.name}`') - if raise_errors: - raise exc from e - - if not raise_errors and not cls.__pydantic_fields_complete__: - # No need to continue with schema gen, it is guaranteed to fail - return False - - assert cls.__pydantic_fields_complete__ - - gen_schema = GenerateSchema( - config_wrapper, - ns_resolver, - typevars_map, - ) - - try: - schema = gen_schema.generate_schema(cls) - except PydanticUndefinedAnnotation as e: - if raise_errors: - raise - set_model_mocks(cls, f'`{e.name}`') - return False - - core_config = config_wrapper.core_config(title=cls.__name__) - - try: - schema = gen_schema.clean_schema(schema) - except InvalidSchemaError: - set_model_mocks(cls) - return False - - # This needs to happen *after* model schema generation, as the return types - # of the properties are evaluated and the `ComputedFieldInfo` are recreated: - cls.__pydantic_computed_fields__ = {k: v.info for k, v in cls.__pydantic_decorators__.computed_fields.items()} - - set_deprecated_descriptors(cls) - - cls.__pydantic_core_schema__ = schema - - cls.__pydantic_validator__ = create_schema_validator( - schema, - cls, - create_model_module or cls.__module__, - cls.__qualname__, - 'create_model' if create_model_module else 'BaseModel', - core_config, - config_wrapper.plugin_settings, - _use_prebuilt=not is_force_rebuild, - ) - cls.__pydantic_serializer__ = SchemaSerializer(schema, core_config, _use_prebuilt=not is_force_rebuild) - - # set __signature__ attr only for model class, but not for its instances - # (because instances can define `__call__`, and `inspect.signature` shouldn't - # use the `__signature__` attribute and instead generate from `__call__`). - cls.__signature__ = LazyClassAttribute( - '__signature__', - partial( - generate_pydantic_signature, - init=cls.__init__, - fields=cls.__pydantic_fields__, - validate_by_name=config_wrapper.validate_by_name, - extra=config_wrapper.extra, - ), - ) - - cls.__pydantic_complete__ = True - - if call_on_complete_hook: - cls.__pydantic_on_complete__() - - return True - - -def set_deprecated_descriptors(cls: type[BaseModel]) -> None: - """Set data descriptors on the class for deprecated fields.""" - for field, field_info in cls.__pydantic_fields__.items(): - if (msg := field_info.deprecation_message) is not None: - desc = _DeprecatedFieldDescriptor(msg) - desc.__set_name__(cls, field) - setattr(cls, field, desc) - - for field, computed_field_info in cls.__pydantic_computed_fields__.items(): - if ( - (msg := computed_field_info.deprecation_message) is not None - # Avoid having two warnings emitted: - and not hasattr(unwrap_wrapped_function(computed_field_info.wrapped_property), '__deprecated__') - ): - desc = _DeprecatedFieldDescriptor(msg, computed_field_info.wrapped_property) - desc.__set_name__(cls, field) - setattr(cls, field, desc) - - -class _DeprecatedFieldDescriptor: - """Read-only data descriptor used to emit a runtime deprecation warning before accessing a deprecated field. - - Attributes: - msg: The deprecation message to be emitted. - wrapped_property: The property instance if the deprecated field is a computed field, or `None`. - field_name: The name of the field being deprecated. - """ - - field_name: str - - def __init__(self, msg: str, wrapped_property: property | None = None) -> None: - self.msg = msg - self.wrapped_property = wrapped_property - - def __set_name__(self, cls: type[BaseModel], name: str) -> None: - self.field_name = name - - def __get__(self, obj: BaseModel | None, obj_type: type[BaseModel] | None = None) -> Any: - if obj is None: - if self.wrapped_property is not None: - return self.wrapped_property.__get__(None, obj_type) - raise AttributeError(self.field_name) - - warnings.warn(self.msg, DeprecationWarning, stacklevel=2) - - if self.wrapped_property is not None: - return self.wrapped_property.__get__(obj, obj_type) - return obj.__dict__[self.field_name] - - # Defined to make it a data descriptor and take precedence over the instance's dictionary. - # Note that it will not be called when setting a value on a model instance - # as `BaseModel.__setattr__` is defined and takes priority. - def __set__(self, obj: Any, value: Any) -> NoReturn: - raise AttributeError(self.field_name) - - -class _PydanticWeakRef: - """Wrapper for `weakref.ref` that enables `pickle` serialization. - - Cloudpickle fails to serialize weakref.ref objects due to an arcane error related to - to abstract base classes (`abc.ABC`). This class works around the issue by wrapping - `weakref.ref` instead of subclassing it. - - See https://github.com/pydantic/pydantic/issues/6763 for context. - - Semantics: - - If not pickled, behaves the same as a `weakref.ref`. - - If pickled along with the referenced object, the same `weakref.ref` behavior - will be maintained between them after unpickling. - - If pickled without the referenced object, after unpickling the underlying - reference will be cleared (`__call__` will always return `None`). - """ - - def __init__(self, obj: Any): - if obj is None: - # The object will be `None` upon deserialization if the serialized weakref - # had lost its underlying object. - self._wr = None - else: - self._wr = weakref.ref(obj) - - def __call__(self) -> Any: - if self._wr is None: - return None - else: - return self._wr() - - def __reduce__(self) -> tuple[Callable, tuple[weakref.ReferenceType | None]]: - return _PydanticWeakRef, (self(),) - - -def build_lenient_weakvaluedict(d: dict[str, Any] | None) -> dict[str, Any] | None: - """Takes an input dictionary, and produces a new value that (invertibly) replaces the values with weakrefs. - - We can't just use a WeakValueDictionary because many types (including int, str, etc.) can't be stored as values - in a WeakValueDictionary. - - The `unpack_lenient_weakvaluedict` function can be used to reverse this operation. - """ - if d is None: - return None - result = {} - for k, v in d.items(): - try: - proxy = _PydanticWeakRef(v) - except TypeError: - proxy = v - result[k] = proxy - return result - - -def unpack_lenient_weakvaluedict(d: dict[str, Any] | None) -> dict[str, Any] | None: - """Inverts the transform performed by `build_lenient_weakvaluedict`.""" - if d is None: - return None - - result = {} - for k, v in d.items(): - if isinstance(v, _PydanticWeakRef): - v = v() - if v is not None: - result[k] = v - else: - result[k] = v - return result - - -@cache -def default_ignored_types() -> tuple[type[Any], ...]: - from ..fields import ComputedFieldInfo - - ignored_types = [ - FunctionType, - property, - classmethod, - staticmethod, - PydanticDescriptorProxy, - ComputedFieldInfo, - TypeAliasType, # from `typing_extensions` - ] - - if sys.version_info >= (3, 12): - ignored_types.append(typing.TypeAliasType) - - return tuple(ignored_types) diff --git a/functions/python/data-processor/pydantic/_internal/_namespace_utils.py b/functions/python/data-processor/pydantic/_internal/_namespace_utils.py deleted file mode 100644 index af0cddb03..000000000 --- a/functions/python/data-processor/pydantic/_internal/_namespace_utils.py +++ /dev/null @@ -1,293 +0,0 @@ -from __future__ import annotations - -import sys -from collections.abc import Generator, Iterator, Mapping -from contextlib import contextmanager -from functools import cached_property -from typing import Any, Callable, NamedTuple, TypeVar - -from typing_extensions import ParamSpec, TypeAlias, TypeAliasType, TypeVarTuple - -GlobalsNamespace: TypeAlias = 'dict[str, Any]' -"""A global namespace. - -In most cases, this is a reference to the `__dict__` attribute of a module. -This namespace type is expected as the `globals` argument during annotations evaluation. -""" - -MappingNamespace: TypeAlias = Mapping[str, Any] -"""Any kind of namespace. - -In most cases, this is a local namespace (e.g. the `__dict__` attribute of a class, -the [`f_locals`][frame.f_locals] attribute of a frame object, when dealing with types -defined inside functions). -This namespace type is expected as the `locals` argument during annotations evaluation. -""" - -_TypeVarLike: TypeAlias = 'TypeVar | ParamSpec | TypeVarTuple' - - -class NamespacesTuple(NamedTuple): - """A tuple of globals and locals to be used during annotations evaluation. - - This datastructure is defined as a named tuple so that it can easily be unpacked: - - ```python {lint="skip" test="skip"} - def eval_type(typ: type[Any], ns: NamespacesTuple) -> None: - return eval(typ, *ns) - ``` - """ - - globals: GlobalsNamespace - """The namespace to be used as the `globals` argument during annotations evaluation.""" - - locals: MappingNamespace - """The namespace to be used as the `locals` argument during annotations evaluation.""" - - -def get_module_ns_of(obj: Any) -> dict[str, Any]: - """Get the namespace of the module where the object is defined. - - Caution: this function does not return a copy of the module namespace, so the result - should not be mutated. The burden of enforcing this is on the caller. - """ - module_name = getattr(obj, '__module__', None) - if module_name: - try: - return sys.modules[module_name].__dict__ - except KeyError: - # happens occasionally, see https://github.com/pydantic/pydantic/issues/2363 - return {} - return {} - - -# Note that this class is almost identical to `collections.ChainMap`, but need to enforce -# immutable mappings here: -class LazyLocalNamespace(Mapping[str, Any]): - """A lazily evaluated mapping, to be used as the `locals` argument during annotations evaluation. - - While the [`eval`][eval] function expects a mapping as the `locals` argument, it only - performs `__getitem__` calls. The [`Mapping`][collections.abc.Mapping] abstract base class - is fully implemented only for type checking purposes. - - Args: - *namespaces: The namespaces to consider, in ascending order of priority. - - Example: - ```python {lint="skip" test="skip"} - ns = LazyLocalNamespace({'a': 1, 'b': 2}, {'a': 3}) - ns['a'] - #> 3 - ns['b'] - #> 2 - ``` - """ - - def __init__(self, *namespaces: MappingNamespace) -> None: - self._namespaces = namespaces - - @cached_property - def data(self) -> dict[str, Any]: - return {k: v for ns in self._namespaces for k, v in ns.items()} - - def __len__(self) -> int: - return len(self.data) - - def __getitem__(self, key: str) -> Any: - return self.data[key] - - def __contains__(self, key: object) -> bool: - return key in self.data - - def __iter__(self) -> Iterator[str]: - return iter(self.data) - - -def ns_for_function(obj: Callable[..., Any], parent_namespace: MappingNamespace | None = None) -> NamespacesTuple: - """Return the global and local namespaces to be used when evaluating annotations for the provided function. - - The global namespace will be the `__dict__` attribute of the module the function was defined in. - The local namespace will contain the `__type_params__` introduced by PEP 695. - - Args: - obj: The object to use when building namespaces. - parent_namespace: Optional namespace to be added with the lowest priority in the local namespace. - If the passed function is a method, the `parent_namespace` will be the namespace of the class - the method is defined in. Thus, we also fetch type `__type_params__` from there (i.e. the - class-scoped type variables). - """ - locals_list: list[MappingNamespace] = [] - if parent_namespace is not None: - locals_list.append(parent_namespace) - - # Get the `__type_params__` attribute introduced by PEP 695. - # Note that the `typing._eval_type` function expects type params to be - # passed as a separate argument. However, internally, `_eval_type` calls - # `ForwardRef._evaluate` which will merge type params with the localns, - # essentially mimicking what we do here. - type_params: tuple[_TypeVarLike, ...] = getattr(obj, '__type_params__', ()) - if parent_namespace is not None: - # We also fetch type params from the parent namespace. If present, it probably - # means the function was defined in a class. This is to support the following: - # https://github.com/python/cpython/issues/124089. - type_params += parent_namespace.get('__type_params__', ()) - - locals_list.append({t.__name__: t for t in type_params}) - - # What about short-circuiting to `obj.__globals__`? - globalns = get_module_ns_of(obj) - - return NamespacesTuple(globalns, LazyLocalNamespace(*locals_list)) - - -class NsResolver: - """A class responsible for the namespaces resolving logic for annotations evaluation. - - This class handles the namespace logic when evaluating annotations mainly for class objects. - - It holds a stack of classes that are being inspected during the core schema building, - and the `types_namespace` property exposes the globals and locals to be used for - type annotation evaluation. Additionally -- if no class is present in the stack -- a - fallback globals and locals can be provided using the `namespaces_tuple` argument - (this is useful when generating a schema for a simple annotation, e.g. when using - `TypeAdapter`). - - The namespace creation logic is unfortunately flawed in some cases, for backwards - compatibility reasons and to better support valid edge cases. See the description - for the `parent_namespace` argument and the example for more details. - - Args: - namespaces_tuple: The default globals and locals to use if no class is present - on the stack. This can be useful when using the `GenerateSchema` class - with `TypeAdapter`, where the "type" being analyzed is a simple annotation. - parent_namespace: An optional parent namespace that will be added to the locals - with the lowest priority. For a given class defined in a function, the locals - of this function are usually used as the parent namespace: - - ```python {lint="skip" test="skip"} - from pydantic import BaseModel - - def func() -> None: - SomeType = int - - class Model(BaseModel): - f: 'SomeType' - - # when collecting fields, an namespace resolver instance will be created - # this way: - # ns_resolver = NsResolver(parent_namespace={'SomeType': SomeType}) - ``` - - For backwards compatibility reasons and to support valid edge cases, this parent - namespace will be used for *every* type being pushed to the stack. In the future, - we might want to be smarter by only doing so when the type being pushed is defined - in the same module as the parent namespace. - - Example: - ```python {lint="skip" test="skip"} - ns_resolver = NsResolver( - parent_namespace={'fallback': 1}, - ) - - class Sub: - m: 'Model' - - class Model: - some_local = 1 - sub: Sub - - ns_resolver = NsResolver() - - # This is roughly what happens when we build a core schema for `Model`: - with ns_resolver.push(Model): - ns_resolver.types_namespace - #> NamespacesTuple({'Sub': Sub}, {'Model': Model, 'some_local': 1}) - # First thing to notice here, the model being pushed is added to the locals. - # Because `NsResolver` is being used during the model definition, it is not - # yet added to the globals. This is useful when resolving self-referencing annotations. - - with ns_resolver.push(Sub): - ns_resolver.types_namespace - #> NamespacesTuple({'Sub': Sub}, {'Sub': Sub, 'Model': Model}) - # Second thing to notice: `Sub` is present in both the globals and locals. - # This is not an issue, just that as described above, the model being pushed - # is added to the locals, but it happens to be present in the globals as well - # because it is already defined. - # Third thing to notice: `Model` is also added in locals. This is a backwards - # compatibility workaround that allows for `Sub` to be able to resolve `'Model'` - # correctly (as otherwise models would have to be rebuilt even though this - # doesn't look necessary). - ``` - """ - - def __init__( - self, - namespaces_tuple: NamespacesTuple | None = None, - parent_namespace: MappingNamespace | None = None, - ) -> None: - self._base_ns_tuple = namespaces_tuple or NamespacesTuple({}, {}) - self._parent_ns = parent_namespace - self._types_stack: list[type[Any] | TypeAliasType] = [] - - @cached_property - def types_namespace(self) -> NamespacesTuple: - """The current global and local namespaces to be used for annotations evaluation.""" - if not self._types_stack: - # TODO: should we merge the parent namespace here? - # This is relevant for TypeAdapter, where there are no types on the stack, and we might - # need access to the parent_ns. Right now, we sidestep this in `type_adapter.py` by passing - # locals to both parent_ns and the base_ns_tuple, but this is a bit hacky. - # we might consider something like: - # if self._parent_ns is not None: - # # Hacky workarounds, see class docstring: - # # An optional parent namespace that will be added to the locals with the lowest priority - # locals_list: list[MappingNamespace] = [self._parent_ns, self._base_ns_tuple.locals] - # return NamespacesTuple(self._base_ns_tuple.globals, LazyLocalNamespace(*locals_list)) - return self._base_ns_tuple - - typ = self._types_stack[-1] - - globalns = get_module_ns_of(typ) - - locals_list: list[MappingNamespace] = [] - # Hacky workarounds, see class docstring: - # An optional parent namespace that will be added to the locals with the lowest priority - if self._parent_ns is not None: - locals_list.append(self._parent_ns) - if len(self._types_stack) > 1: - first_type = self._types_stack[0] - locals_list.append({first_type.__name__: first_type}) - - # Adding `__type_params__` *before* `vars(typ)`, as the latter takes priority - # (see https://github.com/python/cpython/pull/120272). - # TODO `typ.__type_params__` when we drop support for Python 3.11: - type_params: tuple[_TypeVarLike, ...] = getattr(typ, '__type_params__', ()) - if type_params: - # Adding `__type_params__` is mostly useful for generic classes defined using - # PEP 695 syntax *and* using forward annotations (see the example in - # https://github.com/python/cpython/issues/114053). For TypeAliasType instances, - # it is way less common, but still required if using a string annotation in the alias - # value, e.g. `type A[T] = 'T'` (which is not necessary in most cases). - locals_list.append({t.__name__: t for t in type_params}) - - # TypeAliasType instances don't have a `__dict__` attribute, so the check - # is necessary: - if hasattr(typ, '__dict__'): - locals_list.append(vars(typ)) - - # The `len(self._types_stack) > 1` check above prevents this from being added twice: - locals_list.append({typ.__name__: typ}) - - return NamespacesTuple(globalns, LazyLocalNamespace(*locals_list)) - - @contextmanager - def push(self, typ: type[Any] | TypeAliasType, /) -> Generator[None]: - """Push a type to the stack.""" - self._types_stack.append(typ) - # Reset the cached property: - self.__dict__.pop('types_namespace', None) - try: - yield - finally: - self._types_stack.pop() - self.__dict__.pop('types_namespace', None) diff --git a/functions/python/data-processor/pydantic/_internal/_repr.py b/functions/python/data-processor/pydantic/_internal/_repr.py deleted file mode 100644 index 7e80a9c83..000000000 --- a/functions/python/data-processor/pydantic/_internal/_repr.py +++ /dev/null @@ -1,124 +0,0 @@ -"""Tools to provide pretty/human-readable display of objects.""" - -from __future__ import annotations as _annotations - -import types -from collections.abc import Callable, Collection, Generator, Iterable -from typing import TYPE_CHECKING, Any, ForwardRef, cast - -import typing_extensions -from typing_extensions import TypeAlias -from typing_inspection import typing_objects -from typing_inspection.introspection import is_union_origin - -from . import _typing_extra - -if TYPE_CHECKING: - # TODO remove type error comments when we drop support for Python 3.9 - ReprArgs: TypeAlias = Iterable[tuple[str | None, Any]] # pyright: ignore[reportGeneralTypeIssues] - RichReprResult: TypeAlias = Iterable[Any | tuple[Any] | tuple[str, Any] | tuple[str, Any, Any]] # pyright: ignore[reportGeneralTypeIssues] - - -class PlainRepr(str): - """String class where repr doesn't include quotes. Useful with Representation when you want to return a string - representation of something that is valid (or pseudo-valid) python. - """ - - def __repr__(self) -> str: - return str(self) - - -class Representation: - # Mixin to provide `__str__`, `__repr__`, and `__pretty__` and `__rich_repr__` methods. - # `__pretty__` is used by [devtools](https://python-devtools.helpmanual.io/). - # `__rich_repr__` is used by [rich](https://rich.readthedocs.io/en/stable/pretty.html). - # (this is not a docstring to avoid adding a docstring to classes which inherit from Representation) - - __slots__ = () - - def __repr_args__(self) -> ReprArgs: - """Returns the attributes to show in __str__, __repr__, and __pretty__ this is generally overridden. - - Can either return: - * name - value pairs, e.g.: `[('foo_name', 'foo'), ('bar_name', ['b', 'a', 'r'])]` - * or, just values, e.g.: `[(None, 'foo'), (None, ['b', 'a', 'r'])]` - """ - attrs_names = cast(Collection[str], self.__slots__) - if not attrs_names and hasattr(self, '__dict__'): - attrs_names = self.__dict__.keys() - attrs = ((s, getattr(self, s)) for s in attrs_names) - return [(a, v if v is not self else self.__repr_recursion__(v)) for a, v in attrs if v is not None] - - def __repr_name__(self) -> str: - """Name of the instance's class, used in __repr__.""" - return self.__class__.__name__ - - def __repr_recursion__(self, object: Any) -> str: - """Returns the string representation of a recursive object.""" - # This is copied over from the stdlib `pprint` module: - return f'' - - def __repr_str__(self, join_str: str) -> str: - return join_str.join(repr(v) if a is None else f'{a}={v!r}' for a, v in self.__repr_args__()) - - def __pretty__(self, fmt: Callable[[Any], Any], **kwargs: Any) -> Generator[Any]: - """Used by devtools (https://python-devtools.helpmanual.io/) to pretty print objects.""" - yield self.__repr_name__() + '(' - yield 1 - for name, value in self.__repr_args__(): - if name is not None: - yield name + '=' - yield fmt(value) - yield ',' - yield 0 - yield -1 - yield ')' - - def __rich_repr__(self) -> RichReprResult: - """Used by Rich (https://rich.readthedocs.io/en/stable/pretty.html) to pretty print objects.""" - for name, field_repr in self.__repr_args__(): - if name is None: - yield field_repr - else: - yield name, field_repr - - def __str__(self) -> str: - return self.__repr_str__(' ') - - def __repr__(self) -> str: - return f'{self.__repr_name__()}({self.__repr_str__(", ")})' - - -def display_as_type(obj: Any) -> str: - """Pretty representation of a type, should be as close as possible to the original type definition string. - - Takes some logic from `typing._type_repr`. - """ - if isinstance(obj, (types.FunctionType, types.BuiltinFunctionType)): - return obj.__name__ - elif obj is ...: - return '...' - elif isinstance(obj, Representation): - return repr(obj) - elif isinstance(obj, ForwardRef) or typing_objects.is_typealiastype(obj): - return str(obj) - - if not isinstance(obj, (_typing_extra.typing_base, _typing_extra.WithArgsTypes, type)): - obj = obj.__class__ - - if is_union_origin(typing_extensions.get_origin(obj)): - args = ', '.join(map(display_as_type, typing_extensions.get_args(obj))) - return f'Union[{args}]' - elif isinstance(obj, _typing_extra.WithArgsTypes): - if typing_objects.is_literal(typing_extensions.get_origin(obj)): - args = ', '.join(map(repr, typing_extensions.get_args(obj))) - else: - args = ', '.join(map(display_as_type, typing_extensions.get_args(obj))) - try: - return f'{obj.__qualname__}[{args}]' - except AttributeError: - return str(obj).replace('typing.', '').replace('typing_extensions.', '') # handles TypeAliasType in 3.12 - elif isinstance(obj, type): - return obj.__qualname__ - else: - return repr(obj).replace('typing.', '').replace('typing_extensions.', '') diff --git a/functions/python/data-processor/pydantic/_internal/_schema_gather.py b/functions/python/data-processor/pydantic/_internal/_schema_gather.py deleted file mode 100644 index 8e07b20e7..000000000 --- a/functions/python/data-processor/pydantic/_internal/_schema_gather.py +++ /dev/null @@ -1,212 +0,0 @@ -# pyright: reportTypedDictNotRequiredAccess=false, reportGeneralTypeIssues=false, reportArgumentType=false, reportAttributeAccessIssue=false -from __future__ import annotations - -from dataclasses import dataclass, field -from typing import TypedDict - -from pydantic_core.core_schema import ( - ComputedField, - CoreSchema, - DefinitionReferenceSchema, - SerSchema, - iter_union_choices, -) -from typing_extensions import TypeAlias - -AllSchemas: TypeAlias = 'CoreSchema | SerSchema | ComputedField' - - -class GatherResult(TypedDict): - """Schema traversing result.""" - - collected_references: dict[str, DefinitionReferenceSchema | None] - """The collected definition references. - - If a definition reference schema can be inlined, it means that there is - only one in the whole core schema. As such, it is stored as the value. - Otherwise, the value is set to `None`. - """ - - deferred_discriminator_schemas: list[CoreSchema] - """The list of core schemas having the discriminator application deferred.""" - - -class MissingDefinitionError(LookupError): - """A reference was pointing to a non-existing core schema.""" - - def __init__(self, schema_reference: str, /) -> None: - self.schema_reference = schema_reference - - -@dataclass -class GatherContext: - """The current context used during core schema traversing. - - Context instances should only be used during schema traversing. - """ - - definitions: dict[str, CoreSchema] - """The available definitions.""" - - deferred_discriminator_schemas: list[CoreSchema] = field(init=False, default_factory=list) - """The list of core schemas having the discriminator application deferred. - - Internally, these core schemas have a specific key set in the core metadata dict. - """ - - collected_references: dict[str, DefinitionReferenceSchema | None] = field(init=False, default_factory=dict) - """The collected definition references. - - If a definition reference schema can be inlined, it means that there is - only one in the whole core schema. As such, it is stored as the value. - Otherwise, the value is set to `None`. - - During schema traversing, definition reference schemas can be added as candidates, or removed - (by setting the value to `None`). - """ - - -def traverse_metadata(schema: AllSchemas, ctx: GatherContext) -> None: - meta = schema.get('metadata') - if meta is not None and 'pydantic_internal_union_discriminator' in meta: - ctx.deferred_discriminator_schemas.append(schema) # pyright: ignore[reportArgumentType] - - -def traverse_definition_ref(def_ref_schema: DefinitionReferenceSchema, ctx: GatherContext) -> None: - schema_ref = def_ref_schema['schema_ref'] - - if schema_ref not in ctx.collected_references: - definition = ctx.definitions.get(schema_ref) - if definition is None: - raise MissingDefinitionError(schema_ref) - - # The `'definition-ref'` schema was only encountered once, make it - # a candidate to be inlined: - ctx.collected_references[schema_ref] = def_ref_schema - traverse_schema(definition, ctx) - if 'serialization' in def_ref_schema: - traverse_schema(def_ref_schema['serialization'], ctx) - traverse_metadata(def_ref_schema, ctx) - else: - # The `'definition-ref'` schema was already encountered, meaning - # the previously encountered schema (and this one) can't be inlined: - ctx.collected_references[schema_ref] = None - - -def traverse_schema(schema: AllSchemas, context: GatherContext) -> None: - # TODO When we drop 3.9, use a match statement to get better type checking and remove - # file-level type ignore. - # (the `'type'` could also be fetched in every `if/elif` statement, but this alters performance). - schema_type = schema['type'] - - if schema_type == 'definition-ref': - traverse_definition_ref(schema, context) - # `traverse_definition_ref` handles the possible serialization and metadata schemas: - return - elif schema_type == 'definitions': - traverse_schema(schema['schema'], context) - for definition in schema['definitions']: - traverse_schema(definition, context) - elif schema_type in {'list', 'set', 'frozenset', 'generator'}: - if 'items_schema' in schema: - traverse_schema(schema['items_schema'], context) - elif schema_type == 'tuple': - if 'items_schema' in schema: - for s in schema['items_schema']: - traverse_schema(s, context) - elif schema_type == 'dict': - if 'keys_schema' in schema: - traverse_schema(schema['keys_schema'], context) - if 'values_schema' in schema: - traverse_schema(schema['values_schema'], context) - elif schema_type == 'union': - for choice in iter_union_choices(schema): - traverse_schema(choice, context) - elif schema_type == 'tagged-union': - for v in schema['choices'].values(): - traverse_schema(v, context) - elif schema_type == 'chain': - for step in schema['steps']: - traverse_schema(step, context) - elif schema_type == 'lax-or-strict': - traverse_schema(schema['lax_schema'], context) - traverse_schema(schema['strict_schema'], context) - elif schema_type == 'json-or-python': - traverse_schema(schema['json_schema'], context) - traverse_schema(schema['python_schema'], context) - elif schema_type in {'model-fields', 'typed-dict'}: - if 'extras_schema' in schema: - traverse_schema(schema['extras_schema'], context) - if 'computed_fields' in schema: - for s in schema['computed_fields']: - traverse_schema(s, context) - for s in schema['fields'].values(): - traverse_schema(s, context) - elif schema_type == 'dataclass-args': - if 'computed_fields' in schema: - for s in schema['computed_fields']: - traverse_schema(s, context) - for s in schema['fields']: - traverse_schema(s, context) - elif schema_type == 'arguments': - for s in schema['arguments_schema']: - traverse_schema(s['schema'], context) - if 'var_args_schema' in schema: - traverse_schema(schema['var_args_schema'], context) - if 'var_kwargs_schema' in schema: - traverse_schema(schema['var_kwargs_schema'], context) - elif schema_type == 'arguments-v3': - for s in schema['arguments_schema']: - traverse_schema(s['schema'], context) - elif schema_type == 'call': - traverse_schema(schema['arguments_schema'], context) - if 'return_schema' in schema: - traverse_schema(schema['return_schema'], context) - elif schema_type == 'computed-field': - traverse_schema(schema['return_schema'], context) - elif schema_type == 'function-before': - if 'schema' in schema: - traverse_schema(schema['schema'], context) - if 'json_schema_input_schema' in schema: - traverse_schema(schema['json_schema_input_schema'], context) - elif schema_type == 'function-plain': - # TODO duplicate schema types for serializers and validators, needs to be deduplicated. - if 'return_schema' in schema: - traverse_schema(schema['return_schema'], context) - if 'json_schema_input_schema' in schema: - traverse_schema(schema['json_schema_input_schema'], context) - elif schema_type == 'function-wrap': - # TODO duplicate schema types for serializers and validators, needs to be deduplicated. - if 'return_schema' in schema: - traverse_schema(schema['return_schema'], context) - if 'schema' in schema: - traverse_schema(schema['schema'], context) - if 'json_schema_input_schema' in schema: - traverse_schema(schema['json_schema_input_schema'], context) - else: - if 'schema' in schema: - traverse_schema(schema['schema'], context) - - if 'serialization' in schema: - traverse_schema(schema['serialization'], context) - traverse_metadata(schema, context) - - -def gather_schemas_for_cleaning(schema: CoreSchema, definitions: dict[str, CoreSchema]) -> GatherResult: - """Traverse the core schema and definitions and return the necessary information for schema cleaning. - - During the core schema traversing, any `'definition-ref'` schema is: - - - Validated: the reference must point to an existing definition. If this is not the case, a - `MissingDefinitionError` exception is raised. - - Stored in the context: the actual reference is stored in the context. Depending on whether - the `'definition-ref'` schema is encountered more that once, the schema itself is also - saved in the context to be inlined (i.e. replaced by the definition it points to). - """ - context = GatherContext(definitions) - traverse_schema(schema, context) - - return { - 'collected_references': context.collected_references, - 'deferred_discriminator_schemas': context.deferred_discriminator_schemas, - } diff --git a/functions/python/data-processor/pydantic/_internal/_schema_generation_shared.py b/functions/python/data-processor/pydantic/_internal/_schema_generation_shared.py deleted file mode 100644 index b231a82e1..000000000 --- a/functions/python/data-processor/pydantic/_internal/_schema_generation_shared.py +++ /dev/null @@ -1,125 +0,0 @@ -"""Types and utility functions used by various other internal tools.""" - -from __future__ import annotations - -from typing import TYPE_CHECKING, Any, Callable, Literal - -from pydantic_core import core_schema - -from ..annotated_handlers import GetCoreSchemaHandler, GetJsonSchemaHandler - -if TYPE_CHECKING: - from ..json_schema import GenerateJsonSchema, JsonSchemaValue - from ._core_utils import CoreSchemaOrField - from ._generate_schema import GenerateSchema - from ._namespace_utils import NamespacesTuple - - GetJsonSchemaFunction = Callable[[CoreSchemaOrField, GetJsonSchemaHandler], JsonSchemaValue] - HandlerOverride = Callable[[CoreSchemaOrField], JsonSchemaValue] - - -class GenerateJsonSchemaHandler(GetJsonSchemaHandler): - """JsonSchemaHandler implementation that doesn't do ref unwrapping by default. - - This is used for any Annotated metadata so that we don't end up with conflicting - modifications to the definition schema. - - Used internally by Pydantic, please do not rely on this implementation. - See `GetJsonSchemaHandler` for the handler API. - """ - - def __init__(self, generate_json_schema: GenerateJsonSchema, handler_override: HandlerOverride | None) -> None: - self.generate_json_schema = generate_json_schema - self.handler = handler_override or generate_json_schema.generate_inner - self.mode = generate_json_schema.mode - - def __call__(self, core_schema: CoreSchemaOrField, /) -> JsonSchemaValue: - return self.handler(core_schema) - - def resolve_ref_schema(self, maybe_ref_json_schema: JsonSchemaValue) -> JsonSchemaValue: - """Resolves `$ref` in the json schema. - - This returns the input json schema if there is no `$ref` in json schema. - - Args: - maybe_ref_json_schema: The input json schema that may contains `$ref`. - - Returns: - Resolved json schema. - - Raises: - LookupError: If it can't find the definition for `$ref`. - """ - if '$ref' not in maybe_ref_json_schema: - return maybe_ref_json_schema - ref = maybe_ref_json_schema['$ref'] - json_schema = self.generate_json_schema.get_schema_from_definitions(ref) - if json_schema is None: - raise LookupError( - f'Could not find a ref for {ref}.' - ' Maybe you tried to call resolve_ref_schema from within a recursive model?' - ) - return json_schema - - -class CallbackGetCoreSchemaHandler(GetCoreSchemaHandler): - """Wrapper to use an arbitrary function as a `GetCoreSchemaHandler`. - - Used internally by Pydantic, please do not rely on this implementation. - See `GetCoreSchemaHandler` for the handler API. - """ - - def __init__( - self, - handler: Callable[[Any], core_schema.CoreSchema], - generate_schema: GenerateSchema, - ref_mode: Literal['to-def', 'unpack'] = 'to-def', - ) -> None: - self._handler = handler - self._generate_schema = generate_schema - self._ref_mode = ref_mode - - def __call__(self, source_type: Any, /) -> core_schema.CoreSchema: - schema = self._handler(source_type) - if self._ref_mode == 'to-def': - ref = schema.get('ref') - if ref is not None: - return self._generate_schema.defs.create_definition_reference_schema(schema) - return schema - else: # ref_mode = 'unpack' - return self.resolve_ref_schema(schema) - - def _get_types_namespace(self) -> NamespacesTuple: - return self._generate_schema._types_namespace - - def generate_schema(self, source_type: Any, /) -> core_schema.CoreSchema: - return self._generate_schema.generate_schema(source_type) - - @property - def field_name(self) -> str | None: - return self._generate_schema.field_name_stack.get() - - def resolve_ref_schema(self, maybe_ref_schema: core_schema.CoreSchema) -> core_schema.CoreSchema: - """Resolves reference in the core schema. - - Args: - maybe_ref_schema: The input core schema that may contains reference. - - Returns: - Resolved core schema. - - Raises: - LookupError: If it can't find the definition for reference. - """ - if maybe_ref_schema['type'] == 'definition-ref': - ref = maybe_ref_schema['schema_ref'] - definition = self._generate_schema.defs.get_schema_from_ref(ref) - if definition is None: - raise LookupError( - f'Could not find a ref for {ref}.' - ' Maybe you tried to call resolve_ref_schema from within a recursive model?' - ) - return definition - elif maybe_ref_schema['type'] == 'definitions': - return self.resolve_ref_schema(maybe_ref_schema['schema']) - return maybe_ref_schema diff --git a/functions/python/data-processor/pydantic/_internal/_serializers.py b/functions/python/data-processor/pydantic/_internal/_serializers.py deleted file mode 100644 index a4058e009..000000000 --- a/functions/python/data-processor/pydantic/_internal/_serializers.py +++ /dev/null @@ -1,53 +0,0 @@ -from __future__ import annotations - -import collections -import collections.abc -import typing -from typing import Any - -from pydantic_core import PydanticOmit, core_schema - -SEQUENCE_ORIGIN_MAP: dict[Any, Any] = { - typing.Deque: collections.deque, # noqa: UP006 - collections.deque: collections.deque, - list: list, - typing.List: list, # noqa: UP006 - tuple: tuple, - typing.Tuple: tuple, # noqa: UP006 - set: set, - typing.AbstractSet: set, - typing.Set: set, # noqa: UP006 - frozenset: frozenset, - typing.FrozenSet: frozenset, # noqa: UP006 - typing.Sequence: list, - typing.MutableSequence: list, - typing.MutableSet: set, - # this doesn't handle subclasses of these - # parametrized typing.Set creates one of these - collections.abc.MutableSet: set, - collections.abc.Set: frozenset, -} - - -def serialize_sequence_via_list( - v: Any, handler: core_schema.SerializerFunctionWrapHandler, info: core_schema.SerializationInfo -) -> Any: - items: list[Any] = [] - - mapped_origin = SEQUENCE_ORIGIN_MAP.get(type(v), None) - if mapped_origin is None: - # we shouldn't hit this branch, should probably add a serialization error or something - return v - - for index, item in enumerate(v): - try: - v = handler(item, index) - except PydanticOmit: # noqa: PERF203 - pass - else: - items.append(v) - - if info.mode_is_json(): - return items - else: - return mapped_origin(items) diff --git a/functions/python/data-processor/pydantic/_internal/_signature.py b/functions/python/data-processor/pydantic/_internal/_signature.py deleted file mode 100644 index 3b0c5ae8c..000000000 --- a/functions/python/data-processor/pydantic/_internal/_signature.py +++ /dev/null @@ -1,189 +0,0 @@ -from __future__ import annotations - -import dataclasses -from inspect import Parameter, Signature -from typing import TYPE_CHECKING, Any, Callable - -from pydantic_core import PydanticUndefined - -from ._typing_extra import signature_no_eval -from ._utils import is_valid_identifier - -if TYPE_CHECKING: - from ..config import ExtraValues - from ..fields import FieldInfo - - -# Copied over from stdlib dataclasses -class _HAS_DEFAULT_FACTORY_CLASS: - def __repr__(self): - return '' - - -_HAS_DEFAULT_FACTORY = _HAS_DEFAULT_FACTORY_CLASS() - - -def _field_name_for_signature(field_name: str, field_info: FieldInfo) -> str: - """Extract the correct name to use for the field when generating a signature. - - Assuming the field has a valid alias, this will return the alias. Otherwise, it will return the field name. - First priority is given to the alias, then the validation_alias, then the field name. - - Args: - field_name: The name of the field - field_info: The corresponding FieldInfo object. - - Returns: - The correct name to use when generating a signature. - """ - if isinstance(field_info.alias, str) and is_valid_identifier(field_info.alias): - return field_info.alias - if isinstance(field_info.validation_alias, str) and is_valid_identifier(field_info.validation_alias): - return field_info.validation_alias - - return field_name - - -def _process_param_defaults(param: Parameter) -> Parameter: - """Modify the signature for a parameter in a dataclass where the default value is a FieldInfo instance. - - Args: - param (Parameter): The parameter - - Returns: - Parameter: The custom processed parameter - """ - from ..fields import FieldInfo - - param_default = param.default - if isinstance(param_default, FieldInfo): - annotation = param.annotation - # Replace the annotation if appropriate - # inspect does "clever" things to show annotations as strings because we have - # `from __future__ import annotations` in main, we don't want that - if annotation == 'Any': - annotation = Any - - # Replace the field default - default = param_default.default - if default is PydanticUndefined: - if param_default.default_factory is None: - default = Signature.empty - else: - # this is used by dataclasses to indicate a factory exists: - default = dataclasses._HAS_DEFAULT_FACTORY # type: ignore - return param.replace( - annotation=annotation, name=_field_name_for_signature(param.name, param_default), default=default - ) - return param - - -def _generate_signature_parameters( # noqa: C901 (ignore complexity, could use a refactor) - init: Callable[..., None], - fields: dict[str, FieldInfo], - validate_by_name: bool, - extra: ExtraValues | None, -) -> dict[str, Parameter]: - """Generate a mapping of parameter names to Parameter objects for a pydantic BaseModel or dataclass.""" - from itertools import islice - - present_params = signature_no_eval(init).parameters.values() - merged_params: dict[str, Parameter] = {} - var_kw = None - use_var_kw = False - - for param in islice(present_params, 1, None): # skip self arg - # inspect does "clever" things to show annotations as strings because we have - # `from __future__ import annotations` in main, we don't want that - if fields.get(param.name): - # exclude params with init=False - if getattr(fields[param.name], 'init', True) is False: - continue - param = param.replace(name=_field_name_for_signature(param.name, fields[param.name])) - if param.annotation == 'Any': - param = param.replace(annotation=Any) - if param.kind is param.VAR_KEYWORD: - var_kw = param - continue - merged_params[param.name] = param - - if var_kw: # if custom init has no var_kw, fields which are not declared in it cannot be passed through - allow_names = validate_by_name - for field_name, field in fields.items(): - # when alias is a str it should be used for signature generation - param_name = _field_name_for_signature(field_name, field) - - if field_name in merged_params or param_name in merged_params: - continue - - if not is_valid_identifier(param_name): - if allow_names: - param_name = field_name - else: - use_var_kw = True - continue - - if field.is_required(): - default = Parameter.empty - elif field.default_factory is not None: - # Mimics stdlib dataclasses: - default = _HAS_DEFAULT_FACTORY - else: - default = field.default - merged_params[param_name] = Parameter( - param_name, - Parameter.KEYWORD_ONLY, - annotation=field.rebuild_annotation(), - default=default, - ) - - if extra == 'allow': - use_var_kw = True - - if var_kw and use_var_kw: - # Make sure the parameter for extra kwargs - # does not have the same name as a field - default_model_signature = [ - ('self', Parameter.POSITIONAL_ONLY), - ('data', Parameter.VAR_KEYWORD), - ] - if [(p.name, p.kind) for p in present_params] == default_model_signature: - # if this is the standard model signature, use extra_data as the extra args name - var_kw_name = 'extra_data' - else: - # else start from var_kw - var_kw_name = var_kw.name - - # generate a name that's definitely unique - while var_kw_name in fields: - var_kw_name += '_' - merged_params[var_kw_name] = var_kw.replace(name=var_kw_name) - - return merged_params - - -def generate_pydantic_signature( - init: Callable[..., None], - fields: dict[str, FieldInfo], - validate_by_name: bool, - extra: ExtraValues | None, - is_dataclass: bool = False, -) -> Signature: - """Generate signature for a pydantic BaseModel or dataclass. - - Args: - init: The class init. - fields: The model fields. - validate_by_name: The `validate_by_name` value of the config. - extra: The `extra` value of the config. - is_dataclass: Whether the model is a dataclass. - - Returns: - The dataclass/BaseModel subclass signature. - """ - merged_params = _generate_signature_parameters(init, fields, validate_by_name, extra) - - if is_dataclass: - merged_params = {k: _process_param_defaults(v) for k, v in merged_params.items()} - - return Signature(parameters=list(merged_params.values()), return_annotation=None) diff --git a/functions/python/data-processor/pydantic/_internal/_typing_extra.py b/functions/python/data-processor/pydantic/_internal/_typing_extra.py deleted file mode 100644 index c217f1127..000000000 --- a/functions/python/data-processor/pydantic/_internal/_typing_extra.py +++ /dev/null @@ -1,785 +0,0 @@ -"""Logic for interacting with type annotations, mostly extensions, shims and hacks to wrap Python's typing module.""" - -from __future__ import annotations - -import collections.abc -import re -import sys -import types -import typing -from functools import partial -from inspect import Signature, signature -from typing import TYPE_CHECKING, Any, Callable, cast - -import typing_extensions -from typing_extensions import deprecated, get_args, get_origin -from typing_inspection import typing_objects -from typing_inspection.introspection import is_union_origin - -from pydantic.version import version_short - -from ._namespace_utils import GlobalsNamespace, MappingNamespace, NsResolver, get_module_ns_of - -if sys.version_info < (3, 10): - NoneType = type(None) - EllipsisType = type(Ellipsis) -else: - from types import EllipsisType as EllipsisType - from types import NoneType as NoneType - -if sys.version_info >= (3, 14): - import annotationlib - -if TYPE_CHECKING: - from pydantic import BaseModel - from pydantic.fields import FieldInfo - -# As per https://typing-extensions.readthedocs.io/en/latest/#runtime-use-of-types, -# always check for both `typing` and `typing_extensions` variants of a typing construct. -# (this is implemented differently than the suggested approach in the `typing_extensions` -# docs for performance). - - -_t_annotated = typing.Annotated -_te_annotated = typing_extensions.Annotated - - -def is_annotated(tp: Any, /) -> bool: - """Return whether the provided argument is a `Annotated` special form. - - ```python {test="skip" lint="skip"} - is_annotated(Annotated[int, ...]) - #> True - ``` - """ - origin = get_origin(tp) - return origin is _t_annotated or origin is _te_annotated - - -def annotated_type(tp: Any, /) -> Any | None: - """Return the type of the `Annotated` special form, or `None`.""" - return tp.__origin__ if typing_objects.is_annotated(get_origin(tp)) else None - - -def unpack_type(tp: Any, /) -> Any | None: - """Return the type wrapped by the `Unpack` special form, or `None`.""" - return get_args(tp)[0] if typing_objects.is_unpack(get_origin(tp)) else None - - -def is_hashable(tp: Any, /) -> bool: - """Return whether the provided argument is the `Hashable` class. - - ```python {test="skip" lint="skip"} - is_hashable(Hashable) - #> True - ``` - """ - # `get_origin` is documented as normalizing any typing-module aliases to `collections` classes, - # hence the second check: - return tp is collections.abc.Hashable or get_origin(tp) is collections.abc.Hashable - - -def is_callable(tp: Any, /) -> bool: - """Return whether the provided argument is a `Callable`, parametrized or not. - - ```python {test="skip" lint="skip"} - is_callable(Callable[[int], str]) - #> True - is_callable(typing.Callable) - #> True - is_callable(collections.abc.Callable) - #> True - ``` - """ - # `get_origin` is documented as normalizing any typing-module aliases to `collections` classes, - # hence the second check: - return tp is collections.abc.Callable or get_origin(tp) is collections.abc.Callable - - -_classvar_re = re.compile(r'((\w+\.)?Annotated\[)?(\w+\.)?ClassVar\[') - - -def is_classvar_annotation(tp: Any, /) -> bool: - """Return whether the provided argument represents a class variable annotation. - - Although not explicitly stated by the typing specification, `ClassVar` can be used - inside `Annotated` and as such, this function checks for this specific scenario. - - Because this function is used to detect class variables before evaluating forward references - (or because evaluation failed), we also implement a naive regex match implementation. This is - required because class variables are inspected before fields are collected, so we try to be - as accurate as possible. - """ - if typing_objects.is_classvar(tp): - return True - - origin = get_origin(tp) - - if typing_objects.is_classvar(origin): - return True - - if typing_objects.is_annotated(origin): - annotated_type = tp.__origin__ - if typing_objects.is_classvar(annotated_type) or typing_objects.is_classvar(get_origin(annotated_type)): - return True - - str_ann: str | None = None - if isinstance(tp, typing.ForwardRef): - str_ann = tp.__forward_arg__ - if isinstance(tp, str): - str_ann = tp - - if str_ann is not None and _classvar_re.match(str_ann): - # stdlib dataclasses do something similar, although a bit more advanced - # (see `dataclass._is_type`). - return True - - return False - - -_t_final = typing.Final -_te_final = typing_extensions.Final - - -# TODO implement `is_finalvar_annotation` as Final can be wrapped with other special forms: -def is_finalvar(tp: Any, /) -> bool: - """Return whether the provided argument is a `Final` special form, parametrized or not. - - ```python {test="skip" lint="skip"} - is_finalvar(Final[int]) - #> True - is_finalvar(Final) - #> True - """ - # Final is not necessarily parametrized: - if tp is _t_final or tp is _te_final: - return True - origin = get_origin(tp) - return origin is _t_final or origin is _te_final - - -_NONE_TYPES: tuple[Any, ...] = (None, NoneType, typing.Literal[None], typing_extensions.Literal[None]) - - -def is_none_type(tp: Any, /) -> bool: - """Return whether the argument represents the `None` type as part of an annotation. - - ```python {test="skip" lint="skip"} - is_none_type(None) - #> True - is_none_type(NoneType) - #> True - is_none_type(Literal[None]) - #> True - is_none_type(type[None]) - #> False - """ - return tp in _NONE_TYPES - - -def is_namedtuple(tp: Any, /) -> bool: - """Return whether the provided argument is a named tuple class. - - The class can be created using `typing.NamedTuple` or `collections.namedtuple`. - Parametrized generic classes are *not* assumed to be named tuples. - """ - from ._utils import lenient_issubclass # circ. import - - return lenient_issubclass(tp, tuple) and hasattr(tp, '_fields') - - -# TODO In 2.12, delete this export. It is currently defined only to not break -# pydantic-settings which relies on it: -origin_is_union = is_union_origin - - -def is_generic_alias(tp: Any, /) -> bool: - return isinstance(tp, (types.GenericAlias, typing._GenericAlias)) # pyright: ignore[reportAttributeAccessIssue] - - -# TODO: Ideally, we should avoid relying on the private `typing` constructs: - -if sys.version_info < (3, 10): - WithArgsTypes: tuple[Any, ...] = (typing._GenericAlias, types.GenericAlias) # pyright: ignore[reportAttributeAccessIssue] -else: - WithArgsTypes: tuple[Any, ...] = (typing._GenericAlias, types.GenericAlias, types.UnionType) # pyright: ignore[reportAttributeAccessIssue] - - -# Similarly, we shouldn't rely on this `_Final` class, which is even more private than `_GenericAlias`: -typing_base: Any = typing._Final # pyright: ignore[reportAttributeAccessIssue] - - -### Annotation evaluations functions: - - -def parent_frame_namespace(*, parent_depth: int = 2, force: bool = False) -> dict[str, Any] | None: - """Fetch the local namespace of the parent frame where this function is called. - - Using this function is mostly useful to resolve forward annotations pointing to members defined in a local namespace, - such as assignments inside a function. Using the standard library tools, it is currently not possible to resolve - such annotations: - - ```python {lint="skip" test="skip"} - from typing import get_type_hints - - def func() -> None: - Alias = int - - class C: - a: 'Alias' - - # Raises a `NameError: 'Alias' is not defined` - get_type_hints(C) - ``` - - Pydantic uses this function when a Pydantic model is being defined to fetch the parent frame locals. However, - this only allows us to fetch the parent frame namespace and not other parents (e.g. a model defined in a function, - itself defined in another function). Inspecting the next outer frames (using `f_back`) is not reliable enough - (see https://discuss.python.org/t/20659). - - Because this function is mostly used to better resolve forward annotations, nothing is returned if the parent frame's - code object is defined at the module level. In this case, the locals of the frame will be the same as the module - globals where the class is defined (see `_namespace_utils.get_module_ns_of`). However, if you still want to fetch - the module globals (e.g. when rebuilding a model, where the frame where the rebuild call is performed might contain - members that you want to use for forward annotations evaluation), you can use the `force` parameter. - - Args: - parent_depth: The depth at which to get the frame. Defaults to 2, meaning the parent frame where this function - is called will be used. - force: Whether to always return the frame locals, even if the frame's code object is defined at the module level. - - Returns: - The locals of the namespace, or `None` if it was skipped as per the described logic. - """ - frame = sys._getframe(parent_depth) - - if frame.f_code.co_name.startswith('`, - # and we need to skip this frame as it is irrelevant. - frame = cast(types.FrameType, frame.f_back) # guaranteed to not be `None` - - # note, we don't copy frame.f_locals here (or during the last return call), because we don't expect the namespace to be - # modified down the line if this becomes a problem, we could implement some sort of frozen mapping structure to enforce this. - if force: - return frame.f_locals - - # If either of the following conditions are true, the class is defined at the top module level. - # To better understand why we need both of these checks, see - # https://github.com/pydantic/pydantic/pull/10113#discussion_r1714981531. - if frame.f_back is None or frame.f_code.co_name == '': - return None - - return frame.f_locals - - -def _type_convert(arg: Any) -> Any: - """Convert `None` to `NoneType` and strings to `ForwardRef` instances. - - This is a backport of the private `typing._type_convert` function. When - evaluating a type, `ForwardRef._evaluate` ends up being called, and is - responsible for making this conversion. However, we still have to apply - it for the first argument passed to our type evaluation functions, similarly - to the `typing.get_type_hints` function. - """ - if arg is None: - return NoneType - if isinstance(arg, str): - # Like `typing.get_type_hints`, assume the arg can be in any context, - # hence the proper `is_argument` and `is_class` args: - return _make_forward_ref(arg, is_argument=False, is_class=True) - return arg - - -def safe_get_annotations(obj: Any) -> dict[str, Any]: - """Get the annotations for the provided object, accounting for potential deferred forward references. - - Starting with Python 3.14, accessing the `__annotations__` attribute might raise a `NameError` if - a referenced symbol isn't defined yet. In this case, we return the annotation in the *forward ref* - format. - """ - if sys.version_info >= (3, 14): - return annotationlib.get_annotations(obj, format=annotationlib.Format.FORWARDREF) - else: - # TODO just do getattr(obj, '__annotations__', {}) when dropping support for Python 3.9: - if isinstance(obj, type): - return obj.__dict__.get('__annotations__', {}) - else: - return getattr(obj, '__annotations__', {}) - - -def get_model_type_hints( - model_class: type[BaseModel], - *, - ns_resolver: NsResolver | None = None, -) -> dict[str, tuple[Any, bool]]: - """Collect annotations from a Pydantic model class, including those from parent classes. - - Args: - model_class: The Pydantic model class to inspect. - ns_resolver: A namespace resolver instance to use. Defaults to an empty instance. - - Returns: - A dictionary mapping annotation names to a two-tuple: the first element is the evaluated - type or the original annotation if a `NameError` occurred, the second element is a boolean - indicating if whether the evaluation succeeded. - """ - hints: dict[str, Any] | dict[str, tuple[Any, bool]] = {} - ns_resolver = ns_resolver or NsResolver() - - for base in reversed(model_class.__mro__[:-1]): - # For Python 3.14, we could also use `Format.VALUE` and pass the globals/locals - # from the ns_resolver, but we want to be able to know which specific field failed - # to evaluate: - ann = safe_get_annotations(base) - - if not ann: - continue - - with ns_resolver.push(base): - base_model_fields: dict[str, FieldInfo] | None = base.__dict__.get('__pydantic_fields__') - - for name, value in ann.items(): - if name.startswith('_'): - globalns, localns = ns_resolver.types_namespace - - # For private attributes, we only need the annotation to detect the `ClassVar` special form. - # For this reason, we still try to evaluate it, but we also catch any possible exception (on - # top of the `NameError`s caught in `try_eval_type`) that could happen so that users are free - # to use any kind of forward annotation for private fields (e.g. circular imports, new typing - # syntax, etc). - try: - hints[name] = try_eval_type(value, globalns, localns) - except Exception: - hints[name] = (value, False) - else: - if base_model_fields is not None and name in base_model_fields: - # Avoid unnecessarily evaluating annotations from parent models, as we'll end up - # copying the `FieldInfo` instance from it anyway if we need to. - # We use the `annotation` attribute here, but in reality could put anything here, - # As we are guaranteed to not make use of it: - hints[name] = (base_model_fields[name].annotation, True) - else: - globalns, localns = ns_resolver.types_namespace - - hints[name] = try_eval_type(value, globalns, localns) - - return hints - - -def get_cls_type_hints( - obj: type[Any], - *, - ns_resolver: NsResolver | None = None, -) -> dict[str, Any]: - """Collect annotations from a class, including those from parent classes. - - Args: - obj: The class to inspect. - ns_resolver: A namespace resolver instance to use. Defaults to an empty instance. - """ - hints: dict[str, Any] = {} - ns_resolver = ns_resolver or NsResolver() - - for base in reversed(obj.__mro__): - # For Python 3.14, we could also use `Format.VALUE` and pass the globals/locals - # from the ns_resolver, but we want to be able to know which specific field failed - # to evaluate: - ann = safe_get_annotations(base) - - if not ann: - continue - - with ns_resolver.push(base): - globalns, localns = ns_resolver.types_namespace - for name, value in ann.items(): - hints[name] = eval_type(value, globalns, localns) - return hints - - -def try_eval_type( - value: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, -) -> tuple[Any, bool]: - """Try evaluating the annotation using the provided namespaces. - - Args: - value: The value to evaluate. If `None`, it will be replaced by `type[None]`. If an instance - of `str`, it will be converted to a `ForwardRef`. - localns: The global namespace to use during annotation evaluation. - globalns: The local namespace to use during annotation evaluation. - - Returns: - A two-tuple containing the possibly evaluated type and a boolean indicating - whether the evaluation succeeded or not. - """ - value = _type_convert(value) - - try: - return eval_type_backport(value, globalns, localns), True - except NameError: - return value, False - - -def eval_type( - value: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, -) -> Any: - """Evaluate the annotation using the provided namespaces. - - Args: - value: The value to evaluate. If `None`, it will be replaced by `type[None]`. If an instance - of `str`, it will be converted to a `ForwardRef`. - localns: The global namespace to use during annotation evaluation. - globalns: The local namespace to use during annotation evaluation. - """ - value = _type_convert(value) - return eval_type_backport(value, globalns, localns) - - -@deprecated( - '`eval_type_lenient` is deprecated, use `try_eval_type` instead.', - category=None, -) -def eval_type_lenient( - value: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, -) -> Any: - ev, _ = try_eval_type(value, globalns, localns) - return ev - - -def eval_type_backport( - value: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, - type_params: tuple[Any, ...] | None = None, -) -> Any: - """An enhanced version of `typing._eval_type` which will fall back to using the `eval_type_backport` - package if it's installed to let older Python versions use newer typing constructs. - - Specifically, this transforms `X | Y` into `typing.Union[X, Y]` and `list[X]` into `typing.List[X]` - (as well as all the types made generic in PEP 585) if the original syntax is not supported in the - current Python version. - - This function will also display a helpful error if the value passed fails to evaluate. - """ - try: - return _eval_type_backport(value, globalns, localns, type_params) - except TypeError as e: - if 'Unable to evaluate type annotation' in str(e): - raise - - # If it is a `TypeError` and value isn't a `ForwardRef`, it would have failed during annotation definition. - # Thus we assert here for type checking purposes: - assert isinstance(value, typing.ForwardRef) - - message = f'Unable to evaluate type annotation {value.__forward_arg__!r}.' - if sys.version_info >= (3, 11): - e.add_note(message) - raise - else: - raise TypeError(message) from e - except RecursionError as e: - # TODO ideally recursion errors should be checked in `eval_type` above, but `eval_type_backport` - # is used directly in some places. - message = ( - "If you made use of an implicit recursive type alias (e.g. `MyType = list['MyType']), " - 'consider using PEP 695 type aliases instead. For more details, refer to the documentation: ' - f'https://docs.pydantic.dev/{version_short()}/concepts/types/#named-recursive-types' - ) - if sys.version_info >= (3, 11): - e.add_note(message) - raise - else: - raise RecursionError(f'{e.args[0]}\n{message}') - - -def _eval_type_backport( - value: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, - type_params: tuple[Any, ...] | None = None, -) -> Any: - try: - return _eval_type(value, globalns, localns, type_params) - except TypeError as e: - if not (isinstance(value, typing.ForwardRef) and is_backport_fixable_error(e)): - raise - - try: - from eval_type_backport import eval_type_backport - except ImportError: - raise TypeError( - f'Unable to evaluate type annotation {value.__forward_arg__!r}. If you are making use ' - 'of the new typing syntax (unions using `|` since Python 3.10 or builtins subscripting ' - 'since Python 3.9), you should either replace the use of new syntax with the existing ' - '`typing` constructs or install the `eval_type_backport` package.' - ) from e - - return eval_type_backport( - value, - globalns, - localns, # pyright: ignore[reportArgumentType], waiting on a new `eval_type_backport` release. - try_default=False, - ) - - -def _eval_type( - value: Any, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, - type_params: tuple[Any, ...] | None = None, -) -> Any: - if sys.version_info >= (3, 14): - # Starting in 3.14, `_eval_type()` does *not* apply `_type_convert()` - # anymore. This means the `None` -> `type(None)` conversion does not apply: - evaluated = typing._eval_type( # type: ignore - value, - globalns, - localns, - type_params=type_params, - # This is relevant when evaluating types from `TypedDict` classes, where string annotations - # are automatically converted to `ForwardRef` instances with a module set. In this case, - # Our `globalns` is irrelevant and we need to indicate `typing._eval_type()` that it should - # infer it from the `ForwardRef.__forward_module__` attribute instead (`typing.get_type_hints()` - # does the same). Note that this would probably be unnecessary if we properly iterated over the - # `__orig_bases__` for TypedDicts in `get_cls_type_hints()`: - prefer_fwd_module=True, - ) - if evaluated is None: - evaluated = type(None) - return evaluated - elif sys.version_info >= (3, 13): - return typing._eval_type( # type: ignore - value, globalns, localns, type_params=type_params - ) - else: - return typing._eval_type( # type: ignore - value, globalns, localns - ) - - -def is_backport_fixable_error(e: TypeError) -> bool: - msg = str(e) - - return sys.version_info < (3, 10) and msg.startswith('unsupported operand type(s) for |: ') - - -def signature_no_eval(f: Callable[..., Any]) -> Signature: - """Get the signature of a callable without evaluating any annotations.""" - if sys.version_info >= (3, 14): - from annotationlib import Format - - return signature(f, annotation_format=Format.FORWARDREF) - else: - return signature(f) - - -def get_function_type_hints( - function: Callable[..., Any], - *, - include_keys: set[str] | None = None, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, -) -> dict[str, Any]: - """Return type hints for a function. - - This is similar to the `typing.get_type_hints` function, with a few differences: - - Support `functools.partial` by using the underlying `func` attribute. - - Do not wrap type annotation of a parameter with `Optional` if it has a default value of `None` - (related bug: https://github.com/python/cpython/issues/90353, only fixed in 3.11+). - """ - if isinstance(function, partial): - annotations = safe_get_annotations(function.func) - else: - annotations = safe_get_annotations(function) - - if globalns is None: - globalns = get_module_ns_of(function) - type_params: tuple[Any, ...] | None = None - if localns is None: - # If localns was specified, it is assumed to already contain type params. This is because - # Pydantic has more advanced logic to do so (see `_namespace_utils.ns_for_function`). - type_params = getattr(function, '__type_params__', ()) - - type_hints = {} - for name, value in annotations.items(): - if include_keys is not None and name not in include_keys: - continue - if value is None: - value = NoneType - elif isinstance(value, str): - value = _make_forward_ref(value) - - type_hints[name] = eval_type_backport(value, globalns, localns, type_params) - - return type_hints - - -# TODO use typing.ForwardRef directly when we stop supporting 3.9: -if sys.version_info < (3, 9, 8) or (3, 10) <= sys.version_info < (3, 10, 1): - - def _make_forward_ref( - arg: Any, - is_argument: bool = True, - *, - is_class: bool = False, - ) -> typing.ForwardRef: - """Wrapper for ForwardRef that accounts for the `is_class` argument missing in older versions. - The `module` argument is omitted as it breaks <3.9.8, =3.10.0 and isn't used in the calls below. - - See https://github.com/python/cpython/pull/28560 for some background. - The backport happened on 3.9.8, see: - https://github.com/pydantic/pydantic/discussions/6244#discussioncomment-6275458, - and on 3.10.1 for the 3.10 branch, see: - https://github.com/pydantic/pydantic/issues/6912 - - Implemented as EAFP with memory. - """ - return typing.ForwardRef(arg, is_argument) # pyright: ignore[reportCallIssue] - -else: - _make_forward_ref = typing.ForwardRef # pyright: ignore[reportAssignmentType] - - -if sys.version_info >= (3, 10): - get_type_hints = typing.get_type_hints - -else: - """ - For older versions of python, we have a custom implementation of `get_type_hints` which is a close as possible to - the implementation in CPython 3.10.8. - """ - - @typing.no_type_check - def get_type_hints( # noqa: C901 - obj: Any, - globalns: dict[str, Any] | None = None, - localns: dict[str, Any] | None = None, - include_extras: bool = False, - ) -> dict[str, Any]: # pragma: no cover - """Taken verbatim from python 3.10.8 unchanged, except: - * type annotations of the function definition above. - * prefixing `typing.` where appropriate - * Use `_make_forward_ref` instead of `typing.ForwardRef` to handle the `is_class` argument. - - https://github.com/python/cpython/blob/aaaf5174241496afca7ce4d4584570190ff972fe/Lib/typing.py#L1773-L1875 - - DO NOT CHANGE THIS METHOD UNLESS ABSOLUTELY NECESSARY. - ====================================================== - - Return type hints for an object. - - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, adds Optional[t] if a - default value equal to None is set and recursively replaces all - 'Annotated[T, ...]' with 'T' (unless 'include_extras=True'). - - The argument may be a module, class, method, or function. The annotations - are returned as a dictionary. For classes, annotations include also - inherited members. - - TypeError is raised if the argument is not of a type that can contain - annotations, and an empty dictionary is returned if no annotations are - present. - - BEWARE -- the behavior of globalns and localns is counterintuitive - (unless you are familiar with how eval() and exec() work). The - search order is locals first, then globals. - - - If no dict arguments are passed, an attempt is made to use the - globals from obj (or the respective module's globals for classes), - and these are also used as the locals. If the object does not appear - to have globals, an empty dictionary is used. For classes, the search - order is globals first then locals. - - - If one dict argument is passed, it is used for both globals and - locals. - - - If two dict arguments are passed, they specify globals and - locals, respectively. - """ - if getattr(obj, '__no_type_check__', None): - return {} - # Classes require a special treatment. - if isinstance(obj, type): - hints = {} - for base in reversed(obj.__mro__): - if globalns is None: - base_globals = getattr(sys.modules.get(base.__module__, None), '__dict__', {}) - else: - base_globals = globalns - ann = base.__dict__.get('__annotations__', {}) - if isinstance(ann, types.GetSetDescriptorType): - ann = {} - base_locals = dict(vars(base)) if localns is None else localns - if localns is None and globalns is None: - # This is surprising, but required. Before Python 3.10, - # get_type_hints only evaluated the globalns of - # a class. To maintain backwards compatibility, we reverse - # the globalns and localns order so that eval() looks into - # *base_globals* first rather than *base_locals*. - # This only affects ForwardRefs. - base_globals, base_locals = base_locals, base_globals - for name, value in ann.items(): - if value is None: - value = type(None) - if isinstance(value, str): - value = _make_forward_ref(value, is_argument=False, is_class=True) - - value = eval_type_backport(value, base_globals, base_locals) - hints[name] = value - if not include_extras and hasattr(typing, '_strip_annotations'): - return { - k: typing._strip_annotations(t) # type: ignore - for k, t in hints.items() - } - else: - return hints - - if globalns is None: - if isinstance(obj, types.ModuleType): - globalns = obj.__dict__ - else: - nsobj = obj - # Find globalns for the unwrapped object. - while hasattr(nsobj, '__wrapped__'): - nsobj = nsobj.__wrapped__ - globalns = getattr(nsobj, '__globals__', {}) - if localns is None: - localns = globalns - elif localns is None: - localns = globalns - hints = getattr(obj, '__annotations__', None) - if hints is None: - # Return empty annotations for something that _could_ have them. - if isinstance(obj, typing._allowed_types): # type: ignore - return {} - else: - raise TypeError(f'{obj!r} is not a module, class, method, or function.') - defaults = typing._get_defaults(obj) # type: ignore - hints = dict(hints) - for name, value in hints.items(): - if value is None: - value = type(None) - if isinstance(value, str): - # class-level forward refs were handled above, this must be either - # a module-level annotation or a function argument annotation - - value = _make_forward_ref( - value, - is_argument=not isinstance(obj, types.ModuleType), - is_class=False, - ) - value = eval_type_backport(value, globalns, localns) - if name in defaults and defaults[name] is None: - value = typing.Optional[value] - hints[name] = value - return hints if include_extras else {k: typing._strip_annotations(t) for k, t in hints.items()} # type: ignore diff --git a/functions/python/data-processor/pydantic/_internal/_utils.py b/functions/python/data-processor/pydantic/_internal/_utils.py deleted file mode 100644 index 7a0346d84..000000000 --- a/functions/python/data-processor/pydantic/_internal/_utils.py +++ /dev/null @@ -1,446 +0,0 @@ -"""Bucket of reusable internal utilities. - -This should be reduced as much as possible with functions only used in one place, moved to that place. -""" - -from __future__ import annotations as _annotations - -import dataclasses -import keyword -import sys -import warnings -import weakref -from collections import OrderedDict, defaultdict, deque -from collections.abc import Callable, Iterable, Mapping -from collections.abc import Set as AbstractSet -from copy import deepcopy -from functools import cached_property -from inspect import Parameter -from itertools import zip_longest -from types import BuiltinFunctionType, CodeType, FunctionType, GeneratorType, LambdaType, ModuleType -from typing import TYPE_CHECKING, Any, Generic, TypeVar, overload - -from pydantic_core import MISSING, PydanticUndefined -from typing_extensions import TypeAlias, TypeGuard, deprecated - -from pydantic import PydanticDeprecatedSince211 - -from . import _repr, _typing_extra -from ._import_utils import import_cached_base_model - -if TYPE_CHECKING: - # TODO remove type error comments when we drop support for Python 3.9 - MappingIntStrAny: TypeAlias = Mapping[int, Any] | Mapping[str, Any] # pyright: ignore[reportGeneralTypeIssues] - AbstractSetIntStr: TypeAlias = AbstractSet[int] | AbstractSet[str] # pyright: ignore[reportGeneralTypeIssues] - from ..main import BaseModel - - -# these are types that are returned unchanged by deepcopy -IMMUTABLE_NON_COLLECTIONS_TYPES: set[type[Any]] = { - int, - float, - complex, - str, - bool, - bytes, - type, - _typing_extra.NoneType, - FunctionType, - BuiltinFunctionType, - LambdaType, - weakref.ref, - CodeType, - # note: including ModuleType will differ from behaviour of deepcopy by not producing error. - # It might be not a good idea in general, but considering that this function used only internally - # against default values of fields, this will allow to actually have a field with module as default value - ModuleType, - NotImplemented.__class__, - Ellipsis.__class__, -} - -# these are types that if empty, might be copied with simple copy() instead of deepcopy() -BUILTIN_COLLECTIONS: set[type[Any]] = { - list, - set, - tuple, - frozenset, - dict, - OrderedDict, - defaultdict, - deque, -} - - -def can_be_positional(param: Parameter) -> bool: - """Return whether the parameter accepts a positional argument. - - ```python {test="skip" lint="skip"} - def func(a, /, b, *, c): - pass - - params = inspect.signature(func).parameters - can_be_positional(params['a']) - #> True - can_be_positional(params['b']) - #> True - can_be_positional(params['c']) - #> False - ``` - """ - return param.kind in (Parameter.POSITIONAL_ONLY, Parameter.POSITIONAL_OR_KEYWORD) - - -def sequence_like(v: Any) -> bool: - return isinstance(v, (list, tuple, set, frozenset, GeneratorType, deque)) - - -def lenient_isinstance(o: Any, class_or_tuple: type[Any] | tuple[type[Any], ...] | None) -> bool: # pragma: no cover - try: - return isinstance(o, class_or_tuple) # type: ignore[arg-type] - except TypeError: - return False - - -def lenient_issubclass(cls: Any, class_or_tuple: Any) -> bool: # pragma: no cover - try: - return isinstance(cls, type) and issubclass(cls, class_or_tuple) - except TypeError: - if isinstance(cls, _typing_extra.WithArgsTypes): - return False - raise # pragma: no cover - - -def is_model_class(cls: Any) -> TypeGuard[type[BaseModel]]: - """Returns true if cls is a _proper_ subclass of BaseModel, and provides proper type-checking, - unlike raw calls to lenient_issubclass. - """ - BaseModel = import_cached_base_model() - - return lenient_issubclass(cls, BaseModel) and cls is not BaseModel - - -def is_valid_identifier(identifier: str) -> bool: - """Checks that a string is a valid identifier and not a Python keyword. - :param identifier: The identifier to test. - :return: True if the identifier is valid. - """ - return identifier.isidentifier() and not keyword.iskeyword(identifier) - - -KeyType = TypeVar('KeyType') - - -def deep_update(mapping: dict[KeyType, Any], *updating_mappings: dict[KeyType, Any]) -> dict[KeyType, Any]: - updated_mapping = mapping.copy() - for updating_mapping in updating_mappings: - for k, v in updating_mapping.items(): - if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict): - updated_mapping[k] = deep_update(updated_mapping[k], v) - else: - updated_mapping[k] = v - return updated_mapping - - -def update_not_none(mapping: dict[Any, Any], **update: Any) -> None: - mapping.update({k: v for k, v in update.items() if v is not None}) - - -T = TypeVar('T') - - -def unique_list( - input_list: list[T] | tuple[T, ...], - *, - name_factory: Callable[[T], str] = str, -) -> list[T]: - """Make a list unique while maintaining order. - We update the list if another one with the same name is set - (e.g. model validator overridden in subclass). - """ - result: list[T] = [] - result_names: list[str] = [] - for v in input_list: - v_name = name_factory(v) - if v_name not in result_names: - result_names.append(v_name) - result.append(v) - else: - result[result_names.index(v_name)] = v - - return result - - -class ValueItems(_repr.Representation): - """Class for more convenient calculation of excluded or included fields on values.""" - - __slots__ = ('_items', '_type') - - def __init__(self, value: Any, items: AbstractSetIntStr | MappingIntStrAny) -> None: - items = self._coerce_items(items) - - if isinstance(value, (list, tuple)): - items = self._normalize_indexes(items, len(value)) # type: ignore - - self._items: MappingIntStrAny = items # type: ignore - - def is_excluded(self, item: Any) -> bool: - """Check if item is fully excluded. - - :param item: key or index of a value - """ - return self.is_true(self._items.get(item)) - - def is_included(self, item: Any) -> bool: - """Check if value is contained in self._items. - - :param item: key or index of value - """ - return item in self._items - - def for_element(self, e: int | str) -> AbstractSetIntStr | MappingIntStrAny | None: - """:param e: key or index of element on value - :return: raw values for element if self._items is dict and contain needed element - """ - item = self._items.get(e) # type: ignore - return item if not self.is_true(item) else None - - def _normalize_indexes(self, items: MappingIntStrAny, v_length: int) -> dict[int | str, Any]: - """:param items: dict or set of indexes which will be normalized - :param v_length: length of sequence indexes of which will be - - >>> self._normalize_indexes({0: True, -2: True, -1: True}, 4) - {0: True, 2: True, 3: True} - >>> self._normalize_indexes({'__all__': True}, 4) - {0: True, 1: True, 2: True, 3: True} - """ - normalized_items: dict[int | str, Any] = {} - all_items = None - for i, v in items.items(): - if not (isinstance(v, Mapping) or isinstance(v, AbstractSet) or self.is_true(v)): - raise TypeError(f'Unexpected type of exclude value for index "{i}" {v.__class__}') - if i == '__all__': - all_items = self._coerce_value(v) - continue - if not isinstance(i, int): - raise TypeError( - 'Excluding fields from a sequence of sub-models or dicts must be performed index-wise: ' - 'expected integer keys or keyword "__all__"' - ) - normalized_i = v_length + i if i < 0 else i - normalized_items[normalized_i] = self.merge(v, normalized_items.get(normalized_i)) - - if not all_items: - return normalized_items - if self.is_true(all_items): - for i in range(v_length): - normalized_items.setdefault(i, ...) - return normalized_items - for i in range(v_length): - normalized_item = normalized_items.setdefault(i, {}) - if not self.is_true(normalized_item): - normalized_items[i] = self.merge(all_items, normalized_item) - return normalized_items - - @classmethod - def merge(cls, base: Any, override: Any, intersect: bool = False) -> Any: - """Merge a `base` item with an `override` item. - - Both `base` and `override` are converted to dictionaries if possible. - Sets are converted to dictionaries with the sets entries as keys and - Ellipsis as values. - - Each key-value pair existing in `base` is merged with `override`, - while the rest of the key-value pairs are updated recursively with this function. - - Merging takes place based on the "union" of keys if `intersect` is - set to `False` (default) and on the intersection of keys if - `intersect` is set to `True`. - """ - override = cls._coerce_value(override) - base = cls._coerce_value(base) - if override is None: - return base - if cls.is_true(base) or base is None: - return override - if cls.is_true(override): - return base if intersect else override - - # intersection or union of keys while preserving ordering: - if intersect: - merge_keys = [k for k in base if k in override] + [k for k in override if k in base] - else: - merge_keys = list(base) + [k for k in override if k not in base] - - merged: dict[int | str, Any] = {} - for k in merge_keys: - merged_item = cls.merge(base.get(k), override.get(k), intersect=intersect) - if merged_item is not None: - merged[k] = merged_item - - return merged - - @staticmethod - def _coerce_items(items: AbstractSetIntStr | MappingIntStrAny) -> MappingIntStrAny: - if isinstance(items, Mapping): - pass - elif isinstance(items, AbstractSet): - items = dict.fromkeys(items, ...) # type: ignore - else: - class_name = getattr(items, '__class__', '???') - raise TypeError(f'Unexpected type of exclude value {class_name}') - return items # type: ignore - - @classmethod - def _coerce_value(cls, value: Any) -> Any: - if value is None or cls.is_true(value): - return value - return cls._coerce_items(value) - - @staticmethod - def is_true(v: Any) -> bool: - return v is True or v is ... - - def __repr_args__(self) -> _repr.ReprArgs: - return [(None, self._items)] - - -if TYPE_CHECKING: - - def LazyClassAttribute(name: str, get_value: Callable[[], T]) -> T: ... - -else: - - class LazyClassAttribute: - """A descriptor exposing an attribute only accessible on a class (hidden from instances). - - The attribute is lazily computed and cached during the first access. - """ - - def __init__(self, name: str, get_value: Callable[[], Any]) -> None: - self.name = name - self.get_value = get_value - - @cached_property - def value(self) -> Any: - return self.get_value() - - def __get__(self, instance: Any, owner: type[Any]) -> None: - if instance is None: - return self.value - raise AttributeError(f'{self.name!r} attribute of {owner.__name__!r} is class-only') - - -Obj = TypeVar('Obj') - - -def smart_deepcopy(obj: Obj) -> Obj: - """Return type as is for immutable built-in types - Use obj.copy() for built-in empty collections - Use copy.deepcopy() for non-empty collections and unknown objects. - """ - if obj is MISSING or obj is PydanticUndefined: - return obj # pyright: ignore[reportReturnType] - obj_type = obj.__class__ - if obj_type in IMMUTABLE_NON_COLLECTIONS_TYPES: - return obj # fastest case: obj is immutable and not collection therefore will not be copied anyway - try: - if not obj and obj_type in BUILTIN_COLLECTIONS: - # faster way for empty collections, no need to copy its members - return obj if obj_type is tuple else obj.copy() # tuple doesn't have copy method # type: ignore - except (TypeError, ValueError, RuntimeError): - # do we really dare to catch ALL errors? Seems a bit risky - pass - - return deepcopy(obj) # slowest way when we actually might need a deepcopy - - -_SENTINEL = object() - - -def all_identical(left: Iterable[Any], right: Iterable[Any]) -> bool: - """Check that the items of `left` are the same objects as those in `right`. - - >>> a, b = object(), object() - >>> all_identical([a, b, a], [a, b, a]) - True - >>> all_identical([a, b, [a]], [a, b, [a]]) # new list object, while "equal" is not "identical" - False - """ - for left_item, right_item in zip_longest(left, right, fillvalue=_SENTINEL): - if left_item is not right_item: - return False - return True - - -def get_first_not_none(a: Any, b: Any) -> Any: - """Return the first argument if it is not `None`, otherwise return the second argument.""" - return a if a is not None else b - - -@dataclasses.dataclass(frozen=True) -class SafeGetItemProxy: - """Wrapper redirecting `__getitem__` to `get` with a sentinel value as default - - This makes is safe to use in `operator.itemgetter` when some keys may be missing - """ - - # Define __slots__manually for performances - # @dataclasses.dataclass() only support slots=True in python>=3.10 - __slots__ = ('wrapped',) - - wrapped: Mapping[str, Any] - - def __getitem__(self, key: str, /) -> Any: - return self.wrapped.get(key, _SENTINEL) - - # required to pass the object to operator.itemgetter() instances due to a quirk of typeshed - # https://github.com/python/mypy/issues/13713 - # https://github.com/python/typeshed/pull/8785 - # Since this is typing-only, hide it in a typing.TYPE_CHECKING block - if TYPE_CHECKING: - - def __contains__(self, key: str, /) -> bool: - return self.wrapped.__contains__(key) - - -_ModelT = TypeVar('_ModelT', bound='BaseModel') -_RT = TypeVar('_RT') - - -class deprecated_instance_property(Generic[_ModelT, _RT]): - """A decorator exposing the decorated class method as a property, with a warning on instance access. - - This decorator takes a class method defined on the `BaseModel` class and transforms it into - an attribute. The attribute can be accessed on both the class and instances of the class. If accessed - via an instance, a deprecation warning is emitted stating that instance access will be removed in V3. - """ - - def __init__(self, fget: Callable[[type[_ModelT]], _RT], /) -> None: - # Note: fget should be a classmethod: - self.fget = fget - - @overload - def __get__(self, instance: None, objtype: type[_ModelT]) -> _RT: ... - @overload - @deprecated( - 'Accessing this attribute on the instance is deprecated, and will be removed in Pydantic V3. ' - 'Instead, you should access this attribute from the model class.', - category=None, - ) - def __get__(self, instance: _ModelT, objtype: type[_ModelT]) -> _RT: ... - def __get__(self, instance: _ModelT | None, objtype: type[_ModelT]) -> _RT: - if instance is not None: - # fmt: off - attr_name = ( - self.fget.__name__ - if sys.version_info >= (3, 10) - else self.fget.__func__.__name__ # pyright: ignore[reportFunctionMemberAccess] - ) - # fmt: on - warnings.warn( - f'Accessing the {attr_name!r} attribute on the instance is deprecated. ' - 'Instead, you should access this attribute from the model class.', - category=PydanticDeprecatedSince211, - stacklevel=2, - ) - return self.fget.__get__(instance, objtype)() diff --git a/functions/python/data-processor/pydantic/_internal/_validate_call.py b/functions/python/data-processor/pydantic/_internal/_validate_call.py deleted file mode 100644 index 49ccd4f2d..000000000 --- a/functions/python/data-processor/pydantic/_internal/_validate_call.py +++ /dev/null @@ -1,141 +0,0 @@ -from __future__ import annotations as _annotations - -import functools -import inspect -from collections.abc import Awaitable -from functools import partial -from typing import Any, Callable - -import pydantic_core - -from ..config import ConfigDict -from ..plugin._schema_validator import create_schema_validator -from ._config import ConfigWrapper -from ._generate_schema import GenerateSchema, ValidateCallSupportedTypes -from ._namespace_utils import MappingNamespace, NsResolver, ns_for_function -from ._typing_extra import signature_no_eval - - -def extract_function_name(func: ValidateCallSupportedTypes) -> str: - """Extract the name of a `ValidateCallSupportedTypes` object.""" - return f'partial({func.func.__name__})' if isinstance(func, functools.partial) else func.__name__ - - -def extract_function_qualname(func: ValidateCallSupportedTypes) -> str: - """Extract the qualname of a `ValidateCallSupportedTypes` object.""" - return f'partial({func.func.__qualname__})' if isinstance(func, functools.partial) else func.__qualname__ - - -def update_wrapper_attributes(wrapped: ValidateCallSupportedTypes, wrapper: Callable[..., Any]): - """Update the `wrapper` function with the attributes of the `wrapped` function. Return the updated function.""" - if inspect.iscoroutinefunction(wrapped): - - @functools.wraps(wrapped) - async def wrapper_function(*args, **kwargs): # type: ignore - return await wrapper(*args, **kwargs) - else: - - @functools.wraps(wrapped) - def wrapper_function(*args, **kwargs): - return wrapper(*args, **kwargs) - - # We need to manually update this because `partial` object has no `__name__` and `__qualname__`. - wrapper_function.__name__ = extract_function_name(wrapped) - wrapper_function.__qualname__ = extract_function_qualname(wrapped) - wrapper_function.raw_function = wrapped # type: ignore - - return wrapper_function - - -class ValidateCallWrapper: - """This is a wrapper around a function that validates the arguments passed to it, and optionally the return value.""" - - __slots__ = ( - 'function', - 'validate_return', - 'schema_type', - 'module', - 'qualname', - 'ns_resolver', - 'config_wrapper', - '__pydantic_complete__', - '__pydantic_validator__', - '__return_pydantic_validator__', - ) - - def __init__( - self, - function: ValidateCallSupportedTypes, - config: ConfigDict | None, - validate_return: bool, - parent_namespace: MappingNamespace | None, - ) -> None: - self.function = function - self.validate_return = validate_return - if isinstance(function, partial): - self.schema_type = function.func - self.module = function.func.__module__ - else: - self.schema_type = function - self.module = function.__module__ - self.qualname = extract_function_qualname(function) - - self.ns_resolver = NsResolver( - namespaces_tuple=ns_for_function(self.schema_type, parent_namespace=parent_namespace) - ) - self.config_wrapper = ConfigWrapper(config) - if not self.config_wrapper.defer_build: - self._create_validators() - else: - self.__pydantic_complete__ = False - - def _create_validators(self) -> None: - gen_schema = GenerateSchema(self.config_wrapper, self.ns_resolver) - schema = gen_schema.clean_schema(gen_schema.generate_schema(self.function)) - core_config = self.config_wrapper.core_config(title=self.qualname) - - self.__pydantic_validator__ = create_schema_validator( - schema, - self.schema_type, - self.module, - self.qualname, - 'validate_call', - core_config, - self.config_wrapper.plugin_settings, - ) - if self.validate_return: - signature = signature_no_eval(self.function) - return_type = signature.return_annotation if signature.return_annotation is not signature.empty else Any - gen_schema = GenerateSchema(self.config_wrapper, self.ns_resolver) - schema = gen_schema.clean_schema(gen_schema.generate_schema(return_type)) - validator = create_schema_validator( - schema, - self.schema_type, - self.module, - self.qualname, - 'validate_call', - core_config, - self.config_wrapper.plugin_settings, - ) - if inspect.iscoroutinefunction(self.function): - - async def return_val_wrapper(aw: Awaitable[Any]) -> None: - return validator.validate_python(await aw) - - self.__return_pydantic_validator__ = return_val_wrapper - else: - self.__return_pydantic_validator__ = validator.validate_python - else: - self.__return_pydantic_validator__ = None - - self.__pydantic_complete__ = True - - def __call__(self, *args: Any, **kwargs: Any) -> Any: - if not self.__pydantic_complete__: - self._create_validators() - - res = self.__pydantic_validator__.validate_python(pydantic_core.ArgsKwargs(args, kwargs)) - if self.__return_pydantic_validator__: - return self.__return_pydantic_validator__(res) - else: - return res diff --git a/functions/python/data-processor/pydantic/_internal/_validators.py b/functions/python/data-processor/pydantic/_internal/_validators.py deleted file mode 100644 index 3ad6c9279..000000000 --- a/functions/python/data-processor/pydantic/_internal/_validators.py +++ /dev/null @@ -1,534 +0,0 @@ -"""Validator functions for standard library types. - -Import of this module is deferred since it contains imports of many standard library modules. -""" - -from __future__ import annotations as _annotations - -import collections.abc -import math -import re -import typing -from collections.abc import Sequence -from decimal import Decimal -from fractions import Fraction -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from typing import Any, Callable, TypeVar, Union, cast -from zoneinfo import ZoneInfo, ZoneInfoNotFoundError - -import typing_extensions -from pydantic_core import PydanticCustomError, PydanticKnownError, core_schema -from typing_extensions import get_args, get_origin -from typing_inspection import typing_objects - -from pydantic._internal._import_utils import import_cached_field_info -from pydantic.errors import PydanticSchemaGenerationError - - -def sequence_validator( - input_value: Sequence[Any], - /, - validator: core_schema.ValidatorFunctionWrapHandler, -) -> Sequence[Any]: - """Validator for `Sequence` types, isinstance(v, Sequence) has already been called.""" - value_type = type(input_value) - - # We don't accept any plain string as a sequence - # Relevant issue: https://github.com/pydantic/pydantic/issues/5595 - if issubclass(value_type, (str, bytes)): - raise PydanticCustomError( - 'sequence_str', - "'{type_name}' instances are not allowed as a Sequence value", - {'type_name': value_type.__name__}, - ) - - # TODO: refactor sequence validation to validate with either a list or a tuple - # schema, depending on the type of the value. - # Additionally, we should be able to remove one of either this validator or the - # SequenceValidator in _std_types_schema.py (preferably this one, while porting over some logic). - # Effectively, a refactor for sequence validation is needed. - if value_type is tuple: - input_value = list(input_value) - - v_list = validator(input_value) - - # the rest of the logic is just re-creating the original type from `v_list` - if value_type is list: - return v_list - elif issubclass(value_type, range): - # return the list as we probably can't re-create the range - return v_list - elif value_type is tuple: - return tuple(v_list) - else: - # best guess at how to re-create the original type, more custom construction logic might be required - return value_type(v_list) # type: ignore[call-arg] - - -def import_string(value: Any) -> Any: - if isinstance(value, str): - try: - return _import_string_logic(value) - except ImportError as e: - raise PydanticCustomError('import_error', 'Invalid python path: {error}', {'error': str(e)}) from e - else: - # otherwise we just return the value and let the next validator do the rest of the work - return value - - -def _import_string_logic(dotted_path: str) -> Any: - """Inspired by uvicorn — dotted paths should include a colon before the final item if that item is not a module. - (This is necessary to distinguish between a submodule and an attribute when there is a conflict.). - - If the dotted path does not include a colon and the final item is not a valid module, importing as an attribute - rather than a submodule will be attempted automatically. - - So, for example, the following values of `dotted_path` result in the following returned values: - * 'collections': - * 'collections.abc': - * 'collections.abc:Mapping': - * `collections.abc.Mapping`: (though this is a bit slower than the previous line) - - An error will be raised under any of the following scenarios: - * `dotted_path` contains more than one colon (e.g., 'collections:abc:Mapping') - * the substring of `dotted_path` before the colon is not a valid module in the environment (e.g., '123:Mapping') - * the substring of `dotted_path` after the colon is not an attribute of the module (e.g., 'collections:abc123') - """ - from importlib import import_module - - components = dotted_path.strip().split(':') - if len(components) > 2: - raise ImportError(f"Import strings should have at most one ':'; received {dotted_path!r}") - attribute = None - if len(components) == 2: - attribute = components[1] - module_path = components[0] - if not module_path: - raise ImportError(f'Import strings should have a nonempty module name; received {dotted_path!r}') - - try: - module = import_module(module_path) - except ModuleNotFoundError: - if attribute is None and '.' in module_path: - # Try interpreting the final dotted segment as an attribute, not a submodule - maybe_module_path, maybe_attribute = module_path.rsplit('.', 1) - - try: - return _import_string_logic(f'{maybe_module_path}:{maybe_attribute}') - except ImportError: - pass - raise - - if attribute is not None: - try: - return getattr(module, attribute) - except AttributeError as e: - raise ImportError(f'cannot import name {attribute!r} from {module_path!r}') from e - else: - return module - - -def pattern_either_validator(input_value: Any, /) -> re.Pattern[Any]: - if isinstance(input_value, re.Pattern): - return input_value - elif isinstance(input_value, (str, bytes)): - # todo strict mode - return compile_pattern(input_value) # type: ignore - else: - raise PydanticCustomError('pattern_type', 'Input should be a valid pattern') - - -def pattern_str_validator(input_value: Any, /) -> re.Pattern[str]: - if isinstance(input_value, re.Pattern): - if isinstance(input_value.pattern, str): - return input_value - else: - raise PydanticCustomError('pattern_str_type', 'Input should be a string pattern') - elif isinstance(input_value, str): - return compile_pattern(input_value) - elif isinstance(input_value, bytes): - raise PydanticCustomError('pattern_str_type', 'Input should be a string pattern') - else: - raise PydanticCustomError('pattern_type', 'Input should be a valid pattern') - - -def pattern_bytes_validator(input_value: Any, /) -> re.Pattern[bytes]: - if isinstance(input_value, re.Pattern): - if isinstance(input_value.pattern, bytes): - return input_value - else: - raise PydanticCustomError('pattern_bytes_type', 'Input should be a bytes pattern') - elif isinstance(input_value, bytes): - return compile_pattern(input_value) - elif isinstance(input_value, str): - raise PydanticCustomError('pattern_bytes_type', 'Input should be a bytes pattern') - else: - raise PydanticCustomError('pattern_type', 'Input should be a valid pattern') - - -PatternType = TypeVar('PatternType', str, bytes) - - -def compile_pattern(pattern: PatternType) -> re.Pattern[PatternType]: - try: - return re.compile(pattern) - except re.error: - raise PydanticCustomError('pattern_regex', 'Input should be a valid regular expression') - - -def ip_v4_address_validator(input_value: Any, /) -> IPv4Address: - if isinstance(input_value, IPv4Address): - return input_value - - try: - return IPv4Address(input_value) - except ValueError: - raise PydanticCustomError('ip_v4_address', 'Input is not a valid IPv4 address') - - -def ip_v6_address_validator(input_value: Any, /) -> IPv6Address: - if isinstance(input_value, IPv6Address): - return input_value - - try: - return IPv6Address(input_value) - except ValueError: - raise PydanticCustomError('ip_v6_address', 'Input is not a valid IPv6 address') - - -def ip_v4_network_validator(input_value: Any, /) -> IPv4Network: - """Assume IPv4Network initialised with a default `strict` argument. - - See more: - https://docs.python.org/library/ipaddress.html#ipaddress.IPv4Network - """ - if isinstance(input_value, IPv4Network): - return input_value - - try: - return IPv4Network(input_value) - except ValueError: - raise PydanticCustomError('ip_v4_network', 'Input is not a valid IPv4 network') - - -def ip_v6_network_validator(input_value: Any, /) -> IPv6Network: - """Assume IPv6Network initialised with a default `strict` argument. - - See more: - https://docs.python.org/library/ipaddress.html#ipaddress.IPv6Network - """ - if isinstance(input_value, IPv6Network): - return input_value - - try: - return IPv6Network(input_value) - except ValueError: - raise PydanticCustomError('ip_v6_network', 'Input is not a valid IPv6 network') - - -def ip_v4_interface_validator(input_value: Any, /) -> IPv4Interface: - if isinstance(input_value, IPv4Interface): - return input_value - - try: - return IPv4Interface(input_value) - except ValueError: - raise PydanticCustomError('ip_v4_interface', 'Input is not a valid IPv4 interface') - - -def ip_v6_interface_validator(input_value: Any, /) -> IPv6Interface: - if isinstance(input_value, IPv6Interface): - return input_value - - try: - return IPv6Interface(input_value) - except ValueError: - raise PydanticCustomError('ip_v6_interface', 'Input is not a valid IPv6 interface') - - -def fraction_validator(input_value: Any, /) -> Fraction: - if isinstance(input_value, Fraction): - return input_value - - try: - return Fraction(input_value) - except ValueError: - raise PydanticCustomError('fraction_parsing', 'Input is not a valid fraction') - - -def forbid_inf_nan_check(x: Any) -> Any: - if not math.isfinite(x): - raise PydanticKnownError('finite_number') - return x - - -def _safe_repr(v: Any) -> int | float | str: - """The context argument for `PydanticKnownError` requires a number or str type, so we do a simple repr() coercion for types like timedelta. - - See tests/test_types.py::test_annotated_metadata_any_order for some context. - """ - if isinstance(v, (int, float, str)): - return v - return repr(v) - - -def greater_than_validator(x: Any, gt: Any) -> Any: - try: - if not (x > gt): - raise PydanticKnownError('greater_than', {'gt': _safe_repr(gt)}) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'gt' to supplied value {x}") - - -def greater_than_or_equal_validator(x: Any, ge: Any) -> Any: - try: - if not (x >= ge): - raise PydanticKnownError('greater_than_equal', {'ge': _safe_repr(ge)}) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'ge' to supplied value {x}") - - -def less_than_validator(x: Any, lt: Any) -> Any: - try: - if not (x < lt): - raise PydanticKnownError('less_than', {'lt': _safe_repr(lt)}) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'lt' to supplied value {x}") - - -def less_than_or_equal_validator(x: Any, le: Any) -> Any: - try: - if not (x <= le): - raise PydanticKnownError('less_than_equal', {'le': _safe_repr(le)}) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'le' to supplied value {x}") - - -def multiple_of_validator(x: Any, multiple_of: Any) -> Any: - try: - if x % multiple_of: - raise PydanticKnownError('multiple_of', {'multiple_of': _safe_repr(multiple_of)}) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'multiple_of' to supplied value {x}") - - -def min_length_validator(x: Any, min_length: Any) -> Any: - try: - if not (len(x) >= min_length): - raise PydanticKnownError( - 'too_short', {'field_type': 'Value', 'min_length': min_length, 'actual_length': len(x)} - ) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'min_length' to supplied value {x}") - - -def max_length_validator(x: Any, max_length: Any) -> Any: - try: - if len(x) > max_length: - raise PydanticKnownError( - 'too_long', - {'field_type': 'Value', 'max_length': max_length, 'actual_length': len(x)}, - ) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'max_length' to supplied value {x}") - - -def _extract_decimal_digits_info(decimal: Decimal) -> tuple[int, int]: - """Compute the total number of digits and decimal places for a given [`Decimal`][decimal.Decimal] instance. - - This function handles both normalized and non-normalized Decimal instances. - Example: Decimal('1.230') -> 4 digits, 3 decimal places - - Args: - decimal (Decimal): The decimal number to analyze. - - Returns: - tuple[int, int]: A tuple containing the number of decimal places and total digits. - - Though this could be divided into two separate functions, the logic is easier to follow if we couple the computation - of the number of decimals and digits together. - """ - try: - decimal_tuple = decimal.as_tuple() - - assert isinstance(decimal_tuple.exponent, int) - - exponent = decimal_tuple.exponent - num_digits = len(decimal_tuple.digits) - - if exponent >= 0: - # A positive exponent adds that many trailing zeros - # Ex: digit_tuple=(1, 2, 3), exponent=2 -> 12300 -> 0 decimal places, 5 digits - num_digits += exponent - decimal_places = 0 - else: - # If the absolute value of the negative exponent is larger than the - # number of digits, then it's the same as the number of digits, - # because it'll consume all the digits in digit_tuple and then - # add abs(exponent) - len(digit_tuple) leading zeros after the decimal point. - # Ex: digit_tuple=(1, 2, 3), exponent=-2 -> 1.23 -> 2 decimal places, 3 digits - # Ex: digit_tuple=(1, 2, 3), exponent=-4 -> 0.0123 -> 4 decimal places, 4 digits - decimal_places = abs(exponent) - num_digits = max(num_digits, decimal_places) - - return decimal_places, num_digits - except (AssertionError, AttributeError): - raise TypeError(f'Unable to extract decimal digits info from supplied value {decimal}') - - -def max_digits_validator(x: Any, max_digits: Any) -> Any: - try: - _, num_digits = _extract_decimal_digits_info(x) - _, normalized_num_digits = _extract_decimal_digits_info(x.normalize()) - if (num_digits > max_digits) and (normalized_num_digits > max_digits): - raise PydanticKnownError( - 'decimal_max_digits', - {'max_digits': max_digits}, - ) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'max_digits' to supplied value {x}") - - -def decimal_places_validator(x: Any, decimal_places: Any) -> Any: - try: - decimal_places_, _ = _extract_decimal_digits_info(x) - if decimal_places_ > decimal_places: - normalized_decimal_places, _ = _extract_decimal_digits_info(x.normalize()) - if normalized_decimal_places > decimal_places: - raise PydanticKnownError( - 'decimal_max_places', - {'decimal_places': decimal_places}, - ) - return x - except TypeError: - raise TypeError(f"Unable to apply constraint 'decimal_places' to supplied value {x}") - - -def deque_validator(input_value: Any, handler: core_schema.ValidatorFunctionWrapHandler) -> collections.deque[Any]: - return collections.deque(handler(input_value), maxlen=getattr(input_value, 'maxlen', None)) - - -def defaultdict_validator( - input_value: Any, handler: core_schema.ValidatorFunctionWrapHandler, default_default_factory: Callable[[], Any] -) -> collections.defaultdict[Any, Any]: - if isinstance(input_value, collections.defaultdict): - default_factory = input_value.default_factory - return collections.defaultdict(default_factory, handler(input_value)) - else: - return collections.defaultdict(default_default_factory, handler(input_value)) - - -def get_defaultdict_default_default_factory(values_source_type: Any) -> Callable[[], Any]: - FieldInfo = import_cached_field_info() - - values_type_origin = get_origin(values_source_type) - - def infer_default() -> Callable[[], Any]: - allowed_default_types: dict[Any, Any] = { - tuple: tuple, - collections.abc.Sequence: tuple, - collections.abc.MutableSequence: list, - list: list, - typing.Sequence: list, - set: set, - typing.MutableSet: set, - collections.abc.MutableSet: set, - collections.abc.Set: frozenset, - typing.MutableMapping: dict, - typing.Mapping: dict, - collections.abc.Mapping: dict, - collections.abc.MutableMapping: dict, - float: float, - int: int, - str: str, - bool: bool, - } - values_type = values_type_origin or values_source_type - instructions = 'set using `DefaultDict[..., Annotated[..., Field(default_factory=...)]]`' - if typing_objects.is_typevar(values_type): - - def type_var_default_factory() -> None: - raise RuntimeError( - 'Generic defaultdict cannot be used without a concrete value type or an' - ' explicit default factory, ' + instructions - ) - - return type_var_default_factory - elif values_type not in allowed_default_types: - # a somewhat subjective set of types that have reasonable default values - allowed_msg = ', '.join([t.__name__ for t in set(allowed_default_types.values())]) - raise PydanticSchemaGenerationError( - f'Unable to infer a default factory for keys of type {values_source_type}.' - f' Only {allowed_msg} are supported, other types require an explicit default factory' - ' ' + instructions - ) - return allowed_default_types[values_type] - - # Assume Annotated[..., Field(...)] - if typing_objects.is_annotated(values_type_origin): - field_info = next((v for v in get_args(values_source_type) if isinstance(v, FieldInfo)), None) - else: - field_info = None - if field_info and field_info.default_factory: - # Assume the default factory does not take any argument: - default_default_factory = cast(Callable[[], Any], field_info.default_factory) - else: - default_default_factory = infer_default() - return default_default_factory - - -def validate_str_is_valid_iana_tz(value: Any, /) -> ZoneInfo: - if isinstance(value, ZoneInfo): - return value - try: - return ZoneInfo(value) - except (ZoneInfoNotFoundError, ValueError, TypeError): - raise PydanticCustomError('zoneinfo_str', 'invalid timezone: {value}', {'value': value}) - - -NUMERIC_VALIDATOR_LOOKUP: dict[str, Callable] = { - 'gt': greater_than_validator, - 'ge': greater_than_or_equal_validator, - 'lt': less_than_validator, - 'le': less_than_or_equal_validator, - 'multiple_of': multiple_of_validator, - 'min_length': min_length_validator, - 'max_length': max_length_validator, - 'max_digits': max_digits_validator, - 'decimal_places': decimal_places_validator, -} - -IpType = Union[IPv4Address, IPv6Address, IPv4Network, IPv6Network, IPv4Interface, IPv6Interface] - -IP_VALIDATOR_LOOKUP: dict[type[IpType], Callable] = { - IPv4Address: ip_v4_address_validator, - IPv6Address: ip_v6_address_validator, - IPv4Network: ip_v4_network_validator, - IPv6Network: ip_v6_network_validator, - IPv4Interface: ip_v4_interface_validator, - IPv6Interface: ip_v6_interface_validator, -} - -MAPPING_ORIGIN_MAP: dict[Any, Any] = { - typing.DefaultDict: collections.defaultdict, # noqa: UP006 - collections.defaultdict: collections.defaultdict, - typing.OrderedDict: collections.OrderedDict, # noqa: UP006 - collections.OrderedDict: collections.OrderedDict, - typing_extensions.OrderedDict: collections.OrderedDict, - typing.Counter: collections.Counter, - collections.Counter: collections.Counter, - # this doesn't handle subclasses of these - typing.Mapping: dict, - typing.MutableMapping: dict, - # parametrized typing.{Mutable}Mapping creates one of these - collections.abc.Mapping: dict, - collections.abc.MutableMapping: dict, -} diff --git a/functions/python/data-processor/pydantic/_migration.py b/functions/python/data-processor/pydantic/_migration.py deleted file mode 100644 index b4ecd283a..000000000 --- a/functions/python/data-processor/pydantic/_migration.py +++ /dev/null @@ -1,316 +0,0 @@ -import sys -from typing import Any, Callable - -from pydantic.warnings import PydanticDeprecatedSince20 - -from .version import version_short - -MOVED_IN_V2 = { - 'pydantic.utils:version_info': 'pydantic.version:version_info', - 'pydantic.error_wrappers:ValidationError': 'pydantic:ValidationError', - 'pydantic.utils:to_camel': 'pydantic.alias_generators:to_pascal', - 'pydantic.utils:to_lower_camel': 'pydantic.alias_generators:to_camel', - 'pydantic:PyObject': 'pydantic.types:ImportString', - 'pydantic.types:PyObject': 'pydantic.types:ImportString', - 'pydantic.generics:GenericModel': 'pydantic.BaseModel', -} - -DEPRECATED_MOVED_IN_V2 = { - 'pydantic.tools:schema_of': 'pydantic.deprecated.tools:schema_of', - 'pydantic.tools:parse_obj_as': 'pydantic.deprecated.tools:parse_obj_as', - 'pydantic.tools:schema_json_of': 'pydantic.deprecated.tools:schema_json_of', - 'pydantic.json:pydantic_encoder': 'pydantic.deprecated.json:pydantic_encoder', - 'pydantic:validate_arguments': 'pydantic.deprecated.decorator:validate_arguments', - 'pydantic.json:custom_pydantic_encoder': 'pydantic.deprecated.json:custom_pydantic_encoder', - 'pydantic.json:timedelta_isoformat': 'pydantic.deprecated.json:timedelta_isoformat', - 'pydantic.decorator:validate_arguments': 'pydantic.deprecated.decorator:validate_arguments', - 'pydantic.class_validators:validator': 'pydantic.deprecated.class_validators:validator', - 'pydantic.class_validators:root_validator': 'pydantic.deprecated.class_validators:root_validator', - 'pydantic.config:BaseConfig': 'pydantic.deprecated.config:BaseConfig', - 'pydantic.config:Extra': 'pydantic.deprecated.config:Extra', -} - -REDIRECT_TO_V1 = { - f'pydantic.utils:{obj}': f'pydantic.v1.utils:{obj}' - for obj in ( - 'deep_update', - 'GetterDict', - 'lenient_issubclass', - 'lenient_isinstance', - 'is_valid_field', - 'update_not_none', - 'import_string', - 'Representation', - 'ROOT_KEY', - 'smart_deepcopy', - 'sequence_like', - ) -} - - -REMOVED_IN_V2 = { - 'pydantic:ConstrainedBytes', - 'pydantic:ConstrainedDate', - 'pydantic:ConstrainedDecimal', - 'pydantic:ConstrainedFloat', - 'pydantic:ConstrainedFrozenSet', - 'pydantic:ConstrainedInt', - 'pydantic:ConstrainedList', - 'pydantic:ConstrainedSet', - 'pydantic:ConstrainedStr', - 'pydantic:JsonWrapper', - 'pydantic:NoneBytes', - 'pydantic:NoneStr', - 'pydantic:NoneStrBytes', - 'pydantic:Protocol', - 'pydantic:Required', - 'pydantic:StrBytes', - 'pydantic:compiled', - 'pydantic.config:get_config', - 'pydantic.config:inherit_config', - 'pydantic.config:prepare_config', - 'pydantic:create_model_from_namedtuple', - 'pydantic:create_model_from_typeddict', - 'pydantic.dataclasses:create_pydantic_model_from_dataclass', - 'pydantic.dataclasses:make_dataclass_validator', - 'pydantic.dataclasses:set_validation', - 'pydantic.datetime_parse:parse_date', - 'pydantic.datetime_parse:parse_time', - 'pydantic.datetime_parse:parse_datetime', - 'pydantic.datetime_parse:parse_duration', - 'pydantic.error_wrappers:ErrorWrapper', - 'pydantic.errors:AnyStrMaxLengthError', - 'pydantic.errors:AnyStrMinLengthError', - 'pydantic.errors:ArbitraryTypeError', - 'pydantic.errors:BoolError', - 'pydantic.errors:BytesError', - 'pydantic.errors:CallableError', - 'pydantic.errors:ClassError', - 'pydantic.errors:ColorError', - 'pydantic.errors:ConfigError', - 'pydantic.errors:DataclassTypeError', - 'pydantic.errors:DateError', - 'pydantic.errors:DateNotInTheFutureError', - 'pydantic.errors:DateNotInThePastError', - 'pydantic.errors:DateTimeError', - 'pydantic.errors:DecimalError', - 'pydantic.errors:DecimalIsNotFiniteError', - 'pydantic.errors:DecimalMaxDigitsError', - 'pydantic.errors:DecimalMaxPlacesError', - 'pydantic.errors:DecimalWholeDigitsError', - 'pydantic.errors:DictError', - 'pydantic.errors:DurationError', - 'pydantic.errors:EmailError', - 'pydantic.errors:EnumError', - 'pydantic.errors:EnumMemberError', - 'pydantic.errors:ExtraError', - 'pydantic.errors:FloatError', - 'pydantic.errors:FrozenSetError', - 'pydantic.errors:FrozenSetMaxLengthError', - 'pydantic.errors:FrozenSetMinLengthError', - 'pydantic.errors:HashableError', - 'pydantic.errors:IPv4AddressError', - 'pydantic.errors:IPv4InterfaceError', - 'pydantic.errors:IPv4NetworkError', - 'pydantic.errors:IPv6AddressError', - 'pydantic.errors:IPv6InterfaceError', - 'pydantic.errors:IPv6NetworkError', - 'pydantic.errors:IPvAnyAddressError', - 'pydantic.errors:IPvAnyInterfaceError', - 'pydantic.errors:IPvAnyNetworkError', - 'pydantic.errors:IntEnumError', - 'pydantic.errors:IntegerError', - 'pydantic.errors:InvalidByteSize', - 'pydantic.errors:InvalidByteSizeUnit', - 'pydantic.errors:InvalidDiscriminator', - 'pydantic.errors:InvalidLengthForBrand', - 'pydantic.errors:JsonError', - 'pydantic.errors:JsonTypeError', - 'pydantic.errors:ListError', - 'pydantic.errors:ListMaxLengthError', - 'pydantic.errors:ListMinLengthError', - 'pydantic.errors:ListUniqueItemsError', - 'pydantic.errors:LuhnValidationError', - 'pydantic.errors:MissingDiscriminator', - 'pydantic.errors:MissingError', - 'pydantic.errors:NoneIsAllowedError', - 'pydantic.errors:NoneIsNotAllowedError', - 'pydantic.errors:NotDigitError', - 'pydantic.errors:NotNoneError', - 'pydantic.errors:NumberNotGeError', - 'pydantic.errors:NumberNotGtError', - 'pydantic.errors:NumberNotLeError', - 'pydantic.errors:NumberNotLtError', - 'pydantic.errors:NumberNotMultipleError', - 'pydantic.errors:PathError', - 'pydantic.errors:PathNotADirectoryError', - 'pydantic.errors:PathNotAFileError', - 'pydantic.errors:PathNotExistsError', - 'pydantic.errors:PatternError', - 'pydantic.errors:PyObjectError', - 'pydantic.errors:PydanticTypeError', - 'pydantic.errors:PydanticValueError', - 'pydantic.errors:SequenceError', - 'pydantic.errors:SetError', - 'pydantic.errors:SetMaxLengthError', - 'pydantic.errors:SetMinLengthError', - 'pydantic.errors:StrError', - 'pydantic.errors:StrRegexError', - 'pydantic.errors:StrictBoolError', - 'pydantic.errors:SubclassError', - 'pydantic.errors:TimeError', - 'pydantic.errors:TupleError', - 'pydantic.errors:TupleLengthError', - 'pydantic.errors:UUIDError', - 'pydantic.errors:UUIDVersionError', - 'pydantic.errors:UrlError', - 'pydantic.errors:UrlExtraError', - 'pydantic.errors:UrlHostError', - 'pydantic.errors:UrlHostTldError', - 'pydantic.errors:UrlPortError', - 'pydantic.errors:UrlSchemeError', - 'pydantic.errors:UrlSchemePermittedError', - 'pydantic.errors:UrlUserInfoError', - 'pydantic.errors:WrongConstantError', - 'pydantic.main:validate_model', - 'pydantic.networks:stricturl', - 'pydantic:parse_file_as', - 'pydantic:parse_raw_as', - 'pydantic:stricturl', - 'pydantic.tools:parse_file_as', - 'pydantic.tools:parse_raw_as', - 'pydantic.types:ConstrainedBytes', - 'pydantic.types:ConstrainedDate', - 'pydantic.types:ConstrainedDecimal', - 'pydantic.types:ConstrainedFloat', - 'pydantic.types:ConstrainedFrozenSet', - 'pydantic.types:ConstrainedInt', - 'pydantic.types:ConstrainedList', - 'pydantic.types:ConstrainedSet', - 'pydantic.types:ConstrainedStr', - 'pydantic.types:JsonWrapper', - 'pydantic.types:NoneBytes', - 'pydantic.types:NoneStr', - 'pydantic.types:NoneStrBytes', - 'pydantic.types:StrBytes', - 'pydantic.typing:evaluate_forwardref', - 'pydantic.typing:AbstractSetIntStr', - 'pydantic.typing:AnyCallable', - 'pydantic.typing:AnyClassMethod', - 'pydantic.typing:CallableGenerator', - 'pydantic.typing:DictAny', - 'pydantic.typing:DictIntStrAny', - 'pydantic.typing:DictStrAny', - 'pydantic.typing:IntStr', - 'pydantic.typing:ListStr', - 'pydantic.typing:MappingIntStrAny', - 'pydantic.typing:NoArgAnyCallable', - 'pydantic.typing:NoneType', - 'pydantic.typing:ReprArgs', - 'pydantic.typing:SetStr', - 'pydantic.typing:StrPath', - 'pydantic.typing:TupleGenerator', - 'pydantic.typing:WithArgsTypes', - 'pydantic.typing:all_literal_values', - 'pydantic.typing:display_as_type', - 'pydantic.typing:get_all_type_hints', - 'pydantic.typing:get_args', - 'pydantic.typing:get_origin', - 'pydantic.typing:get_sub_types', - 'pydantic.typing:is_callable_type', - 'pydantic.typing:is_classvar', - 'pydantic.typing:is_finalvar', - 'pydantic.typing:is_literal_type', - 'pydantic.typing:is_namedtuple', - 'pydantic.typing:is_new_type', - 'pydantic.typing:is_none_type', - 'pydantic.typing:is_typeddict', - 'pydantic.typing:is_typeddict_special', - 'pydantic.typing:is_union', - 'pydantic.typing:new_type_supertype', - 'pydantic.typing:resolve_annotations', - 'pydantic.typing:typing_base', - 'pydantic.typing:update_field_forward_refs', - 'pydantic.typing:update_model_forward_refs', - 'pydantic.utils:ClassAttribute', - 'pydantic.utils:DUNDER_ATTRIBUTES', - 'pydantic.utils:PyObjectStr', - 'pydantic.utils:ValueItems', - 'pydantic.utils:almost_equal_floats', - 'pydantic.utils:get_discriminator_alias_and_values', - 'pydantic.utils:get_model', - 'pydantic.utils:get_unique_discriminator_alias', - 'pydantic.utils:in_ipython', - 'pydantic.utils:is_valid_identifier', - 'pydantic.utils:path_type', - 'pydantic.utils:validate_field_name', - 'pydantic:validate_model', -} - - -def getattr_migration(module: str) -> Callable[[str], Any]: - """Implement PEP 562 for objects that were either moved or removed on the migration - to V2. - - Args: - module: The module name. - - Returns: - A callable that will raise an error if the object is not found. - """ - # This avoids circular import with errors.py. - from .errors import PydanticImportError - - def wrapper(name: str) -> object: - """Raise an error if the object is not found, or warn if it was moved. - - In case it was moved, it still returns the object. - - Args: - name: The object name. - - Returns: - The object. - """ - if name == '__path__': - raise AttributeError(f'module {module!r} has no attribute {name!r}') - - import warnings - - from ._internal._validators import import_string - - import_path = f'{module}:{name}' - if import_path in MOVED_IN_V2.keys(): - new_location = MOVED_IN_V2[import_path] - warnings.warn( - f'`{import_path}` has been moved to `{new_location}`.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return import_string(MOVED_IN_V2[import_path]) - if import_path in DEPRECATED_MOVED_IN_V2: - # skip the warning here because a deprecation warning will be raised elsewhere - return import_string(DEPRECATED_MOVED_IN_V2[import_path]) - if import_path in REDIRECT_TO_V1: - new_location = REDIRECT_TO_V1[import_path] - warnings.warn( - f'`{import_path}` has been removed. We are importing from `{new_location}` instead.' - 'See the migration guide for more details: https://docs.pydantic.dev/latest/migration/', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return import_string(REDIRECT_TO_V1[import_path]) - if import_path == 'pydantic:BaseSettings': - raise PydanticImportError( - '`BaseSettings` has been moved to the `pydantic-settings` package. ' - f'See https://docs.pydantic.dev/{version_short()}/migration/#basesettings-has-moved-to-pydantic-settings ' - 'for more details.' - ) - if import_path in REMOVED_IN_V2: - raise PydanticImportError(f'`{import_path}` has been removed in V2.') - globals: dict[str, Any] = sys.modules[module].__dict__ - if name in globals: - return globals[name] - raise AttributeError(f'module {module!r} has no attribute {name!r}') - - return wrapper diff --git a/functions/python/data-processor/pydantic/alias_generators.py b/functions/python/data-processor/pydantic/alias_generators.py deleted file mode 100644 index 0b7653f58..000000000 --- a/functions/python/data-processor/pydantic/alias_generators.py +++ /dev/null @@ -1,62 +0,0 @@ -"""Alias generators for converting between different capitalization conventions.""" - -import re - -__all__ = ('to_pascal', 'to_camel', 'to_snake') - -# TODO: in V3, change the argument names to be more descriptive -# Generally, don't only convert from snake_case, or name the functions -# more specifically like snake_to_camel. - - -def to_pascal(snake: str) -> str: - """Convert a snake_case string to PascalCase. - - Args: - snake: The string to convert. - - Returns: - The PascalCase string. - """ - camel = snake.title() - return re.sub('([0-9A-Za-z])_(?=[0-9A-Z])', lambda m: m.group(1), camel) - - -def to_camel(snake: str) -> str: - """Convert a snake_case string to camelCase. - - Args: - snake: The string to convert. - - Returns: - The converted camelCase string. - """ - # If the string is already in camelCase and does not contain a digit followed - # by a lowercase letter, return it as it is - if re.match('^[a-z]+[A-Za-z0-9]*$', snake) and not re.search(r'\d[a-z]', snake): - return snake - - camel = to_pascal(snake) - return re.sub('(^_*[A-Z])', lambda m: m.group(1).lower(), camel) - - -def to_snake(camel: str) -> str: - """Convert a PascalCase, camelCase, or kebab-case string to snake_case. - - Args: - camel: The string to convert. - - Returns: - The converted string in snake_case. - """ - # Handle the sequence of uppercase letters followed by a lowercase letter - snake = re.sub(r'([A-Z]+)([A-Z][a-z])', lambda m: f'{m.group(1)}_{m.group(2)}', camel) - # Insert an underscore between a lowercase letter and an uppercase letter - snake = re.sub(r'([a-z])([A-Z])', lambda m: f'{m.group(1)}_{m.group(2)}', snake) - # Insert an underscore between a digit and an uppercase letter - snake = re.sub(r'([0-9])([A-Z])', lambda m: f'{m.group(1)}_{m.group(2)}', snake) - # Insert an underscore between a lowercase letter and a digit - snake = re.sub(r'([a-z])([0-9])', lambda m: f'{m.group(1)}_{m.group(2)}', snake) - # Replace hyphens with underscores to handle kebab-case - snake = snake.replace('-', '_') - return snake.lower() diff --git a/functions/python/data-processor/pydantic/aliases.py b/functions/python/data-processor/pydantic/aliases.py deleted file mode 100644 index ac2273701..000000000 --- a/functions/python/data-processor/pydantic/aliases.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Support for alias configurations.""" - -from __future__ import annotations - -import dataclasses -from typing import Any, Callable, Literal - -from pydantic_core import PydanticUndefined - -from ._internal import _internal_dataclass - -__all__ = ('AliasGenerator', 'AliasPath', 'AliasChoices') - - -@dataclasses.dataclass(**_internal_dataclass.slots_true) -class AliasPath: - """!!! abstract "Usage Documentation" - [`AliasPath` and `AliasChoices`](../concepts/alias.md#aliaspath-and-aliaschoices) - - A data class used by `validation_alias` as a convenience to create aliases. - - Attributes: - path: A list of string or integer aliases. - """ - - path: list[int | str] - - def __init__(self, first_arg: str, *args: str | int) -> None: - self.path = [first_arg] + list(args) - - def convert_to_aliases(self) -> list[str | int]: - """Converts arguments to a list of string or integer aliases. - - Returns: - The list of aliases. - """ - return self.path - - def search_dict_for_path(self, d: dict) -> Any: - """Searches a dictionary for the path specified by the alias. - - Returns: - The value at the specified path, or `PydanticUndefined` if the path is not found. - """ - v = d - for k in self.path: - if isinstance(v, str): - # disallow indexing into a str, like for AliasPath('x', 0) and x='abc' - return PydanticUndefined - try: - v = v[k] - except (KeyError, IndexError, TypeError): - return PydanticUndefined - return v - - -@dataclasses.dataclass(**_internal_dataclass.slots_true) -class AliasChoices: - """!!! abstract "Usage Documentation" - [`AliasPath` and `AliasChoices`](../concepts/alias.md#aliaspath-and-aliaschoices) - - A data class used by `validation_alias` as a convenience to create aliases. - - Attributes: - choices: A list containing a string or `AliasPath`. - """ - - choices: list[str | AliasPath] - - def __init__(self, first_choice: str | AliasPath, *choices: str | AliasPath) -> None: - self.choices = [first_choice] + list(choices) - - def convert_to_aliases(self) -> list[list[str | int]]: - """Converts arguments to a list of lists containing string or integer aliases. - - Returns: - The list of aliases. - """ - aliases: list[list[str | int]] = [] - for c in self.choices: - if isinstance(c, AliasPath): - aliases.append(c.convert_to_aliases()) - else: - aliases.append([c]) - return aliases - - -@dataclasses.dataclass(**_internal_dataclass.slots_true) -class AliasGenerator: - """!!! abstract "Usage Documentation" - [Using an `AliasGenerator`](../concepts/alias.md#using-an-aliasgenerator) - - A data class used by `alias_generator` as a convenience to create various aliases. - - Attributes: - alias: A callable that takes a field name and returns an alias for it. - validation_alias: A callable that takes a field name and returns a validation alias for it. - serialization_alias: A callable that takes a field name and returns a serialization alias for it. - """ - - alias: Callable[[str], str] | None = None - validation_alias: Callable[[str], str | AliasPath | AliasChoices] | None = None - serialization_alias: Callable[[str], str] | None = None - - def _generate_alias( - self, - alias_kind: Literal['alias', 'validation_alias', 'serialization_alias'], - allowed_types: tuple[type[str] | type[AliasPath] | type[AliasChoices], ...], - field_name: str, - ) -> str | AliasPath | AliasChoices | None: - """Generate an alias of the specified kind. Returns None if the alias generator is None. - - Raises: - TypeError: If the alias generator produces an invalid type. - """ - alias = None - if alias_generator := getattr(self, alias_kind): - alias = alias_generator(field_name) - if alias and not isinstance(alias, allowed_types): - raise TypeError( - f'Invalid `{alias_kind}` type. `{alias_kind}` generator must produce one of `{allowed_types}`' - ) - return alias - - def generate_aliases(self, field_name: str) -> tuple[str | None, str | AliasPath | AliasChoices | None, str | None]: - """Generate `alias`, `validation_alias`, and `serialization_alias` for a field. - - Returns: - A tuple of three aliases - validation, alias, and serialization. - """ - alias = self._generate_alias('alias', (str,), field_name) - validation_alias = self._generate_alias('validation_alias', (str, AliasChoices, AliasPath), field_name) - serialization_alias = self._generate_alias('serialization_alias', (str,), field_name) - - return alias, validation_alias, serialization_alias # type: ignore diff --git a/functions/python/data-processor/pydantic/annotated_handlers.py b/functions/python/data-processor/pydantic/annotated_handlers.py deleted file mode 100644 index d0cb5d3d4..000000000 --- a/functions/python/data-processor/pydantic/annotated_handlers.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Type annotations to use with `__get_pydantic_core_schema__` and `__get_pydantic_json_schema__`.""" - -from __future__ import annotations as _annotations - -from typing import TYPE_CHECKING, Any, Union - -from pydantic_core import core_schema - -if TYPE_CHECKING: - from ._internal._namespace_utils import NamespacesTuple - from .json_schema import JsonSchemaMode, JsonSchemaValue - - CoreSchemaOrField = Union[ - core_schema.CoreSchema, - core_schema.ModelField, - core_schema.DataclassField, - core_schema.TypedDictField, - core_schema.ComputedField, - ] - -__all__ = 'GetJsonSchemaHandler', 'GetCoreSchemaHandler' - - -class GetJsonSchemaHandler: - """Handler to call into the next JSON schema generation function. - - Attributes: - mode: Json schema mode, can be `validation` or `serialization`. - """ - - mode: JsonSchemaMode - - def __call__(self, core_schema: CoreSchemaOrField, /) -> JsonSchemaValue: - """Call the inner handler and get the JsonSchemaValue it returns. - This will call the next JSON schema modifying function up until it calls - into `pydantic.json_schema.GenerateJsonSchema`, which will raise a - `pydantic.errors.PydanticInvalidForJsonSchema` error if it cannot generate - a JSON schema. - - Args: - core_schema: A `pydantic_core.core_schema.CoreSchema`. - - Returns: - JsonSchemaValue: The JSON schema generated by the inner JSON schema modify - functions. - """ - raise NotImplementedError - - def resolve_ref_schema(self, maybe_ref_json_schema: JsonSchemaValue, /) -> JsonSchemaValue: - """Get the real schema for a `{"$ref": ...}` schema. - If the schema given is not a `$ref` schema, it will be returned as is. - This means you don't have to check before calling this function. - - Args: - maybe_ref_json_schema: A JsonSchemaValue which may be a `$ref` schema. - - Raises: - LookupError: If the ref is not found. - - Returns: - JsonSchemaValue: A JsonSchemaValue that has no `$ref`. - """ - raise NotImplementedError - - -class GetCoreSchemaHandler: - """Handler to call into the next CoreSchema schema generation function.""" - - def __call__(self, source_type: Any, /) -> core_schema.CoreSchema: - """Call the inner handler and get the CoreSchema it returns. - This will call the next CoreSchema modifying function up until it calls - into Pydantic's internal schema generation machinery, which will raise a - `pydantic.errors.PydanticSchemaGenerationError` error if it cannot generate - a CoreSchema for the given source type. - - Args: - source_type: The input type. - - Returns: - CoreSchema: The `pydantic-core` CoreSchema generated. - """ - raise NotImplementedError - - def generate_schema(self, source_type: Any, /) -> core_schema.CoreSchema: - """Generate a schema unrelated to the current context. - Use this function if e.g. you are handling schema generation for a sequence - and want to generate a schema for its items. - Otherwise, you may end up doing something like applying a `min_length` constraint - that was intended for the sequence itself to its items! - - Args: - source_type: The input type. - - Returns: - CoreSchema: The `pydantic-core` CoreSchema generated. - """ - raise NotImplementedError - - def resolve_ref_schema(self, maybe_ref_schema: core_schema.CoreSchema, /) -> core_schema.CoreSchema: - """Get the real schema for a `definition-ref` schema. - If the schema given is not a `definition-ref` schema, it will be returned as is. - This means you don't have to check before calling this function. - - Args: - maybe_ref_schema: A `CoreSchema`, `ref`-based or not. - - Raises: - LookupError: If the `ref` is not found. - - Returns: - A concrete `CoreSchema`. - """ - raise NotImplementedError - - @property - def field_name(self) -> str | None: - """Get the name of the closest field to this validator.""" - raise NotImplementedError - - def _get_types_namespace(self) -> NamespacesTuple: - """Internal method used during type resolution for serializer annotations.""" - raise NotImplementedError diff --git a/functions/python/data-processor/pydantic/class_validators.py b/functions/python/data-processor/pydantic/class_validators.py deleted file mode 100644 index 9977150c9..000000000 --- a/functions/python/data-processor/pydantic/class_validators.py +++ /dev/null @@ -1,5 +0,0 @@ -"""`class_validators` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/color.py b/functions/python/data-processor/pydantic/color.py deleted file mode 100644 index 9a42d586f..000000000 --- a/functions/python/data-processor/pydantic/color.py +++ /dev/null @@ -1,604 +0,0 @@ -"""Color definitions are used as per the CSS3 -[CSS Color Module Level 3](http://www.w3.org/TR/css3-color/#svg-color) specification. - -A few colors have multiple names referring to the sames colors, eg. `grey` and `gray` or `aqua` and `cyan`. - -In these cases the _last_ color when sorted alphabetically takes preferences, -eg. `Color((0, 255, 255)).as_named() == 'cyan'` because "cyan" comes after "aqua". - -Warning: Deprecated - The `Color` class is deprecated, use `pydantic_extra_types` instead. - See [`pydantic-extra-types.Color`](../usage/types/extra_types/color_types.md) - for more information. -""" - -import math -import re -from colorsys import hls_to_rgb, rgb_to_hls -from typing import Any, Callable, Optional, Union, cast - -from pydantic_core import CoreSchema, PydanticCustomError, core_schema -from typing_extensions import deprecated - -from ._internal import _repr -from ._internal._schema_generation_shared import GetJsonSchemaHandler as _GetJsonSchemaHandler -from .json_schema import JsonSchemaValue -from .warnings import PydanticDeprecatedSince20 - -ColorTuple = Union[tuple[int, int, int], tuple[int, int, int, float]] -ColorType = Union[ColorTuple, str] -HslColorTuple = Union[tuple[float, float, float], tuple[float, float, float, float]] - - -class RGBA: - """Internal use only as a representation of a color.""" - - __slots__ = 'r', 'g', 'b', 'alpha', '_tuple' - - def __init__(self, r: float, g: float, b: float, alpha: Optional[float]): - self.r = r - self.g = g - self.b = b - self.alpha = alpha - - self._tuple: tuple[float, float, float, Optional[float]] = (r, g, b, alpha) - - def __getitem__(self, item: Any) -> Any: - return self._tuple[item] - - -# these are not compiled here to avoid import slowdown, they'll be compiled the first time they're used, then cached -_r_255 = r'(\d{1,3}(?:\.\d+)?)' -_r_comma = r'\s*,\s*' -_r_alpha = r'(\d(?:\.\d+)?|\.\d+|\d{1,2}%)' -_r_h = r'(-?\d+(?:\.\d+)?|-?\.\d+)(deg|rad|turn)?' -_r_sl = r'(\d{1,3}(?:\.\d+)?)%' -r_hex_short = r'\s*(?:#|0x)?([0-9a-f])([0-9a-f])([0-9a-f])([0-9a-f])?\s*' -r_hex_long = r'\s*(?:#|0x)?([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})?\s*' -# CSS3 RGB examples: rgb(0, 0, 0), rgba(0, 0, 0, 0.5), rgba(0, 0, 0, 50%) -r_rgb = rf'\s*rgba?\(\s*{_r_255}{_r_comma}{_r_255}{_r_comma}{_r_255}(?:{_r_comma}{_r_alpha})?\s*\)\s*' -# CSS3 HSL examples: hsl(270, 60%, 50%), hsla(270, 60%, 50%, 0.5), hsla(270, 60%, 50%, 50%) -r_hsl = rf'\s*hsla?\(\s*{_r_h}{_r_comma}{_r_sl}{_r_comma}{_r_sl}(?:{_r_comma}{_r_alpha})?\s*\)\s*' -# CSS4 RGB examples: rgb(0 0 0), rgb(0 0 0 / 0.5), rgb(0 0 0 / 50%), rgba(0 0 0 / 50%) -r_rgb_v4_style = rf'\s*rgba?\(\s*{_r_255}\s+{_r_255}\s+{_r_255}(?:\s*/\s*{_r_alpha})?\s*\)\s*' -# CSS4 HSL examples: hsl(270 60% 50%), hsl(270 60% 50% / 0.5), hsl(270 60% 50% / 50%), hsla(270 60% 50% / 50%) -r_hsl_v4_style = rf'\s*hsla?\(\s*{_r_h}\s+{_r_sl}\s+{_r_sl}(?:\s*/\s*{_r_alpha})?\s*\)\s*' - -# colors where the two hex characters are the same, if all colors match this the short version of hex colors can be used -repeat_colors = {int(c * 2, 16) for c in '0123456789abcdef'} -rads = 2 * math.pi - - -@deprecated( - 'The `Color` class is deprecated, use `pydantic_extra_types` instead. ' - 'See https://docs.pydantic.dev/latest/api/pydantic_extra_types_color/.', - category=PydanticDeprecatedSince20, -) -class Color(_repr.Representation): - """Represents a color.""" - - __slots__ = '_original', '_rgba' - - def __init__(self, value: ColorType) -> None: - self._rgba: RGBA - self._original: ColorType - if isinstance(value, (tuple, list)): - self._rgba = parse_tuple(value) - elif isinstance(value, str): - self._rgba = parse_str(value) - elif isinstance(value, Color): - self._rgba = value._rgba - value = value._original - else: - raise PydanticCustomError( - 'color_error', 'value is not a valid color: value must be a tuple, list or string' - ) - - # if we've got here value must be a valid color - self._original = value - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = {} - field_schema.update(type='string', format='color') - return field_schema - - def original(self) -> ColorType: - """Original value passed to `Color`.""" - return self._original - - def as_named(self, *, fallback: bool = False) -> str: - """Returns the name of the color if it can be found in `COLORS_BY_VALUE` dictionary, - otherwise returns the hexadecimal representation of the color or raises `ValueError`. - - Args: - fallback: If True, falls back to returning the hexadecimal representation of - the color instead of raising a ValueError when no named color is found. - - Returns: - The name of the color, or the hexadecimal representation of the color. - - Raises: - ValueError: When no named color is found and fallback is `False`. - """ - if self._rgba.alpha is None: - rgb = cast(tuple[int, int, int], self.as_rgb_tuple()) - try: - return COLORS_BY_VALUE[rgb] - except KeyError as e: - if fallback: - return self.as_hex() - else: - raise ValueError('no named color found, use fallback=True, as_hex() or as_rgb()') from e - else: - return self.as_hex() - - def as_hex(self) -> str: - """Returns the hexadecimal representation of the color. - - Hex string representing the color can be 3, 4, 6, or 8 characters depending on whether the string - a "short" representation of the color is possible and whether there's an alpha channel. - - Returns: - The hexadecimal representation of the color. - """ - values = [float_to_255(c) for c in self._rgba[:3]] - if self._rgba.alpha is not None: - values.append(float_to_255(self._rgba.alpha)) - - as_hex = ''.join(f'{v:02x}' for v in values) - if all(c in repeat_colors for c in values): - as_hex = ''.join(as_hex[c] for c in range(0, len(as_hex), 2)) - return '#' + as_hex - - def as_rgb(self) -> str: - """Color as an `rgb(, , )` or `rgba(, , , )` string.""" - if self._rgba.alpha is None: - return f'rgb({float_to_255(self._rgba.r)}, {float_to_255(self._rgba.g)}, {float_to_255(self._rgba.b)})' - else: - return ( - f'rgba({float_to_255(self._rgba.r)}, {float_to_255(self._rgba.g)}, {float_to_255(self._rgba.b)}, ' - f'{round(self._alpha_float(), 2)})' - ) - - def as_rgb_tuple(self, *, alpha: Optional[bool] = None) -> ColorTuple: - """Returns the color as an RGB or RGBA tuple. - - Args: - alpha: Whether to include the alpha channel. There are three options for this input: - - - `None` (default): Include alpha only if it's set. (e.g. not `None`) - - `True`: Always include alpha. - - `False`: Always omit alpha. - - Returns: - A tuple that contains the values of the red, green, and blue channels in the range 0 to 255. - If alpha is included, it is in the range 0 to 1. - """ - r, g, b = (float_to_255(c) for c in self._rgba[:3]) - if alpha is None: - if self._rgba.alpha is None: - return r, g, b - else: - return r, g, b, self._alpha_float() - elif alpha: - return r, g, b, self._alpha_float() - else: - # alpha is False - return r, g, b - - def as_hsl(self) -> str: - """Color as an `hsl(, , )` or `hsl(, , , )` string.""" - if self._rgba.alpha is None: - h, s, li = self.as_hsl_tuple(alpha=False) # type: ignore - return f'hsl({h * 360:0.0f}, {s:0.0%}, {li:0.0%})' - else: - h, s, li, a = self.as_hsl_tuple(alpha=True) # type: ignore - return f'hsl({h * 360:0.0f}, {s:0.0%}, {li:0.0%}, {round(a, 2)})' - - def as_hsl_tuple(self, *, alpha: Optional[bool] = None) -> HslColorTuple: - """Returns the color as an HSL or HSLA tuple. - - Args: - alpha: Whether to include the alpha channel. - - - `None` (default): Include the alpha channel only if it's set (e.g. not `None`). - - `True`: Always include alpha. - - `False`: Always omit alpha. - - Returns: - The color as a tuple of hue, saturation, lightness, and alpha (if included). - All elements are in the range 0 to 1. - - Note: - This is HSL as used in HTML and most other places, not HLS as used in Python's `colorsys`. - """ - h, l, s = rgb_to_hls(self._rgba.r, self._rgba.g, self._rgba.b) # noqa: E741 - if alpha is None: - if self._rgba.alpha is None: - return h, s, l - else: - return h, s, l, self._alpha_float() - if alpha: - return h, s, l, self._alpha_float() - else: - # alpha is False - return h, s, l - - def _alpha_float(self) -> float: - return 1 if self._rgba.alpha is None else self._rgba.alpha - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: Callable[[Any], CoreSchema] - ) -> core_schema.CoreSchema: - return core_schema.with_info_plain_validator_function( - cls._validate, serialization=core_schema.to_string_ser_schema() - ) - - @classmethod - def _validate(cls, __input_value: Any, _: Any) -> 'Color': - return cls(__input_value) - - def __str__(self) -> str: - return self.as_named(fallback=True) - - def __repr_args__(self) -> '_repr.ReprArgs': - return [(None, self.as_named(fallback=True))] + [('rgb', self.as_rgb_tuple())] - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Color) and self.as_rgb_tuple() == other.as_rgb_tuple() - - def __hash__(self) -> int: - return hash(self.as_rgb_tuple()) - - -def parse_tuple(value: tuple[Any, ...]) -> RGBA: - """Parse a tuple or list to get RGBA values. - - Args: - value: A tuple or list. - - Returns: - An `RGBA` tuple parsed from the input tuple. - - Raises: - PydanticCustomError: If tuple is not valid. - """ - if len(value) == 3: - r, g, b = (parse_color_value(v) for v in value) - return RGBA(r, g, b, None) - elif len(value) == 4: - r, g, b = (parse_color_value(v) for v in value[:3]) - return RGBA(r, g, b, parse_float_alpha(value[3])) - else: - raise PydanticCustomError('color_error', 'value is not a valid color: tuples must have length 3 or 4') - - -def parse_str(value: str) -> RGBA: - """Parse a string representing a color to an RGBA tuple. - - Possible formats for the input string include: - - * named color, see `COLORS_BY_NAME` - * hex short eg. `fff` (prefix can be `#`, `0x` or nothing) - * hex long eg. `ffffff` (prefix can be `#`, `0x` or nothing) - * `rgb(, , )` - * `rgba(, , , )` - - Args: - value: A string representing a color. - - Returns: - An `RGBA` tuple parsed from the input string. - - Raises: - ValueError: If the input string cannot be parsed to an RGBA tuple. - """ - value_lower = value.lower() - try: - r, g, b = COLORS_BY_NAME[value_lower] - except KeyError: - pass - else: - return ints_to_rgba(r, g, b, None) - - m = re.fullmatch(r_hex_short, value_lower) - if m: - *rgb, a = m.groups() - r, g, b = (int(v * 2, 16) for v in rgb) - if a: - alpha: Optional[float] = int(a * 2, 16) / 255 - else: - alpha = None - return ints_to_rgba(r, g, b, alpha) - - m = re.fullmatch(r_hex_long, value_lower) - if m: - *rgb, a = m.groups() - r, g, b = (int(v, 16) for v in rgb) - if a: - alpha = int(a, 16) / 255 - else: - alpha = None - return ints_to_rgba(r, g, b, alpha) - - m = re.fullmatch(r_rgb, value_lower) or re.fullmatch(r_rgb_v4_style, value_lower) - if m: - return ints_to_rgba(*m.groups()) # type: ignore - - m = re.fullmatch(r_hsl, value_lower) or re.fullmatch(r_hsl_v4_style, value_lower) - if m: - return parse_hsl(*m.groups()) # type: ignore - - raise PydanticCustomError('color_error', 'value is not a valid color: string not recognised as a valid color') - - -def ints_to_rgba(r: Union[int, str], g: Union[int, str], b: Union[int, str], alpha: Optional[float] = None) -> RGBA: - """Converts integer or string values for RGB color and an optional alpha value to an `RGBA` object. - - Args: - r: An integer or string representing the red color value. - g: An integer or string representing the green color value. - b: An integer or string representing the blue color value. - alpha: A float representing the alpha value. Defaults to None. - - Returns: - An instance of the `RGBA` class with the corresponding color and alpha values. - """ - return RGBA(parse_color_value(r), parse_color_value(g), parse_color_value(b), parse_float_alpha(alpha)) - - -def parse_color_value(value: Union[int, str], max_val: int = 255) -> float: - """Parse the color value provided and return a number between 0 and 1. - - Args: - value: An integer or string color value. - max_val: Maximum range value. Defaults to 255. - - Raises: - PydanticCustomError: If the value is not a valid color. - - Returns: - A number between 0 and 1. - """ - try: - color = float(value) - except ValueError: - raise PydanticCustomError('color_error', 'value is not a valid color: color values must be a valid number') - if 0 <= color <= max_val: - return color / max_val - else: - raise PydanticCustomError( - 'color_error', - 'value is not a valid color: color values must be in the range 0 to {max_val}', - {'max_val': max_val}, - ) - - -def parse_float_alpha(value: Union[None, str, float, int]) -> Optional[float]: - """Parse an alpha value checking it's a valid float in the range 0 to 1. - - Args: - value: The input value to parse. - - Returns: - The parsed value as a float, or `None` if the value was None or equal 1. - - Raises: - PydanticCustomError: If the input value cannot be successfully parsed as a float in the expected range. - """ - if value is None: - return None - try: - if isinstance(value, str) and value.endswith('%'): - alpha = float(value[:-1]) / 100 - else: - alpha = float(value) - except ValueError: - raise PydanticCustomError('color_error', 'value is not a valid color: alpha values must be a valid float') - - if math.isclose(alpha, 1): - return None - elif 0 <= alpha <= 1: - return alpha - else: - raise PydanticCustomError('color_error', 'value is not a valid color: alpha values must be in the range 0 to 1') - - -def parse_hsl(h: str, h_units: str, sat: str, light: str, alpha: Optional[float] = None) -> RGBA: - """Parse raw hue, saturation, lightness, and alpha values and convert to RGBA. - - Args: - h: The hue value. - h_units: The unit for hue value. - sat: The saturation value. - light: The lightness value. - alpha: Alpha value. - - Returns: - An instance of `RGBA`. - """ - s_value, l_value = parse_color_value(sat, 100), parse_color_value(light, 100) - - h_value = float(h) - if h_units in {None, 'deg'}: - h_value = h_value % 360 / 360 - elif h_units == 'rad': - h_value = h_value % rads / rads - else: - # turns - h_value = h_value % 1 - - r, g, b = hls_to_rgb(h_value, l_value, s_value) - return RGBA(r, g, b, parse_float_alpha(alpha)) - - -def float_to_255(c: float) -> int: - """Converts a float value between 0 and 1 (inclusive) to an integer between 0 and 255 (inclusive). - - Args: - c: The float value to be converted. Must be between 0 and 1 (inclusive). - - Returns: - The integer equivalent of the given float value rounded to the nearest whole number. - - Raises: - ValueError: If the given float value is outside the acceptable range of 0 to 1 (inclusive). - """ - return int(round(c * 255)) - - -COLORS_BY_NAME = { - 'aliceblue': (240, 248, 255), - 'antiquewhite': (250, 235, 215), - 'aqua': (0, 255, 255), - 'aquamarine': (127, 255, 212), - 'azure': (240, 255, 255), - 'beige': (245, 245, 220), - 'bisque': (255, 228, 196), - 'black': (0, 0, 0), - 'blanchedalmond': (255, 235, 205), - 'blue': (0, 0, 255), - 'blueviolet': (138, 43, 226), - 'brown': (165, 42, 42), - 'burlywood': (222, 184, 135), - 'cadetblue': (95, 158, 160), - 'chartreuse': (127, 255, 0), - 'chocolate': (210, 105, 30), - 'coral': (255, 127, 80), - 'cornflowerblue': (100, 149, 237), - 'cornsilk': (255, 248, 220), - 'crimson': (220, 20, 60), - 'cyan': (0, 255, 255), - 'darkblue': (0, 0, 139), - 'darkcyan': (0, 139, 139), - 'darkgoldenrod': (184, 134, 11), - 'darkgray': (169, 169, 169), - 'darkgreen': (0, 100, 0), - 'darkgrey': (169, 169, 169), - 'darkkhaki': (189, 183, 107), - 'darkmagenta': (139, 0, 139), - 'darkolivegreen': (85, 107, 47), - 'darkorange': (255, 140, 0), - 'darkorchid': (153, 50, 204), - 'darkred': (139, 0, 0), - 'darksalmon': (233, 150, 122), - 'darkseagreen': (143, 188, 143), - 'darkslateblue': (72, 61, 139), - 'darkslategray': (47, 79, 79), - 'darkslategrey': (47, 79, 79), - 'darkturquoise': (0, 206, 209), - 'darkviolet': (148, 0, 211), - 'deeppink': (255, 20, 147), - 'deepskyblue': (0, 191, 255), - 'dimgray': (105, 105, 105), - 'dimgrey': (105, 105, 105), - 'dodgerblue': (30, 144, 255), - 'firebrick': (178, 34, 34), - 'floralwhite': (255, 250, 240), - 'forestgreen': (34, 139, 34), - 'fuchsia': (255, 0, 255), - 'gainsboro': (220, 220, 220), - 'ghostwhite': (248, 248, 255), - 'gold': (255, 215, 0), - 'goldenrod': (218, 165, 32), - 'gray': (128, 128, 128), - 'green': (0, 128, 0), - 'greenyellow': (173, 255, 47), - 'grey': (128, 128, 128), - 'honeydew': (240, 255, 240), - 'hotpink': (255, 105, 180), - 'indianred': (205, 92, 92), - 'indigo': (75, 0, 130), - 'ivory': (255, 255, 240), - 'khaki': (240, 230, 140), - 'lavender': (230, 230, 250), - 'lavenderblush': (255, 240, 245), - 'lawngreen': (124, 252, 0), - 'lemonchiffon': (255, 250, 205), - 'lightblue': (173, 216, 230), - 'lightcoral': (240, 128, 128), - 'lightcyan': (224, 255, 255), - 'lightgoldenrodyellow': (250, 250, 210), - 'lightgray': (211, 211, 211), - 'lightgreen': (144, 238, 144), - 'lightgrey': (211, 211, 211), - 'lightpink': (255, 182, 193), - 'lightsalmon': (255, 160, 122), - 'lightseagreen': (32, 178, 170), - 'lightskyblue': (135, 206, 250), - 'lightslategray': (119, 136, 153), - 'lightslategrey': (119, 136, 153), - 'lightsteelblue': (176, 196, 222), - 'lightyellow': (255, 255, 224), - 'lime': (0, 255, 0), - 'limegreen': (50, 205, 50), - 'linen': (250, 240, 230), - 'magenta': (255, 0, 255), - 'maroon': (128, 0, 0), - 'mediumaquamarine': (102, 205, 170), - 'mediumblue': (0, 0, 205), - 'mediumorchid': (186, 85, 211), - 'mediumpurple': (147, 112, 219), - 'mediumseagreen': (60, 179, 113), - 'mediumslateblue': (123, 104, 238), - 'mediumspringgreen': (0, 250, 154), - 'mediumturquoise': (72, 209, 204), - 'mediumvioletred': (199, 21, 133), - 'midnightblue': (25, 25, 112), - 'mintcream': (245, 255, 250), - 'mistyrose': (255, 228, 225), - 'moccasin': (255, 228, 181), - 'navajowhite': (255, 222, 173), - 'navy': (0, 0, 128), - 'oldlace': (253, 245, 230), - 'olive': (128, 128, 0), - 'olivedrab': (107, 142, 35), - 'orange': (255, 165, 0), - 'orangered': (255, 69, 0), - 'orchid': (218, 112, 214), - 'palegoldenrod': (238, 232, 170), - 'palegreen': (152, 251, 152), - 'paleturquoise': (175, 238, 238), - 'palevioletred': (219, 112, 147), - 'papayawhip': (255, 239, 213), - 'peachpuff': (255, 218, 185), - 'peru': (205, 133, 63), - 'pink': (255, 192, 203), - 'plum': (221, 160, 221), - 'powderblue': (176, 224, 230), - 'purple': (128, 0, 128), - 'red': (255, 0, 0), - 'rosybrown': (188, 143, 143), - 'royalblue': (65, 105, 225), - 'saddlebrown': (139, 69, 19), - 'salmon': (250, 128, 114), - 'sandybrown': (244, 164, 96), - 'seagreen': (46, 139, 87), - 'seashell': (255, 245, 238), - 'sienna': (160, 82, 45), - 'silver': (192, 192, 192), - 'skyblue': (135, 206, 235), - 'slateblue': (106, 90, 205), - 'slategray': (112, 128, 144), - 'slategrey': (112, 128, 144), - 'snow': (255, 250, 250), - 'springgreen': (0, 255, 127), - 'steelblue': (70, 130, 180), - 'tan': (210, 180, 140), - 'teal': (0, 128, 128), - 'thistle': (216, 191, 216), - 'tomato': (255, 99, 71), - 'turquoise': (64, 224, 208), - 'violet': (238, 130, 238), - 'wheat': (245, 222, 179), - 'white': (255, 255, 255), - 'whitesmoke': (245, 245, 245), - 'yellow': (255, 255, 0), - 'yellowgreen': (154, 205, 50), -} - -COLORS_BY_VALUE = {v: k for k, v in COLORS_BY_NAME.items()} diff --git a/functions/python/data-processor/pydantic/config.py b/functions/python/data-processor/pydantic/config.py deleted file mode 100644 index 6d895258c..000000000 --- a/functions/python/data-processor/pydantic/config.py +++ /dev/null @@ -1,1296 +0,0 @@ -"""Configuration for Pydantic models.""" - -from __future__ import annotations as _annotations - -import warnings -from re import Pattern -from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, Union, cast, overload - -from typing_extensions import TypeAlias, TypedDict, Unpack, deprecated - -from ._migration import getattr_migration -from .aliases import AliasGenerator -from .errors import PydanticUserError -from .warnings import PydanticDeprecatedSince211 - -if TYPE_CHECKING: - from ._internal._generate_schema import GenerateSchema as _GenerateSchema - from .fields import ComputedFieldInfo, FieldInfo - -__all__ = ('ConfigDict', 'with_config') - - -JsonValue: TypeAlias = Union[int, float, str, bool, None, list['JsonValue'], 'JsonDict'] -JsonDict: TypeAlias = dict[str, JsonValue] - -JsonEncoder = Callable[[Any], Any] - -JsonSchemaExtraCallable: TypeAlias = Union[ - Callable[[JsonDict], None], - Callable[[JsonDict, type[Any]], None], -] - -ExtraValues = Literal['allow', 'ignore', 'forbid'] - - -class ConfigDict(TypedDict, total=False): - """A TypedDict for configuring Pydantic behaviour.""" - - title: str | None - """The title for the generated JSON schema, defaults to the model's name""" - - model_title_generator: Callable[[type], str] | None - """A callable that takes a model class and returns the title for it. Defaults to `None`.""" - - field_title_generator: Callable[[str, FieldInfo | ComputedFieldInfo], str] | None - """A callable that takes a field's name and info and returns title for it. Defaults to `None`.""" - - str_to_lower: bool - """Whether to convert all characters to lowercase for str types. Defaults to `False`.""" - - str_to_upper: bool - """Whether to convert all characters to uppercase for str types. Defaults to `False`.""" - - str_strip_whitespace: bool - """Whether to strip leading and trailing whitespace for str types.""" - - str_min_length: int - """The minimum length for str types. Defaults to `None`.""" - - str_max_length: int | None - """The maximum length for str types. Defaults to `None`.""" - - extra: ExtraValues | None - ''' - Whether to ignore, allow, or forbid extra data during model initialization. Defaults to `'ignore'`. - - Three configuration values are available: - - - `'ignore'`: Providing extra data is ignored (the default): - ```python - from pydantic import BaseModel, ConfigDict - - class User(BaseModel): - model_config = ConfigDict(extra='ignore') # (1)! - - name: str - - user = User(name='John Doe', age=20) # (2)! - print(user) - #> name='John Doe' - ``` - - 1. This is the default behaviour. - 2. The `age` argument is ignored. - - - `'forbid'`: Providing extra data is not permitted, and a [`ValidationError`][pydantic_core.ValidationError] - will be raised if this is the case: - ```python - from pydantic import BaseModel, ConfigDict, ValidationError - - - class Model(BaseModel): - x: int - - model_config = ConfigDict(extra='forbid') - - - try: - Model(x=1, y='a') - except ValidationError as exc: - print(exc) - """ - 1 validation error for Model - y - Extra inputs are not permitted [type=extra_forbidden, input_value='a', input_type=str] - """ - ``` - - - `'allow'`: Providing extra data is allowed and stored in the `__pydantic_extra__` dictionary attribute: - ```python - from pydantic import BaseModel, ConfigDict - - - class Model(BaseModel): - x: int - - model_config = ConfigDict(extra='allow') - - - m = Model(x=1, y='a') - assert m.__pydantic_extra__ == {'y': 'a'} - ``` - By default, no validation will be applied to these extra items, but you can set a type for the values by overriding - the type annotation for `__pydantic_extra__`: - ```python - from pydantic import BaseModel, ConfigDict, Field, ValidationError - - - class Model(BaseModel): - __pydantic_extra__: dict[str, int] = Field(init=False) # (1)! - - x: int - - model_config = ConfigDict(extra='allow') - - - try: - Model(x=1, y='a') - except ValidationError as exc: - print(exc) - """ - 1 validation error for Model - y - Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='a', input_type=str] - """ - - m = Model(x=1, y='2') - assert m.x == 1 - assert m.y == 2 - assert m.model_dump() == {'x': 1, 'y': 2} - assert m.__pydantic_extra__ == {'y': 2} - ``` - - 1. The `= Field(init=False)` does not have any effect at runtime, but prevents the `__pydantic_extra__` field from - being included as a parameter to the model's `__init__` method by type checkers. - - As well as specifying an `extra` configuration value on the model, you can also provide it as an argument to the validation methods. - This will override any `extra` configuration value set on the model: - ```python - from pydantic import BaseModel, ConfigDict, ValidationError - - class Model(BaseModel): - x: int - model_config = ConfigDict(extra="allow") - - try: - # Override model config and forbid extra fields just this time - Model.model_validate({"x": 1, "y": 2}, extra="forbid") - except ValidationError as exc: - print(exc) - """ - 1 validation error for Model - y - Extra inputs are not permitted [type=extra_forbidden, input_value=2, input_type=int] - """ - ``` - ''' - - frozen: bool - """ - Whether models are faux-immutable, i.e. whether `__setattr__` is allowed, and also generates - a `__hash__()` method for the model. This makes instances of the model potentially hashable if all the - attributes are hashable. Defaults to `False`. - - Note: - On V1, the inverse of this setting was called `allow_mutation`, and was `True` by default. - """ - - populate_by_name: bool - """ - Whether an aliased field may be populated by its name as given by the model - attribute, as well as the alias. Defaults to `False`. - - !!! warning - `populate_by_name` usage is not recommended in v2.11+ and will be deprecated in v3. - Instead, you should use the [`validate_by_name`][pydantic.config.ConfigDict.validate_by_name] configuration setting. - - When `validate_by_name=True` and `validate_by_alias=True`, this is strictly equivalent to the - previous behavior of `populate_by_name=True`. - - In v2.11, we also introduced a [`validate_by_alias`][pydantic.config.ConfigDict.validate_by_alias] setting that introduces more fine grained - control for validation behavior. - - Here's how you might go about using the new settings to achieve the same behavior: - - ```python - from pydantic import BaseModel, ConfigDict, Field - - class Model(BaseModel): - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) - - my_field: str = Field(alias='my_alias') # (1)! - - m = Model(my_alias='foo') # (2)! - print(m) - #> my_field='foo' - - m = Model(my_field='foo') # (3)! - print(m) - #> my_field='foo' - ``` - - 1. The field `'my_field'` has an alias `'my_alias'`. - 2. The model is populated by the alias `'my_alias'`. - 3. The model is populated by the attribute name `'my_field'`. - """ - - use_enum_values: bool - """ - Whether to populate models with the `value` property of enums, rather than the raw enum. - This may be useful if you want to serialize `model.model_dump()` later. Defaults to `False`. - - !!! note - If you have an `Optional[Enum]` value that you set a default for, you need to use `validate_default=True` - for said Field to ensure that the `use_enum_values` flag takes effect on the default, as extracting an - enum's value occurs during validation, not serialization. - - ```python - from enum import Enum - from typing import Optional - - from pydantic import BaseModel, ConfigDict, Field - - class SomeEnum(Enum): - FOO = 'foo' - BAR = 'bar' - BAZ = 'baz' - - class SomeModel(BaseModel): - model_config = ConfigDict(use_enum_values=True) - - some_enum: SomeEnum - another_enum: Optional[SomeEnum] = Field( - default=SomeEnum.FOO, validate_default=True - ) - - model1 = SomeModel(some_enum=SomeEnum.BAR) - print(model1.model_dump()) - #> {'some_enum': 'bar', 'another_enum': 'foo'} - - model2 = SomeModel(some_enum=SomeEnum.BAR, another_enum=SomeEnum.BAZ) - print(model2.model_dump()) - #> {'some_enum': 'bar', 'another_enum': 'baz'} - ``` - """ - - validate_assignment: bool - """ - Whether to validate the data when the model is changed. Defaults to `False`. - - The default behavior of Pydantic is to validate the data when the model is created. - - In case the user changes the data after the model is created, the model is _not_ revalidated. - - ```python - from pydantic import BaseModel - - class User(BaseModel): - name: str - - user = User(name='John Doe') # (1)! - print(user) - #> name='John Doe' - user.name = 123 # (1)! - print(user) - #> name=123 - ``` - - 1. The validation happens only when the model is created. - 2. The validation does not happen when the data is changed. - - In case you want to revalidate the model when the data is changed, you can use `validate_assignment=True`: - - ```python - from pydantic import BaseModel, ValidationError - - class User(BaseModel, validate_assignment=True): # (1)! - name: str - - user = User(name='John Doe') # (2)! - print(user) - #> name='John Doe' - try: - user.name = 123 # (3)! - except ValidationError as e: - print(e) - ''' - 1 validation error for User - name - Input should be a valid string [type=string_type, input_value=123, input_type=int] - ''' - ``` - - 1. You can either use class keyword arguments, or `model_config` to set `validate_assignment=True`. - 2. The validation happens when the model is created. - 3. The validation _also_ happens when the data is changed. - """ - - arbitrary_types_allowed: bool - """ - Whether arbitrary types are allowed for field types. Defaults to `False`. - - ```python - from pydantic import BaseModel, ConfigDict, ValidationError - - # This is not a pydantic model, it's an arbitrary class - class Pet: - def __init__(self, name: str): - self.name = name - - class Model(BaseModel): - model_config = ConfigDict(arbitrary_types_allowed=True) - - pet: Pet - owner: str - - pet = Pet(name='Hedwig') - # A simple check of instance type is used to validate the data - model = Model(owner='Harry', pet=pet) - print(model) - #> pet=<__main__.Pet object at 0x0123456789ab> owner='Harry' - print(model.pet) - #> <__main__.Pet object at 0x0123456789ab> - print(model.pet.name) - #> Hedwig - print(type(model.pet)) - #> - try: - # If the value is not an instance of the type, it's invalid - Model(owner='Harry', pet='Hedwig') - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - pet - Input should be an instance of Pet [type=is_instance_of, input_value='Hedwig', input_type=str] - ''' - - # Nothing in the instance of the arbitrary type is checked - # Here name probably should have been a str, but it's not validated - pet2 = Pet(name=42) - model2 = Model(owner='Harry', pet=pet2) - print(model2) - #> pet=<__main__.Pet object at 0x0123456789ab> owner='Harry' - print(model2.pet) - #> <__main__.Pet object at 0x0123456789ab> - print(model2.pet.name) - #> 42 - print(type(model2.pet)) - #> - ``` - """ - - from_attributes: bool - """ - Whether to build models and look up discriminators of tagged unions using python object attributes. - """ - - loc_by_alias: bool - """Whether to use the actual key provided in the data (e.g. alias) for error `loc`s rather than the field's name. Defaults to `True`.""" - - alias_generator: Callable[[str], str] | AliasGenerator | None - """ - A callable that takes a field name and returns an alias for it - or an instance of [`AliasGenerator`][pydantic.aliases.AliasGenerator]. Defaults to `None`. - - When using a callable, the alias generator is used for both validation and serialization. - If you want to use different alias generators for validation and serialization, you can use - [`AliasGenerator`][pydantic.aliases.AliasGenerator] instead. - - If data source field names do not match your code style (e.g. CamelCase fields), - you can automatically generate aliases using `alias_generator`. Here's an example with - a basic callable: - - ```python - from pydantic import BaseModel, ConfigDict - from pydantic.alias_generators import to_pascal - - class Voice(BaseModel): - model_config = ConfigDict(alias_generator=to_pascal) - - name: str - language_code: str - - voice = Voice(Name='Filiz', LanguageCode='tr-TR') - print(voice.language_code) - #> tr-TR - print(voice.model_dump(by_alias=True)) - #> {'Name': 'Filiz', 'LanguageCode': 'tr-TR'} - ``` - - If you want to use different alias generators for validation and serialization, you can use - [`AliasGenerator`][pydantic.aliases.AliasGenerator]. - - ```python - from pydantic import AliasGenerator, BaseModel, ConfigDict - from pydantic.alias_generators import to_camel, to_pascal - - class Athlete(BaseModel): - first_name: str - last_name: str - sport: str - - model_config = ConfigDict( - alias_generator=AliasGenerator( - validation_alias=to_camel, - serialization_alias=to_pascal, - ) - ) - - athlete = Athlete(firstName='John', lastName='Doe', sport='track') - print(athlete.model_dump(by_alias=True)) - #> {'FirstName': 'John', 'LastName': 'Doe', 'Sport': 'track'} - ``` - - Note: - Pydantic offers three built-in alias generators: [`to_pascal`][pydantic.alias_generators.to_pascal], - [`to_camel`][pydantic.alias_generators.to_camel], and [`to_snake`][pydantic.alias_generators.to_snake]. - """ - - ignored_types: tuple[type, ...] - """A tuple of types that may occur as values of class attributes without annotations. This is - typically used for custom descriptors (classes that behave like `property`). If an attribute is set on a - class without an annotation and has a type that is not in this tuple (or otherwise recognized by - _pydantic_), an error will be raised. Defaults to `()`. - """ - - allow_inf_nan: bool - """Whether to allow infinity (`+inf` an `-inf`) and NaN values to float and decimal fields. Defaults to `True`.""" - - json_schema_extra: JsonDict | JsonSchemaExtraCallable | None - """A dict or callable to provide extra JSON schema properties. Defaults to `None`.""" - - json_encoders: dict[type[object], JsonEncoder] | None - """ - A `dict` of custom JSON encoders for specific types. Defaults to `None`. - - /// version-deprecated | v2 - This configuration option is a carryover from v1. We originally planned to remove it in v2 but didn't have a 1:1 replacement - so we are keeping it for now. It is still deprecated and will likely be removed in the future. - /// - """ - - # new in V2 - strict: bool - """ - Whether strict validation is applied to all fields on the model. - - By default, Pydantic attempts to coerce values to the correct type, when possible. - - There are situations in which you may want to disable this behavior, and instead raise an error if a value's type - does not match the field's type annotation. - - To configure strict mode for all fields on a model, you can set `strict=True` on the model. - - ```python - from pydantic import BaseModel, ConfigDict - - class Model(BaseModel): - model_config = ConfigDict(strict=True) - - name: str - age: int - ``` - - See [Strict Mode](../concepts/strict_mode.md) for more details. - - See the [Conversion Table](../concepts/conversion_table.md) for more details on how Pydantic converts data in both - strict and lax modes. - - /// version-added | v2 - /// - """ - # whether instances of models and dataclasses (including subclass instances) should re-validate, default 'never' - revalidate_instances: Literal['always', 'never', 'subclass-instances'] - """ - When and how to revalidate models and dataclasses during validation. Can be one of: - - - `'never'`: will *not* revalidate models and dataclasses during validation - - `'always'`: will revalidate models and dataclasses during validation - - `'subclass-instances'`: will revalidate models and dataclasses during validation if the instance is a - subclass of the model or dataclass - - The default is `'never'` (no revalidation). - - This configuration only affects *the current model* it is applied on, and does *not* propagate to the models - referenced in fields. - - ```python - from pydantic import BaseModel - - class User(BaseModel, revalidate_instances='never'): # (1)! - name: str - - class Transaction(BaseModel): - user: User - - my_user = User(name='John') - t = Transaction(user=my_user) - - my_user.name = 1 # (2)! - t = Transaction(user=my_user) # (3)! - print(t) - #> user=User(name=1) - ``` - - 1. This is the default behavior. - 2. The assignment is *not* validated, unless you set [`validate_assignment`][pydantic.ConfigDict.validate_assignment] in the configuration. - 3. Since `revalidate_instances` is set to `'never'`, the user instance is not revalidated. - - Here is an example demonstrating the behavior of `'subclass-instances'`: - - ```python - from pydantic import BaseModel - - class User(BaseModel, revalidate_instances='subclass-instances'): - name: str - - class SubUser(User): - age: int - - class Transaction(BaseModel): - user: User - - my_user = User(name='John') - my_user.name = 1 # (1)! - t = Transaction(user=my_user) # (2)! - print(t) - #> user=User(name=1) - - my_sub_user = SubUser(name='John', age=20) - t = Transaction(user=my_sub_user) - print(t) # (3)! - #> user=User(name='John') - ``` - - 1. The assignment is *not* validated, unless you set [`validate_assignment`][pydantic.ConfigDict.validate_assignment] in the configuration. - 2. Because `my_user` is a "direct" instance of `User`, it is *not* being revalidated. It would have been the case if - `revalidate_instances` was set to `'always'`. - 3. Because `my_sub_user` is an instance of a `User` subclass, it is being revalidated. In this case, Pydantic coerces `my_sub_user` to the defined - `User` class defined on `Transaction`. If one of its fields had an invalid value, a validation error would have been raised. - - /// version-added | v2 - /// - """ - - ser_json_timedelta: Literal['iso8601', 'float'] - """ - The format of JSON serialized timedeltas. Accepts the string values of `'iso8601'` and - `'float'`. Defaults to `'iso8601'`. - - - `'iso8601'` will serialize timedeltas to [ISO 8601 text format](https://en.wikipedia.org/wiki/ISO_8601#Durations). - - `'float'` will serialize timedeltas to the total number of seconds. - - /// version-changed | v2.12 - It is now recommended to use the [`ser_json_temporal`][pydantic.config.ConfigDict.ser_json_temporal] - setting. `ser_json_timedelta` will be deprecated in v3. - /// - """ - - ser_json_temporal: Literal['iso8601', 'seconds', 'milliseconds'] - """ - The format of JSON serialized temporal types from the [`datetime`][] module. This includes: - - - [`datetime.datetime`][] - - [`datetime.date`][] - - [`datetime.time`][] - - [`datetime.timedelta`][] - - Can be one of: - - - `'iso8601'` will serialize date-like types to [ISO 8601 text format](https://en.wikipedia.org/wiki/ISO_8601#Durations). - - `'milliseconds'` will serialize date-like types to a floating point number of milliseconds since the epoch. - - `'seconds'` will serialize date-like types to a floating point number of seconds since the epoch. - - Defaults to `'iso8601'`. - - /// version-added | v2.12 - This setting replaces [`ser_json_timedelta`][pydantic.config.ConfigDict.ser_json_timedelta], - which will be deprecated in v3. `ser_json_temporal` adds more configurability for the other temporal types. - /// - """ - - val_temporal_unit: Literal['seconds', 'milliseconds', 'infer'] - """ - The unit to assume for validating numeric input for datetime-like types ([`datetime.datetime`][] and [`datetime.date`][]). Can be one of: - - - `'seconds'` will validate date or time numeric inputs as seconds since the [epoch]. - - `'milliseconds'` will validate date or time numeric inputs as milliseconds since the [epoch]. - - `'infer'` will infer the unit from the string numeric input on unix time as: - - * seconds since the [epoch] if $-2^{10} <= v <= 2^{10}$ - * milliseconds since the [epoch] (if $v < -2^{10}$ or $v > 2^{10}$). - - Defaults to `'infer'`. - - /// version-added | v2.12 - /// - - [epoch]: https://en.wikipedia.org/wiki/Unix_time - """ - - ser_json_bytes: Literal['utf8', 'base64', 'hex'] - """ - The encoding of JSON serialized bytes. Defaults to `'utf8'`. - Set equal to `val_json_bytes` to get back an equal value after serialization round trip. - - - `'utf8'` will serialize bytes to UTF-8 strings. - - `'base64'` will serialize bytes to URL safe base64 strings. - - `'hex'` will serialize bytes to hexadecimal strings. - """ - - val_json_bytes: Literal['utf8', 'base64', 'hex'] - """ - /// version-added | v2.9 - /// - - The encoding of JSON serialized bytes to decode. Defaults to `'utf8'`. - Set equal to `ser_json_bytes` to get back an equal value after serialization round trip. - - - `'utf8'` will deserialize UTF-8 strings to bytes. - - `'base64'` will deserialize URL safe base64 strings to bytes. - - `'hex'` will deserialize hexadecimal strings to bytes. - """ - - ser_json_inf_nan: Literal['null', 'constants', 'strings'] - """ - The encoding of JSON serialized infinity and NaN float values. Defaults to `'null'`. - - - `'null'` will serialize infinity and NaN values as `null`. - - `'constants'` will serialize infinity and NaN values as `Infinity` and `NaN`. - - `'strings'` will serialize infinity as string `"Infinity"` and NaN as string `"NaN"`. - """ - - # whether to validate default values during validation, default False - validate_default: bool - """Whether to validate default values during validation. Defaults to `False`.""" - - validate_return: bool - """Whether to validate the return value from call validators. Defaults to `False`.""" - - protected_namespaces: tuple[str | Pattern[str], ...] - """ - A tuple of strings and/or regex patterns that prevent models from having fields with names that conflict with its existing members/methods. - - Strings are matched on a prefix basis. For instance, with `'dog'`, having a field named `'dog_name'` will be disallowed. - - Regex patterns are matched on the entire field name. For instance, with the pattern `'^dog$'`, having a field named `'dog'` will be disallowed, - but `'dog_name'` will be accepted. - - Defaults to `('model_validate', 'model_dump')`. This default is used to prevent collisions with the existing (and possibly future) - [validation](../concepts/models.md#validating-data) and [serialization](../concepts/serialization.md#serializing-data) methods. - - ```python - import warnings - - from pydantic import BaseModel - - warnings.filterwarnings('error') # Raise warnings as errors - - try: - - class Model(BaseModel): - model_dump_something: str - - except UserWarning as e: - print(e) - ''' - Field 'model_dump_something' in 'Model' conflicts with protected namespace 'model_dump'. - - You may be able to solve this by setting the 'protected_namespaces' configuration to ('model_validate',). - ''' - ``` - - You can customize this behavior using the `protected_namespaces` setting: - - ```python {test="skip"} - import re - import warnings - - from pydantic import BaseModel, ConfigDict - - with warnings.catch_warnings(record=True) as caught_warnings: - warnings.simplefilter('always') # Catch all warnings - - class Model(BaseModel): - safe_field: str - also_protect_field: str - protect_this: str - - model_config = ConfigDict( - protected_namespaces=( - 'protect_me_', - 'also_protect_', - re.compile('^protect_this$'), - ) - ) - - for warning in caught_warnings: - print(f'{warning.message}') - ''' - Field 'also_protect_field' in 'Model' conflicts with protected namespace 'also_protect_'. - You may be able to solve this by setting the 'protected_namespaces' configuration to ('protect_me_', re.compile('^protect_this$'))`. - - Field 'protect_this' in 'Model' conflicts with protected namespace 're.compile('^protect_this$')'. - You may be able to solve this by setting the 'protected_namespaces' configuration to ('protect_me_', 'also_protect_')`. - ''' - ``` - - While Pydantic will only emit a warning when an item is in a protected namespace but does not actually have a collision, - an error _is_ raised if there is an actual collision with an existing attribute: - - ```python - from pydantic import BaseModel, ConfigDict - - try: - - class Model(BaseModel): - model_validate: str - - model_config = ConfigDict(protected_namespaces=('model_',)) - - except ValueError as e: - print(e) - ''' - Field 'model_validate' conflicts with member > of protected namespace 'model_'. - ''' - ``` - - /// version-changed | v2.10 - The default protected namespaces was changed from `('model_',)` to `('model_validate', 'model_dump')`, to allow - for fields like `model_id`, `model_name` to be used. - /// - """ - - hide_input_in_errors: bool - """ - Whether to hide inputs when printing errors. Defaults to `False`. - - Pydantic shows the input value and type when it raises `ValidationError` during the validation. - - ```python - from pydantic import BaseModel, ValidationError - - class Model(BaseModel): - a: str - - try: - Model(a=123) - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - a - Input should be a valid string [type=string_type, input_value=123, input_type=int] - ''' - ``` - - You can hide the input value and type by setting the `hide_input_in_errors` config to `True`. - - ```python - from pydantic import BaseModel, ConfigDict, ValidationError - - class Model(BaseModel): - a: str - model_config = ConfigDict(hide_input_in_errors=True) - - try: - Model(a=123) - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - a - Input should be a valid string [type=string_type] - ''' - ``` - """ - - defer_build: bool - """ - Whether to defer model validator and serializer construction until the first model validation. Defaults to False. - - This can be useful to avoid the overhead of building models which are only - used nested within other models, or when you want to manually define type namespace via - [`Model.model_rebuild(_types_namespace=...)`][pydantic.BaseModel.model_rebuild]. - - /// version-changed | v2.10 - The setting also applies to [Pydantic dataclasses](../concepts/dataclasses.md) and [type adapters](../concepts/type_adapter.md). - /// - """ - - plugin_settings: dict[str, object] | None - """A `dict` of settings for plugins. Defaults to `None`.""" - - schema_generator: type[_GenerateSchema] | None - """ - The `GenerateSchema` class to use during core schema generation. - - /// version-deprecated | v2.10 - The `GenerateSchema` class is private and highly subject to change. - /// - """ - - json_schema_serialization_defaults_required: bool - """ - Whether fields with default values should be marked as required in the serialization schema. Defaults to `False`. - - This ensures that the serialization schema will reflect the fact a field with a default will always be present - when serializing the model, even though it is not required for validation. - - However, there are scenarios where this may be undesirable — in particular, if you want to share the schema - between validation and serialization, and don't mind fields with defaults being marked as not required during - serialization. See [#7209](https://github.com/pydantic/pydantic/issues/7209) for more details. - - ```python - from pydantic import BaseModel, ConfigDict - - class Model(BaseModel): - a: str = 'a' - - model_config = ConfigDict(json_schema_serialization_defaults_required=True) - - print(Model.model_json_schema(mode='validation')) - ''' - { - 'properties': {'a': {'default': 'a', 'title': 'A', 'type': 'string'}}, - 'title': 'Model', - 'type': 'object', - } - ''' - print(Model.model_json_schema(mode='serialization')) - ''' - { - 'properties': {'a': {'default': 'a', 'title': 'A', 'type': 'string'}}, - 'required': ['a'], - 'title': 'Model', - 'type': 'object', - } - ''' - ``` - - /// version-added | v2.4 - /// - """ - - json_schema_mode_override: Literal['validation', 'serialization', None] - """ - If not `None`, the specified mode will be used to generate the JSON schema regardless of what `mode` was passed to - the function call. Defaults to `None`. - - This provides a way to force the JSON schema generation to reflect a specific mode, e.g., to always use the - validation schema. - - It can be useful when using frameworks (such as FastAPI) that may generate different schemas for validation - and serialization that must both be referenced from the same schema; when this happens, we automatically append - `-Input` to the definition reference for the validation schema and `-Output` to the definition reference for the - serialization schema. By specifying a `json_schema_mode_override` though, this prevents the conflict between - the validation and serialization schemas (since both will use the specified schema), and so prevents the suffixes - from being added to the definition references. - - ```python - from pydantic import BaseModel, ConfigDict, Json - - class Model(BaseModel): - a: Json[int] # requires a string to validate, but will dump an int - - print(Model.model_json_schema(mode='serialization')) - ''' - { - 'properties': {'a': {'title': 'A', 'type': 'integer'}}, - 'required': ['a'], - 'title': 'Model', - 'type': 'object', - } - ''' - - class ForceInputModel(Model): - # the following ensures that even with mode='serialization', we - # will get the schema that would be generated for validation. - model_config = ConfigDict(json_schema_mode_override='validation') - - print(ForceInputModel.model_json_schema(mode='serialization')) - ''' - { - 'properties': { - 'a': { - 'contentMediaType': 'application/json', - 'contentSchema': {'type': 'integer'}, - 'title': 'A', - 'type': 'string', - } - }, - 'required': ['a'], - 'title': 'ForceInputModel', - 'type': 'object', - } - ''' - ``` - - /// version-added | v2.4 - /// - """ - - coerce_numbers_to_str: bool - """ - If `True`, enables automatic coercion of any `Number` type to `str` in "lax" (non-strict) mode. Defaults to `False`. - - Pydantic doesn't allow number types (`int`, `float`, `Decimal`) to be coerced as type `str` by default. - - ```python - from decimal import Decimal - - from pydantic import BaseModel, ConfigDict, ValidationError - - class Model(BaseModel): - value: str - - try: - print(Model(value=42)) - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - value - Input should be a valid string [type=string_type, input_value=42, input_type=int] - ''' - - class Model(BaseModel): - model_config = ConfigDict(coerce_numbers_to_str=True) - - value: str - - repr(Model(value=42).value) - #> "42" - repr(Model(value=42.13).value) - #> "42.13" - repr(Model(value=Decimal('42.13')).value) - #> "42.13" - ``` - """ - - regex_engine: Literal['rust-regex', 'python-re'] - """ - The regex engine to be used for pattern validation. - Defaults to `'rust-regex'`. - - - `'rust-regex'` uses the [`regex`](https://docs.rs/regex) Rust crate, - which is non-backtracking and therefore more DDoS resistant, but does not support all regex features. - - `'python-re'` use the [`re`][] module, which supports all regex features, but may be slower. - - !!! note - If you use a compiled regex pattern, the `'python-re'` engine will be used regardless of this setting. - This is so that flags such as [`re.IGNORECASE`][] are respected. - - ```python - from pydantic import BaseModel, ConfigDict, Field, ValidationError - - class Model(BaseModel): - model_config = ConfigDict(regex_engine='python-re') - - value: str = Field(pattern=r'^abc(?=def)') - - print(Model(value='abcdef').value) - #> abcdef - - try: - print(Model(value='abxyzcdef')) - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - value - String should match pattern '^abc(?=def)' [type=string_pattern_mismatch, input_value='abxyzcdef', input_type=str] - ''' - ``` - - /// version-added | v2.5 - /// - """ - - validation_error_cause: bool - """ - If `True`, Python exceptions that were part of a validation failure will be shown as an exception group as a cause. Can be useful for debugging. Defaults to `False`. - - Note: - Python 3.10 and older don't support exception groups natively. <=3.10, backport must be installed: `pip install exceptiongroup`. - - Note: - The structure of validation errors are likely to change in future Pydantic versions. Pydantic offers no guarantees about their structure. Should be used for visual traceback debugging only. - - /// version-added | v2.5 - /// - """ - - use_attribute_docstrings: bool - ''' - Whether docstrings of attributes (bare string literals immediately following the attribute declaration) - should be used for field descriptions. Defaults to `False`. - - ```python - from pydantic import BaseModel, ConfigDict, Field - - - class Model(BaseModel): - model_config = ConfigDict(use_attribute_docstrings=True) - - x: str - """ - Example of an attribute docstring - """ - - y: int = Field(description="Description in Field") - """ - Description in Field overrides attribute docstring - """ - - - print(Model.model_fields["x"].description) - # > Example of an attribute docstring - print(Model.model_fields["y"].description) - # > Description in Field - ``` - This requires the source code of the class to be available at runtime (and so won't work in the interactive interpreter shell). - - !!! warning "Usage with `TypedDict` and stdlib dataclasses" - Due to current limitations, attribute docstrings detection may not work as expected when using - [`TypedDict`][typing.TypedDict] and stdlib dataclasses, in particular when: - - - inheritance is being used. - - multiple classes have the same name in the same source file (unless Python 3.13 or greater is used). - - /// version-added | v2.7 - /// - ''' - - cache_strings: bool | Literal['all', 'keys', 'none'] - """ - Whether to cache strings to avoid constructing new Python objects. Defaults to True. - - Enabling this setting should significantly improve validation performance while increasing memory usage slightly. - - - `True` or `'all'` (the default): cache all strings - - `'keys'`: cache only dictionary keys - - `False` or `'none'`: no caching - - !!! note - `True` or `'all'` is required to cache strings during general validation because - validators don't know if they're in a key or a value. - - !!! tip - If repeated strings are rare, it's recommended to use `'keys'` or `'none'` to reduce memory usage, - as the performance difference is minimal if repeated strings are rare. - - /// version-added | v2.7 - /// - """ - - validate_by_alias: bool - """ - Whether an aliased field may be populated by its alias. Defaults to `True`. - - Here's an example of disabling validation by alias: - - ```py - from pydantic import BaseModel, ConfigDict, Field - - class Model(BaseModel): - model_config = ConfigDict(validate_by_name=True, validate_by_alias=False) - - my_field: str = Field(validation_alias='my_alias') # (1)! - - m = Model(my_field='foo') # (2)! - print(m) - #> my_field='foo' - ``` - - 1. The field `'my_field'` has an alias `'my_alias'`. - 2. The model can only be populated by the attribute name `'my_field'`. - - !!! warning - You cannot set both `validate_by_alias` and `validate_by_name` to `False`. - This would make it impossible to populate an attribute. - - See [usage errors](../errors/usage_errors.md#validate-by-alias-and-name-false) for an example. - - If you set `validate_by_alias` to `False`, under the hood, Pydantic dynamically sets - `validate_by_name` to `True` to ensure that validation can still occur. - - /// version-added | v2.11 - This setting was introduced in conjunction with [`validate_by_name`][pydantic.ConfigDict.validate_by_name] - to empower users with more fine grained validation control. - /// - """ - - validate_by_name: bool - """ - Whether an aliased field may be populated by its name as given by the model - attribute. Defaults to `False`. - - ```python - from pydantic import BaseModel, ConfigDict, Field - - class Model(BaseModel): - model_config = ConfigDict(validate_by_name=True, validate_by_alias=True) - - my_field: str = Field(validation_alias='my_alias') # (1)! - - m = Model(my_alias='foo') # (2)! - print(m) - #> my_field='foo' - - m = Model(my_field='foo') # (3)! - print(m) - #> my_field='foo' - ``` - - 1. The field `'my_field'` has an alias `'my_alias'`. - 2. The model is populated by the alias `'my_alias'`. - 3. The model is populated by the attribute name `'my_field'`. - - !!! warning - You cannot set both `validate_by_alias` and `validate_by_name` to `False`. - This would make it impossible to populate an attribute. - - See [usage errors](../errors/usage_errors.md#validate-by-alias-and-name-false) for an example. - - /// version-added | v2.11 - This setting was introduced in conjunction with [`validate_by_alias`][pydantic.ConfigDict.validate_by_alias] - to empower users with more fine grained validation control. It is an alternative to [`populate_by_name`][pydantic.ConfigDict.populate_by_name], - that enables validation by name **and** by alias. - /// - """ - - serialize_by_alias: bool - """ - Whether an aliased field should be serialized by its alias. Defaults to `False`. - - Note: In v2.11, `serialize_by_alias` was introduced to address the - [popular request](https://github.com/pydantic/pydantic/issues/8379) - for consistency with alias behavior for validation and serialization settings. - In v3, the default value is expected to change to `True` for consistency with the validation default. - - ```python - from pydantic import BaseModel, ConfigDict, Field - - class Model(BaseModel): - model_config = ConfigDict(serialize_by_alias=True) - - my_field: str = Field(serialization_alias='my_alias') # (1)! - - m = Model(my_field='foo') - print(m.model_dump()) # (2)! - #> {'my_alias': 'foo'} - ``` - - 1. The field `'my_field'` has an alias `'my_alias'`. - 2. The model is serialized using the alias `'my_alias'` for the `'my_field'` attribute. - - - /// version-added | v2.11 - This setting was introduced to address the [popular request](https://github.com/pydantic/pydantic/issues/8379) - for consistency with alias behavior for validation and serialization. - - In v3, the default value is expected to change to `True` for consistency with the validation default. - /// - """ - - url_preserve_empty_path: bool - """ - Whether to preserve empty URL paths when validating values for a URL type. Defaults to `False`. - - ```python - from pydantic import AnyUrl, BaseModel, ConfigDict - - class Model(BaseModel): - model_config = ConfigDict(url_preserve_empty_path=True) - - url: AnyUrl - - m = Model(url='http://example.com') - print(m.url) - #> http://example.com - ``` - - /// version-added | v2.12 - /// - """ - - polymorphic_serialization: bool - """ - Whether to use polymorphic serialization for subclasses of the model or Pydantic dataclass. Defaults to `False`. - """ - - -_TypeT = TypeVar('_TypeT', bound=type) - - -@overload -@deprecated('Passing `config` as a keyword argument is deprecated. Pass `config` as a positional argument instead.') -def with_config(*, config: ConfigDict) -> Callable[[_TypeT], _TypeT]: ... - - -@overload -def with_config(config: ConfigDict, /) -> Callable[[_TypeT], _TypeT]: ... - - -@overload -def with_config(**config: Unpack[ConfigDict]) -> Callable[[_TypeT], _TypeT]: ... - - -def with_config(config: ConfigDict | None = None, /, **kwargs: Any) -> Callable[[_TypeT], _TypeT]: - """!!! abstract "Usage Documentation" - [Configuration with other types](../concepts/config.md#configuration-on-other-supported-types) - - A convenience decorator to set a [Pydantic configuration](config.md) on a `TypedDict` or a `dataclass` from the standard library. - - Although the configuration can be set using the `__pydantic_config__` attribute, it does not play well with type checkers, - especially with `TypedDict`. - - !!! example "Usage" - - ```python - from typing_extensions import TypedDict - - from pydantic import ConfigDict, TypeAdapter, with_config - - @with_config(ConfigDict(str_to_lower=True)) - class TD(TypedDict): - x: str - - ta = TypeAdapter(TD) - - print(ta.validate_python({'x': 'ABC'})) - #> {'x': 'abc'} - ``` - - /// deprecated-removed | v2.11 v3 - Passing `config` as a keyword argument. - /// - - /// version-changed | v2.11 - Keyword arguments can be provided directly instead of a config dictionary. - /// - """ - if config is not None and kwargs: - raise ValueError('Cannot specify both `config` and keyword arguments') - - if len(kwargs) == 1 and (kwargs_conf := kwargs.get('config')) is not None: - warnings.warn( - 'Passing `config` as a keyword argument is deprecated. Pass `config` as a positional argument instead', - category=PydanticDeprecatedSince211, - stacklevel=2, - ) - final_config = cast(ConfigDict, kwargs_conf) - else: - final_config = config if config is not None else cast(ConfigDict, kwargs) - - def inner(class_: _TypeT, /) -> _TypeT: - # Ideally, we would check for `class_` to either be a `TypedDict` or a stdlib dataclass. - # However, the `@with_config` decorator can be applied *after* `@dataclass`. To avoid - # common mistakes, we at least check for `class_` to not be a Pydantic model. - from ._internal._utils import is_model_class - - if is_model_class(class_): - raise PydanticUserError( - f'Cannot use `with_config` on {class_.__name__} as it is a Pydantic model', - code='with-config-on-model', - ) - class_.__pydantic_config__ = final_config - return class_ - - return inner - - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/dataclasses.py b/functions/python/data-processor/pydantic/dataclasses.py deleted file mode 100644 index 46401c310..000000000 --- a/functions/python/data-processor/pydantic/dataclasses.py +++ /dev/null @@ -1,413 +0,0 @@ -"""Provide an enhanced dataclass that performs validation.""" - -from __future__ import annotations as _annotations - -import dataclasses -import functools -import sys -import types -from typing import TYPE_CHECKING, Any, Callable, Generic, Literal, NoReturn, TypeVar, overload -from warnings import warn - -from typing_extensions import TypeGuard, dataclass_transform - -from ._internal import _config, _decorators, _mock_val_ser, _namespace_utils, _typing_extra -from ._internal import _dataclasses as _pydantic_dataclasses -from ._migration import getattr_migration -from .config import ConfigDict -from .errors import PydanticUserError -from .fields import Field, FieldInfo, PrivateAttr - -if TYPE_CHECKING: - from ._internal._dataclasses import PydanticDataclass - from ._internal._namespace_utils import MappingNamespace - -__all__ = 'dataclass', 'rebuild_dataclass' - -_T = TypeVar('_T') - -if sys.version_info >= (3, 10): - - @dataclass_transform(field_specifiers=(dataclasses.field, Field, PrivateAttr)) - @overload - def dataclass( - *, - init: Literal[False] = False, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: ConfigDict | type[object] | None = None, - validate_on_init: bool | None = None, - kw_only: bool = ..., - slots: bool = ..., - ) -> Callable[[type[_T]], type[PydanticDataclass]]: # type: ignore - ... - - @dataclass_transform(field_specifiers=(dataclasses.field, Field, PrivateAttr)) - @overload - def dataclass( - _cls: type[_T], # type: ignore - *, - init: Literal[False] = False, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool | None = None, - config: ConfigDict | type[object] | None = None, - validate_on_init: bool | None = None, - kw_only: bool = ..., - slots: bool = ..., - ) -> type[PydanticDataclass]: ... - -else: - - @dataclass_transform(field_specifiers=(dataclasses.field, Field, PrivateAttr)) - @overload - def dataclass( - *, - init: Literal[False] = False, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool | None = None, - config: ConfigDict | type[object] | None = None, - validate_on_init: bool | None = None, - ) -> Callable[[type[_T]], type[PydanticDataclass]]: # type: ignore - ... - - @dataclass_transform(field_specifiers=(dataclasses.field, Field, PrivateAttr)) - @overload - def dataclass( - _cls: type[_T], # type: ignore - *, - init: Literal[False] = False, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool | None = None, - config: ConfigDict | type[object] | None = None, - validate_on_init: bool | None = None, - ) -> type[PydanticDataclass]: ... - - -@dataclass_transform(field_specifiers=(dataclasses.field, Field, PrivateAttr)) -def dataclass( - _cls: type[_T] | None = None, - *, - init: Literal[False] = False, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool | None = None, - config: ConfigDict | type[object] | None = None, - validate_on_init: bool | None = None, - kw_only: bool = False, - slots: bool = False, -) -> Callable[[type[_T]], type[PydanticDataclass]] | type[PydanticDataclass]: - """!!! abstract "Usage Documentation" - [`dataclasses`](../concepts/dataclasses.md) - - A decorator used to create a Pydantic-enhanced dataclass, similar to the standard Python `dataclass`, - but with added validation. - - This function should be used similarly to `dataclasses.dataclass`. - - Args: - _cls: The target `dataclass`. - init: Included for signature compatibility with `dataclasses.dataclass`, and is passed through to - `dataclasses.dataclass` when appropriate. If specified, must be set to `False`, as pydantic inserts its - own `__init__` function. - repr: A boolean indicating whether to include the field in the `__repr__` output. - eq: Determines if a `__eq__` method should be generated for the class. - order: Determines if comparison magic methods should be generated, such as `__lt__`, but not `__eq__`. - unsafe_hash: Determines if a `__hash__` method should be included in the class, as in `dataclasses.dataclass`. - frozen: Determines if the generated class should be a 'frozen' `dataclass`, which does not allow its - attributes to be modified after it has been initialized. If not set, the value from the provided `config` argument will be used (and will default to `False` otherwise). - config: The Pydantic config to use for the `dataclass`. - validate_on_init: A deprecated parameter included for backwards compatibility; in V2, all Pydantic dataclasses - are validated on init. - kw_only: Determines if `__init__` method parameters must be specified by keyword only. Defaults to `False`. - slots: Determines if the generated class should be a 'slots' `dataclass`, which does not allow the addition of - new attributes after instantiation. - - Returns: - A decorator that accepts a class as its argument and returns a Pydantic `dataclass`. - - Raises: - AssertionError: Raised if `init` is not `False` or `validate_on_init` is `False`. - """ - assert init is False, 'pydantic.dataclasses.dataclass only supports init=False' - assert validate_on_init is not False, 'validate_on_init=False is no longer supported' - - if sys.version_info >= (3, 10): - kwargs = {'kw_only': kw_only, 'slots': slots} - else: - kwargs = {} - - def create_dataclass(cls: type[Any]) -> type[PydanticDataclass]: - """Create a Pydantic dataclass from a regular dataclass. - - Args: - cls: The class to create the Pydantic dataclass from. - - Returns: - A Pydantic dataclass. - """ - from ._internal._utils import is_model_class - - if is_model_class(cls): - raise PydanticUserError( - f'Cannot create a Pydantic dataclass from {cls.__name__} as it is already a Pydantic model', - code='dataclass-on-model', - ) - - original_cls = cls - - # we warn on conflicting config specifications, but only if the class doesn't have a dataclass base - # because a dataclass base might provide a __pydantic_config__ attribute that we don't want to warn about - has_dataclass_base = any(dataclasses.is_dataclass(base) for base in cls.__bases__) - if not has_dataclass_base and config is not None and hasattr(cls, '__pydantic_config__'): - warn( - f'`config` is set via both the `dataclass` decorator and `__pydantic_config__` for dataclass {cls.__name__}. ' - f'The `config` specification from `dataclass` decorator will take priority.', - category=UserWarning, - stacklevel=2, - ) - - # if config is not explicitly provided, try to read it from the type - config_dict = config if config is not None else getattr(cls, '__pydantic_config__', None) - config_wrapper = _config.ConfigWrapper(config_dict) - decorators = _decorators.DecoratorInfos.build(cls, replace_wrapped_methods=True) - decorators.update_from_config(config_wrapper) - - # Keep track of the original __doc__ so that we can restore it after applying the dataclasses decorator - # Otherwise, classes with no __doc__ will have their signature added into the JSON schema description, - # since dataclasses.dataclass will set this as the __doc__ - original_doc = cls.__doc__ - - if _pydantic_dataclasses.is_stdlib_dataclass(cls): - # Vanilla dataclasses include a default docstring (representing the class signature), - # which we don't want to preserve. - original_doc = None - - # We don't want to add validation to the existing std lib dataclass, so we will subclass it - # If the class is generic, we need to make sure the subclass also inherits from Generic - # with all the same parameters. - bases = (cls,) - if issubclass(cls, Generic): - generic_base = Generic[cls.__parameters__] # type: ignore - bases = bases + (generic_base,) - cls = types.new_class(cls.__name__, bases) - - # Respect frozen setting from dataclass constructor and fallback to config setting if not provided - if frozen is not None: - frozen_ = frozen - if config_wrapper.frozen: - # It's not recommended to define both, as the setting from the dataclass decorator will take priority. - warn( - f'`frozen` is set via both the `dataclass` decorator and `config` for dataclass {cls.__name__!r}.' - 'This is not recommended. The `frozen` specification on `dataclass` will take priority.', - category=UserWarning, - stacklevel=2, - ) - else: - frozen_ = config_wrapper.frozen or False - - # Make Pydantic's `Field()` function compatible with stdlib dataclasses. As we'll decorate - # `cls` with the stdlib `@dataclass` decorator first, there are two attributes, `kw_only` and - # `repr` that need to be understood *during* the stdlib creation. We do so in two steps: - - # 1. On the decorated class, wrap `Field()` assignment with `dataclass.field()`, with the - # two attributes set (done in `as_dataclass_field()`) - cls_anns = _typing_extra.safe_get_annotations(cls) - for field_name in cls_anns: - # We should look for assignments in `__dict__` instead, but for now we follow - # the same behavior as stdlib dataclasses (see https://github.com/python/cpython/issues/88609) - field_value = getattr(cls, field_name, None) - if isinstance(field_value, FieldInfo): - setattr(cls, field_name, _pydantic_dataclasses.as_dataclass_field(field_value)) - - # 2. For bases of `cls` that are stdlib dataclasses, we temporarily patch their fields - # (see the docstring of the context manager): - with _pydantic_dataclasses.patch_base_fields(cls): - cls = dataclasses.dataclass( # pyright: ignore[reportCallIssue] - cls, - # the value of init here doesn't affect anything except that it makes it easier to generate a signature - init=True, - repr=repr, - eq=eq, - order=order, - unsafe_hash=unsafe_hash, - frozen=frozen_, - **kwargs, - ) - - if config_wrapper.validate_assignment: - original_setattr = cls.__setattr__ - - @functools.wraps(cls.__setattr__) - def validated_setattr(instance: PydanticDataclass, name: str, value: Any, /) -> None: - if frozen_: - return original_setattr(instance, name, value) # pyright: ignore[reportCallIssue] - inst_cls = type(instance) - attr = getattr(inst_cls, name, None) - - if isinstance(attr, property): - attr.__set__(instance, value) - elif isinstance(attr, functools.cached_property): - instance.__dict__.__setitem__(name, value) - else: - inst_cls.__pydantic_validator__.validate_assignment(instance, name, value) - - cls.__setattr__ = validated_setattr.__get__(None, cls) # type: ignore - - if slots and not hasattr(cls, '__setstate__'): - # If slots is set, `pickle` (relied on by `copy.copy()`) will use - # `__setattr__()` to reconstruct the dataclass. However, the custom - # `__setattr__()` set above relies on `validate_assignment()`, which - # in turn expects all the field values to be already present on the - # instance, resulting in attribute errors. - # As such, we make use of `object.__setattr__()` instead. - # Note that we do so only if `__setstate__()` isn't already set (this is the - # case if on top of `slots`, `frozen` is used). - - # Taken from `dataclasses._dataclass_get/setstate()`: - def _dataclass_getstate(self: Any) -> list[Any]: - return [getattr(self, f.name) for f in dataclasses.fields(self)] - - def _dataclass_setstate(self: Any, state: list[Any]) -> None: - for field, value in zip(dataclasses.fields(self), state): - object.__setattr__(self, field.name, value) - - cls.__getstate__ = _dataclass_getstate # pyright: ignore[reportAttributeAccessIssue] - cls.__setstate__ = _dataclass_setstate # pyright: ignore[reportAttributeAccessIssue] - - # This is an undocumented attribute to distinguish stdlib/Pydantic dataclasses. - # It should be set as early as possible: - cls.__is_pydantic_dataclass__ = True - cls.__pydantic_decorators__ = decorators # type: ignore - cls.__doc__ = original_doc - # Can be non-existent for dynamically created classes: - firstlineno = getattr(original_cls, '__firstlineno__', None) - cls.__module__ = original_cls.__module__ - if sys.version_info >= (3, 13) and firstlineno is not None: - # As per https://docs.python.org/3/reference/datamodel.html#type.__firstlineno__: - # Setting the `__module__` attribute removes the `__firstlineno__` item from the type’s dictionary. - original_cls.__firstlineno__ = firstlineno - cls.__firstlineno__ = firstlineno - cls.__qualname__ = original_cls.__qualname__ - cls.__pydantic_fields_complete__ = classmethod(_pydantic_fields_complete) - cls.__pydantic_complete__ = False # `complete_dataclass` will set it to `True` if successful. - # TODO `parent_namespace` is currently None, but we could do the same thing as Pydantic models: - # fetch the parent ns using `parent_frame_namespace` (if the dataclass was defined in a function), - # and possibly cache it (see the `__pydantic_parent_namespace__` logic for models). - _pydantic_dataclasses.complete_dataclass(cls, config_wrapper, raise_errors=False) - return cls - - return create_dataclass if _cls is None else create_dataclass(_cls) - - -def _pydantic_fields_complete(cls: type[PydanticDataclass]) -> bool: - """Return whether the fields were successfully collected (i.e. type hints were successfully resolved). - - This is a private helper, not meant to be used outside Pydantic. - """ - return all(field_info._complete for field_info in cls.__pydantic_fields__.values()) - - -__getattr__ = getattr_migration(__name__) - -if sys.version_info < (3, 11): - # Monkeypatch dataclasses.InitVar so that typing doesn't error if it occurs as a type when evaluating type hints - # Starting in 3.11, typing.get_type_hints will not raise an error if the retrieved type hints are not callable. - - def _call_initvar(*args: Any, **kwargs: Any) -> NoReturn: - """This function does nothing but raise an error that is as similar as possible to what you'd get - if you were to try calling `InitVar[int]()` without this monkeypatch. The whole purpose is just - to ensure typing._type_check does not error if the type hint evaluates to `InitVar[]`. - """ - raise TypeError("'InitVar' object is not callable") - - dataclasses.InitVar.__call__ = _call_initvar - - -def rebuild_dataclass( - cls: type[PydanticDataclass], - *, - force: bool = False, - raise_errors: bool = True, - _parent_namespace_depth: int = 2, - _types_namespace: MappingNamespace | None = None, -) -> bool | None: - """Try to rebuild the pydantic-core schema for the dataclass. - - This may be necessary when one of the annotations is a ForwardRef which could not be resolved during - the initial attempt to build the schema, and automatic rebuilding fails. - - This is analogous to `BaseModel.model_rebuild`. - - Args: - cls: The class to rebuild the pydantic-core schema for. - force: Whether to force the rebuilding of the schema, defaults to `False`. - raise_errors: Whether to raise errors, defaults to `True`. - _parent_namespace_depth: The depth level of the parent namespace, defaults to 2. - _types_namespace: The types namespace, defaults to `None`. - - Returns: - Returns `None` if the schema is already "complete" and rebuilding was not required. - If rebuilding _was_ required, returns `True` if rebuilding was successful, otherwise `False`. - """ - if not force and cls.__pydantic_complete__: - return None - - for attr in ('__pydantic_core_schema__', '__pydantic_validator__', '__pydantic_serializer__'): - if attr in cls.__dict__ and not isinstance(getattr(cls, attr), _mock_val_ser.MockValSer): - # Deleting the validator/serializer is necessary as otherwise they can get reused in - # pydantic-core. Same applies for the core schema that can be reused in schema generation. - delattr(cls, attr) - - cls.__pydantic_complete__ = False - - if _types_namespace is not None: - rebuild_ns = _types_namespace - elif _parent_namespace_depth > 0: - rebuild_ns = _typing_extra.parent_frame_namespace(parent_depth=_parent_namespace_depth, force=True) or {} - else: - rebuild_ns = {} - - ns_resolver = _namespace_utils.NsResolver( - parent_namespace=rebuild_ns, - ) - - return _pydantic_dataclasses.complete_dataclass( - cls, - _config.ConfigWrapper(cls.__pydantic_config__, check=False), - raise_errors=raise_errors, - ns_resolver=ns_resolver, - # We could provide a different config instead (with `'defer_build'` set to `True`) - # of this explicit `_force_build` argument, but because config can come from the - # decorator parameter or the `__pydantic_config__` attribute, `complete_dataclass` - # will overwrite `__pydantic_config__` with the provided config above: - _force_build=True, - ) - - -def is_pydantic_dataclass(class_: type[Any], /) -> TypeGuard[type[PydanticDataclass]]: - """Whether a class is a pydantic dataclass. - - Args: - class_: The class. - - Returns: - `True` if the class is a pydantic dataclass, `False` otherwise. - """ - try: - return '__is_pydantic_dataclass__' in class_.__dict__ and dataclasses.is_dataclass(class_) - except AttributeError: - return False diff --git a/functions/python/data-processor/pydantic/datetime_parse.py b/functions/python/data-processor/pydantic/datetime_parse.py deleted file mode 100644 index 53d52649e..000000000 --- a/functions/python/data-processor/pydantic/datetime_parse.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `datetime_parse` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/decorator.py b/functions/python/data-processor/pydantic/decorator.py deleted file mode 100644 index 0d97560c1..000000000 --- a/functions/python/data-processor/pydantic/decorator.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `decorator` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/deprecated/__init__.py b/functions/python/data-processor/pydantic/deprecated/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/pydantic/deprecated/class_validators.py b/functions/python/data-processor/pydantic/deprecated/class_validators.py deleted file mode 100644 index e6ba19506..000000000 --- a/functions/python/data-processor/pydantic/deprecated/class_validators.py +++ /dev/null @@ -1,256 +0,0 @@ -"""Old `@validator` and `@root_validator` function validators from V1.""" - -from __future__ import annotations as _annotations - -from functools import partial, partialmethod -from types import FunctionType -from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar, Union, overload -from warnings import warn - -from typing_extensions import Protocol, TypeAlias, deprecated - -from .._internal import _decorators, _decorators_v1 -from ..errors import PydanticUserError -from ..warnings import PydanticDeprecatedSince20 - -_ALLOW_REUSE_WARNING_MESSAGE = '`allow_reuse` is deprecated and will be ignored; it should no longer be necessary' - - -if TYPE_CHECKING: - - class _OnlyValueValidatorClsMethod(Protocol): - def __call__(self, __cls: Any, __value: Any) -> Any: ... - - class _V1ValidatorWithValuesClsMethod(Protocol): - def __call__(self, __cls: Any, __value: Any, values: dict[str, Any]) -> Any: ... - - class _V1ValidatorWithValuesKwOnlyClsMethod(Protocol): - def __call__(self, __cls: Any, __value: Any, *, values: dict[str, Any]) -> Any: ... - - class _V1ValidatorWithKwargsClsMethod(Protocol): - def __call__(self, __cls: Any, **kwargs: Any) -> Any: ... - - class _V1ValidatorWithValuesAndKwargsClsMethod(Protocol): - def __call__(self, __cls: Any, values: dict[str, Any], **kwargs: Any) -> Any: ... - - class _V1RootValidatorClsMethod(Protocol): - def __call__( - self, __cls: Any, __values: _decorators_v1.RootValidatorValues - ) -> _decorators_v1.RootValidatorValues: ... - - V1Validator = Union[ - _OnlyValueValidatorClsMethod, - _V1ValidatorWithValuesClsMethod, - _V1ValidatorWithValuesKwOnlyClsMethod, - _V1ValidatorWithKwargsClsMethod, - _V1ValidatorWithValuesAndKwargsClsMethod, - _decorators_v1.V1ValidatorWithValues, - _decorators_v1.V1ValidatorWithValuesKwOnly, - _decorators_v1.V1ValidatorWithKwargs, - _decorators_v1.V1ValidatorWithValuesAndKwargs, - ] - - V1RootValidator = Union[ - _V1RootValidatorClsMethod, - _decorators_v1.V1RootValidatorFunction, - ] - - _PartialClsOrStaticMethod: TypeAlias = Union[classmethod[Any, Any, Any], staticmethod[Any, Any], partialmethod[Any]] - - # Allow both a V1 (assumed pre=False) or V2 (assumed mode='after') validator - # We lie to type checkers and say we return the same thing we get - # but in reality we return a proxy object that _mostly_ behaves like the wrapped thing - _V1ValidatorType = TypeVar('_V1ValidatorType', V1Validator, _PartialClsOrStaticMethod) - _V1RootValidatorFunctionType = TypeVar( - '_V1RootValidatorFunctionType', - _decorators_v1.V1RootValidatorFunction, - _V1RootValidatorClsMethod, - _PartialClsOrStaticMethod, - ) -else: - # See PyCharm issues https://youtrack.jetbrains.com/issue/PY-21915 - # and https://youtrack.jetbrains.com/issue/PY-51428 - DeprecationWarning = PydanticDeprecatedSince20 - - -@deprecated( - 'Pydantic V1 style `@validator` validators are deprecated.' - ' You should migrate to Pydantic V2 style `@field_validator` validators,' - ' see the migration guide for more details', - category=None, -) -def validator( - __field: str, - *fields: str, - pre: bool = False, - each_item: bool = False, - always: bool = False, - check_fields: bool | None = None, - allow_reuse: bool = False, -) -> Callable[[_V1ValidatorType], _V1ValidatorType]: - """Decorate methods on the class indicating that they should be used to validate fields. - - Args: - __field (str): The first field the validator should be called on; this is separate - from `fields` to ensure an error is raised if you don't pass at least one. - *fields (str): Additional field(s) the validator should be called on. - pre (bool, optional): Whether this validator should be called before the standard - validators (else after). Defaults to False. - each_item (bool, optional): For complex objects (sets, lists etc.) whether to validate - individual elements rather than the whole object. Defaults to False. - always (bool, optional): Whether this method and other validators should be called even if - the value is missing. Defaults to False. - check_fields (bool | None, optional): Whether to check that the fields actually exist on the model. - Defaults to None. - allow_reuse (bool, optional): Whether to track and raise an error if another validator refers to - the decorated function. Defaults to False. - - Returns: - Callable: A decorator that can be used to decorate a - function to be used as a validator. - """ - warn( - 'Pydantic V1 style `@validator` validators are deprecated.' - ' You should migrate to Pydantic V2 style `@field_validator` validators,' - ' see the migration guide for more details', - DeprecationWarning, - stacklevel=2, - ) - - if allow_reuse is True: # pragma: no cover - warn(_ALLOW_REUSE_WARNING_MESSAGE, DeprecationWarning, stacklevel=2) - fields = __field, *fields - if isinstance(fields[0], FunctionType): - raise PydanticUserError( - '`@validator` should be used with fields and keyword arguments, not bare. ' - "E.g. usage should be `@validator('', ...)`", - code='decorator-missing-arguments', - ) - elif not all(isinstance(field, str) for field in fields): - raise PydanticUserError( - '`@validator` fields should be passed as separate string args. ' - "E.g. usage should be `@validator('', '', ...)`", - code='decorator-invalid-fields', - ) - - mode: Literal['before', 'after'] = 'before' if pre is True else 'after' - - def dec(f: Any) -> _decorators.PydanticDescriptorProxy[Any]: - if _decorators.is_instance_method_from_sig(f): - raise PydanticUserError( - '`@validator` cannot be applied to instance methods', code='validator-instance-method' - ) - # auto apply the @classmethod decorator - f = _decorators.ensure_classmethod_based_on_signature(f) - wrap = _decorators_v1.make_generic_v1_field_validator - validator_wrapper_info = _decorators.ValidatorDecoratorInfo( - fields=fields, - mode=mode, - each_item=each_item, - always=always, - check_fields=check_fields, - ) - return _decorators.PydanticDescriptorProxy(f, validator_wrapper_info, shim=wrap) - - return dec # type: ignore[return-value] - - -@overload -def root_validator( - *, - # if you don't specify `pre` the default is `pre=False` - # which means you need to specify `skip_on_failure=True` - skip_on_failure: Literal[True], - allow_reuse: bool = ..., -) -> Callable[ - [_V1RootValidatorFunctionType], - _V1RootValidatorFunctionType, -]: ... - - -@overload -def root_validator( - *, - # if you specify `pre=True` then you don't need to specify - # `skip_on_failure`, in fact it is not allowed as an argument! - pre: Literal[True], - allow_reuse: bool = ..., -) -> Callable[ - [_V1RootValidatorFunctionType], - _V1RootValidatorFunctionType, -]: ... - - -@overload -def root_validator( - *, - # if you explicitly specify `pre=False` then you - # MUST specify `skip_on_failure=True` - pre: Literal[False], - skip_on_failure: Literal[True], - allow_reuse: bool = ..., -) -> Callable[ - [_V1RootValidatorFunctionType], - _V1RootValidatorFunctionType, -]: ... - - -@deprecated( - 'Pydantic V1 style `@root_validator` validators are deprecated.' - ' You should migrate to Pydantic V2 style `@model_validator` validators,' - ' see the migration guide for more details', - category=None, -) -def root_validator( - *__args, - pre: bool = False, - skip_on_failure: bool = False, - allow_reuse: bool = False, -) -> Any: - """Decorate methods on a model indicating that they should be used to validate (and perhaps - modify) data either before or after standard model parsing/validation is performed. - - Args: - pre (bool, optional): Whether this validator should be called before the standard - validators (else after). Defaults to False. - skip_on_failure (bool, optional): Whether to stop validation and return as soon as a - failure is encountered. Defaults to False. - allow_reuse (bool, optional): Whether to track and raise an error if another validator - refers to the decorated function. Defaults to False. - - Returns: - Any: A decorator that can be used to decorate a function to be used as a root_validator. - """ - warn( - 'Pydantic V1 style `@root_validator` validators are deprecated.' - ' You should migrate to Pydantic V2 style `@model_validator` validators,' - ' see the migration guide for more details', - DeprecationWarning, - stacklevel=2, - ) - - if __args: - # Ensure a nice error is raised if someone attempts to use the bare decorator - return root_validator()(*__args) # type: ignore - - if allow_reuse is True: # pragma: no cover - warn(_ALLOW_REUSE_WARNING_MESSAGE, DeprecationWarning, stacklevel=2) - mode: Literal['before', 'after'] = 'before' if pre is True else 'after' - if pre is False and skip_on_failure is not True: - raise PydanticUserError( - 'If you use `@root_validator` with pre=False (the default) you MUST specify `skip_on_failure=True`.' - ' Note that `@root_validator` is deprecated and should be replaced with `@model_validator`.', - code='root-validator-pre-skip', - ) - - wrap = partial(_decorators_v1.make_v1_generic_root_validator, pre=pre) - - def dec(f: Callable[..., Any] | classmethod[Any, Any, Any] | staticmethod[Any, Any]) -> Any: - if _decorators.is_instance_method_from_sig(f): - raise TypeError('`@root_validator` cannot be applied to instance methods') - # auto apply the @classmethod decorator - res = _decorators.ensure_classmethod_based_on_signature(f) - dec_info = _decorators.RootValidatorDecoratorInfo(mode=mode) - return _decorators.PydanticDescriptorProxy(res, dec_info, shim=wrap) - - return dec diff --git a/functions/python/data-processor/pydantic/deprecated/config.py b/functions/python/data-processor/pydantic/deprecated/config.py deleted file mode 100644 index bd4692ace..000000000 --- a/functions/python/data-processor/pydantic/deprecated/config.py +++ /dev/null @@ -1,72 +0,0 @@ -from __future__ import annotations as _annotations - -import warnings -from typing import TYPE_CHECKING, Any, Literal - -from typing_extensions import deprecated - -from .._internal import _config -from ..warnings import PydanticDeprecatedSince20 - -if not TYPE_CHECKING: - # See PyCharm issues https://youtrack.jetbrains.com/issue/PY-21915 - # and https://youtrack.jetbrains.com/issue/PY-51428 - DeprecationWarning = PydanticDeprecatedSince20 - -__all__ = 'BaseConfig', 'Extra' - - -class _ConfigMetaclass(type): - def __getattr__(self, item: str) -> Any: - try: - obj = _config.config_defaults[item] - warnings.warn(_config.DEPRECATION_MESSAGE, DeprecationWarning) - return obj - except KeyError as exc: - raise AttributeError(f"type object '{self.__name__}' has no attribute {exc}") from exc - - -@deprecated('BaseConfig is deprecated. Use the `pydantic.ConfigDict` instead.', category=PydanticDeprecatedSince20) -class BaseConfig(metaclass=_ConfigMetaclass): - """This class is only retained for backwards compatibility. - - !!! Warning "Deprecated" - BaseConfig is deprecated. Use the [`pydantic.ConfigDict`][pydantic.ConfigDict] instead. - """ - - def __getattr__(self, item: str) -> Any: - try: - obj = super().__getattribute__(item) - warnings.warn(_config.DEPRECATION_MESSAGE, DeprecationWarning) - return obj - except AttributeError as exc: - try: - return getattr(type(self), item) - except AttributeError: - # re-raising changes the displayed text to reflect that `self` is not a type - raise AttributeError(str(exc)) from exc - - def __init_subclass__(cls, **kwargs: Any) -> None: - warnings.warn(_config.DEPRECATION_MESSAGE, DeprecationWarning) - return super().__init_subclass__(**kwargs) - - -class _ExtraMeta(type): - def __getattribute__(self, __name: str) -> Any: - # The @deprecated decorator accesses other attributes, so we only emit a warning for the expected ones - if __name in {'allow', 'ignore', 'forbid'}: - warnings.warn( - "`pydantic.config.Extra` is deprecated, use literal values instead (e.g. `extra='allow'`)", - DeprecationWarning, - stacklevel=2, - ) - return super().__getattribute__(__name) - - -@deprecated( - "Extra is deprecated. Use literal values instead (e.g. `extra='allow'`)", category=PydanticDeprecatedSince20 -) -class Extra(metaclass=_ExtraMeta): - allow: Literal['allow'] = 'allow' - ignore: Literal['ignore'] = 'ignore' - forbid: Literal['forbid'] = 'forbid' diff --git a/functions/python/data-processor/pydantic/deprecated/copy_internals.py b/functions/python/data-processor/pydantic/deprecated/copy_internals.py deleted file mode 100644 index 0170dc08b..000000000 --- a/functions/python/data-processor/pydantic/deprecated/copy_internals.py +++ /dev/null @@ -1,224 +0,0 @@ -from __future__ import annotations as _annotations - -import typing -from copy import deepcopy -from enum import Enum -from typing import Any - -import typing_extensions - -from .._internal import ( - _model_construction, - _typing_extra, - _utils, -) - -if typing.TYPE_CHECKING: - from .. import BaseModel - from .._internal._utils import AbstractSetIntStr, MappingIntStrAny - - AnyClassMethod = classmethod[Any, Any, Any] - TupleGenerator = typing.Generator[tuple[str, Any], None, None] - Model = typing.TypeVar('Model', bound='BaseModel') - # should be `set[int] | set[str] | dict[int, IncEx] | dict[str, IncEx] | None`, but mypy can't cope - IncEx: typing_extensions.TypeAlias = 'set[int] | set[str] | dict[int, Any] | dict[str, Any] | None' - -_object_setattr = _model_construction.object_setattr - - -def _iter( - self: BaseModel, - to_dict: bool = False, - by_alias: bool = False, - include: AbstractSetIntStr | MappingIntStrAny | None = None, - exclude: AbstractSetIntStr | MappingIntStrAny | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, -) -> TupleGenerator: - # Merge field set excludes with explicit exclude parameter with explicit overriding field set options. - # The extra "is not None" guards are not logically necessary but optimizes performance for the simple case. - if exclude is not None: - exclude = _utils.ValueItems.merge( - {k: v.exclude for k, v in self.__pydantic_fields__.items() if v.exclude is not None}, exclude - ) - - if include is not None: - include = _utils.ValueItems.merge(dict.fromkeys(self.__pydantic_fields__, True), include, intersect=True) - - allowed_keys = _calculate_keys(self, include=include, exclude=exclude, exclude_unset=exclude_unset) # type: ignore - if allowed_keys is None and not (to_dict or by_alias or exclude_unset or exclude_defaults or exclude_none): - # huge boost for plain _iter() - yield from self.__dict__.items() - if self.__pydantic_extra__: - yield from self.__pydantic_extra__.items() - return - - value_exclude = _utils.ValueItems(self, exclude) if exclude is not None else None - value_include = _utils.ValueItems(self, include) if include is not None else None - - if self.__pydantic_extra__ is None: - items = self.__dict__.items() - else: - items = list(self.__dict__.items()) + list(self.__pydantic_extra__.items()) - - for field_key, v in items: - if (allowed_keys is not None and field_key not in allowed_keys) or (exclude_none and v is None): - continue - - if exclude_defaults: - try: - field = self.__pydantic_fields__[field_key] - except KeyError: - pass - else: - if not field.is_required() and field.default == v: - continue - - if by_alias and field_key in self.__pydantic_fields__: - dict_key = self.__pydantic_fields__[field_key].alias or field_key - else: - dict_key = field_key - - if to_dict or value_include or value_exclude: - v = _get_value( - type(self), - v, - to_dict=to_dict, - by_alias=by_alias, - include=value_include and value_include.for_element(field_key), - exclude=value_exclude and value_exclude.for_element(field_key), - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - yield dict_key, v - - -def _copy_and_set_values( - self: Model, - values: dict[str, Any], - fields_set: set[str], - extra: dict[str, Any] | None = None, - private: dict[str, Any] | None = None, - *, - deep: bool, # UP006 -) -> Model: - if deep: - # chances of having empty dict here are quite low for using smart_deepcopy - values = deepcopy(values) - extra = deepcopy(extra) - private = deepcopy(private) - - cls = self.__class__ - m = cls.__new__(cls) - _object_setattr(m, '__dict__', values) - _object_setattr(m, '__pydantic_extra__', extra) - _object_setattr(m, '__pydantic_fields_set__', fields_set) - _object_setattr(m, '__pydantic_private__', private) - - return m - - -@typing.no_type_check -def _get_value( - cls: type[BaseModel], - v: Any, - to_dict: bool, - by_alias: bool, - include: AbstractSetIntStr | MappingIntStrAny | None, - exclude: AbstractSetIntStr | MappingIntStrAny | None, - exclude_unset: bool, - exclude_defaults: bool, - exclude_none: bool, -) -> Any: - from .. import BaseModel - - if isinstance(v, BaseModel): - if to_dict: - return v.model_dump( - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - include=include, # type: ignore - exclude=exclude, # type: ignore - exclude_none=exclude_none, - ) - else: - return v.copy(include=include, exclude=exclude) - - value_exclude = _utils.ValueItems(v, exclude) if exclude else None - value_include = _utils.ValueItems(v, include) if include else None - - if isinstance(v, dict): - return { - k_: _get_value( - cls, - v_, - to_dict=to_dict, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - include=value_include and value_include.for_element(k_), - exclude=value_exclude and value_exclude.for_element(k_), - exclude_none=exclude_none, - ) - for k_, v_ in v.items() - if (not value_exclude or not value_exclude.is_excluded(k_)) - and (not value_include or value_include.is_included(k_)) - } - - elif _utils.sequence_like(v): - seq_args = ( - _get_value( - cls, - v_, - to_dict=to_dict, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - include=value_include and value_include.for_element(i), - exclude=value_exclude and value_exclude.for_element(i), - exclude_none=exclude_none, - ) - for i, v_ in enumerate(v) - if (not value_exclude or not value_exclude.is_excluded(i)) - and (not value_include or value_include.is_included(i)) - ) - - return v.__class__(*seq_args) if _typing_extra.is_namedtuple(v.__class__) else v.__class__(seq_args) - - elif isinstance(v, Enum) and getattr(cls.model_config, 'use_enum_values', False): - return v.value - - else: - return v - - -def _calculate_keys( - self: BaseModel, - include: MappingIntStrAny | None, - exclude: MappingIntStrAny | None, - exclude_unset: bool, - update: dict[str, Any] | None = None, # noqa UP006 -) -> typing.AbstractSet[str] | None: - if include is None and exclude is None and exclude_unset is False: - return None - - keys: typing.AbstractSet[str] - if exclude_unset: - keys = self.__pydantic_fields_set__.copy() - else: - keys = set(self.__dict__.keys()) - keys = keys | (self.__pydantic_extra__ or {}).keys() - - if include is not None: - keys &= include.keys() - - if update: - keys -= update.keys() - - if exclude: - keys -= {k for k, v in exclude.items() if _utils.ValueItems.is_true(v)} - - return keys diff --git a/functions/python/data-processor/pydantic/deprecated/decorator.py b/functions/python/data-processor/pydantic/deprecated/decorator.py deleted file mode 100644 index e73ad209a..000000000 --- a/functions/python/data-processor/pydantic/deprecated/decorator.py +++ /dev/null @@ -1,284 +0,0 @@ -import warnings -from collections.abc import Mapping -from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Optional, TypeVar, Union, overload - -from typing_extensions import deprecated - -from .._internal import _config, _typing_extra -from ..alias_generators import to_pascal -from ..errors import PydanticUserError -from ..functional_validators import field_validator -from ..main import BaseModel, create_model -from ..warnings import PydanticDeprecatedSince20 - -if not TYPE_CHECKING: - # See PyCharm issues https://youtrack.jetbrains.com/issue/PY-21915 - # and https://youtrack.jetbrains.com/issue/PY-51428 - DeprecationWarning = PydanticDeprecatedSince20 - -__all__ = ('validate_arguments',) - -if TYPE_CHECKING: - AnyCallable = Callable[..., Any] - - AnyCallableT = TypeVar('AnyCallableT', bound=AnyCallable) - ConfigType = Union[None, type[Any], dict[str, Any]] - - -@overload -def validate_arguments( - func: None = None, *, config: 'ConfigType' = None -) -> Callable[['AnyCallableT'], 'AnyCallableT']: ... - - -@overload -def validate_arguments(func: 'AnyCallableT') -> 'AnyCallableT': ... - - -@deprecated( - 'The `validate_arguments` method is deprecated; use `validate_call` instead.', - category=None, -) -def validate_arguments(func: Optional['AnyCallableT'] = None, *, config: 'ConfigType' = None) -> Any: - """Decorator to validate the arguments passed to a function.""" - warnings.warn( - 'The `validate_arguments` method is deprecated; use `validate_call` instead.', - PydanticDeprecatedSince20, - stacklevel=2, - ) - - def validate(_func: 'AnyCallable') -> 'AnyCallable': - vd = ValidatedFunction(_func, config) - - @wraps(_func) - def wrapper_function(*args: Any, **kwargs: Any) -> Any: - return vd.call(*args, **kwargs) - - wrapper_function.vd = vd # type: ignore - wrapper_function.validate = vd.init_model_instance # type: ignore - wrapper_function.raw_function = vd.raw_function # type: ignore - wrapper_function.model = vd.model # type: ignore - return wrapper_function - - if func: - return validate(func) - else: - return validate - - -ALT_V_ARGS = 'v__args' -ALT_V_KWARGS = 'v__kwargs' -V_POSITIONAL_ONLY_NAME = 'v__positional_only' -V_DUPLICATE_KWARGS = 'v__duplicate_kwargs' - - -class ValidatedFunction: - def __init__(self, function: 'AnyCallable', config: 'ConfigType'): - from inspect import Parameter, signature - - parameters: Mapping[str, Parameter] = signature(function).parameters - - if parameters.keys() & {ALT_V_ARGS, ALT_V_KWARGS, V_POSITIONAL_ONLY_NAME, V_DUPLICATE_KWARGS}: - raise PydanticUserError( - f'"{ALT_V_ARGS}", "{ALT_V_KWARGS}", "{V_POSITIONAL_ONLY_NAME}" and "{V_DUPLICATE_KWARGS}" ' - f'are not permitted as argument names when using the "{validate_arguments.__name__}" decorator', - code=None, - ) - - self.raw_function = function - self.arg_mapping: dict[int, str] = {} - self.positional_only_args: set[str] = set() - self.v_args_name = 'args' - self.v_kwargs_name = 'kwargs' - - type_hints = _typing_extra.get_type_hints(function, include_extras=True) - takes_args = False - takes_kwargs = False - fields: dict[str, tuple[Any, Any]] = {} - for i, (name, p) in enumerate(parameters.items()): - if p.annotation is p.empty: - annotation = Any - else: - annotation = type_hints[name] - - default = ... if p.default is p.empty else p.default - if p.kind == Parameter.POSITIONAL_ONLY: - self.arg_mapping[i] = name - fields[name] = annotation, default - fields[V_POSITIONAL_ONLY_NAME] = list[str], None - self.positional_only_args.add(name) - elif p.kind == Parameter.POSITIONAL_OR_KEYWORD: - self.arg_mapping[i] = name - fields[name] = annotation, default - fields[V_DUPLICATE_KWARGS] = list[str], None - elif p.kind == Parameter.KEYWORD_ONLY: - fields[name] = annotation, default - elif p.kind == Parameter.VAR_POSITIONAL: - self.v_args_name = name - fields[name] = tuple[annotation, ...], None - takes_args = True - else: - assert p.kind == Parameter.VAR_KEYWORD, p.kind - self.v_kwargs_name = name - fields[name] = dict[str, annotation], None - takes_kwargs = True - - # these checks avoid a clash between "args" and a field with that name - if not takes_args and self.v_args_name in fields: - self.v_args_name = ALT_V_ARGS - - # same with "kwargs" - if not takes_kwargs and self.v_kwargs_name in fields: - self.v_kwargs_name = ALT_V_KWARGS - - if not takes_args: - # we add the field so validation below can raise the correct exception - fields[self.v_args_name] = list[Any], None - - if not takes_kwargs: - # same with kwargs - fields[self.v_kwargs_name] = dict[Any, Any], None - - self.create_model(fields, takes_args, takes_kwargs, config) - - def init_model_instance(self, *args: Any, **kwargs: Any) -> BaseModel: - values = self.build_values(args, kwargs) - return self.model(**values) - - def call(self, *args: Any, **kwargs: Any) -> Any: - m = self.init_model_instance(*args, **kwargs) - return self.execute(m) - - def build_values(self, args: tuple[Any, ...], kwargs: dict[str, Any]) -> dict[str, Any]: - values: dict[str, Any] = {} - if args: - arg_iter = enumerate(args) - while True: - try: - i, a = next(arg_iter) - except StopIteration: - break - arg_name = self.arg_mapping.get(i) - if arg_name is not None: - values[arg_name] = a - else: - values[self.v_args_name] = [a] + [a for _, a in arg_iter] - break - - var_kwargs: dict[str, Any] = {} - wrong_positional_args = [] - duplicate_kwargs = [] - fields_alias = [ - field.alias - for name, field in self.model.__pydantic_fields__.items() - if name not in (self.v_args_name, self.v_kwargs_name) - ] - non_var_fields = set(self.model.__pydantic_fields__) - {self.v_args_name, self.v_kwargs_name} - for k, v in kwargs.items(): - if k in non_var_fields or k in fields_alias: - if k in self.positional_only_args: - wrong_positional_args.append(k) - if k in values: - duplicate_kwargs.append(k) - values[k] = v - else: - var_kwargs[k] = v - - if var_kwargs: - values[self.v_kwargs_name] = var_kwargs - if wrong_positional_args: - values[V_POSITIONAL_ONLY_NAME] = wrong_positional_args - if duplicate_kwargs: - values[V_DUPLICATE_KWARGS] = duplicate_kwargs - return values - - def execute(self, m: BaseModel) -> Any: - d = { - k: v - for k, v in m.__dict__.items() - if k in m.__pydantic_fields_set__ or m.__pydantic_fields__[k].default_factory - } - var_kwargs = d.pop(self.v_kwargs_name, {}) - - if self.v_args_name in d: - args_: list[Any] = [] - in_kwargs = False - kwargs = {} - for name, value in d.items(): - if in_kwargs: - kwargs[name] = value - elif name == self.v_args_name: - args_ += value - in_kwargs = True - else: - args_.append(value) - return self.raw_function(*args_, **kwargs, **var_kwargs) - elif self.positional_only_args: - args_ = [] - kwargs = {} - for name, value in d.items(): - if name in self.positional_only_args: - args_.append(value) - else: - kwargs[name] = value - return self.raw_function(*args_, **kwargs, **var_kwargs) - else: - return self.raw_function(**d, **var_kwargs) - - def create_model(self, fields: dict[str, Any], takes_args: bool, takes_kwargs: bool, config: 'ConfigType') -> None: - pos_args = len(self.arg_mapping) - - config_wrapper = _config.ConfigWrapper(config) - - if config_wrapper.alias_generator: - raise PydanticUserError( - 'Setting the "alias_generator" property on custom Config for ' - '@validate_arguments is not yet supported, please remove.', - code=None, - ) - if config_wrapper.extra is None: - config_wrapper.config_dict['extra'] = 'forbid' - - class DecoratorBaseModel(BaseModel): - @field_validator(self.v_args_name, check_fields=False) - @classmethod - def check_args(cls, v: Optional[list[Any]]) -> Optional[list[Any]]: - if takes_args or v is None: - return v - - raise TypeError(f'{pos_args} positional arguments expected but {pos_args + len(v)} given') - - @field_validator(self.v_kwargs_name, check_fields=False) - @classmethod - def check_kwargs(cls, v: Optional[dict[str, Any]]) -> Optional[dict[str, Any]]: - if takes_kwargs or v is None: - return v - - plural = '' if len(v) == 1 else 's' - keys = ', '.join(map(repr, v.keys())) - raise TypeError(f'unexpected keyword argument{plural}: {keys}') - - @field_validator(V_POSITIONAL_ONLY_NAME, check_fields=False) - @classmethod - def check_positional_only(cls, v: Optional[list[str]]) -> None: - if v is None: - return - - plural = '' if len(v) == 1 else 's' - keys = ', '.join(map(repr, v)) - raise TypeError(f'positional-only argument{plural} passed as keyword argument{plural}: {keys}') - - @field_validator(V_DUPLICATE_KWARGS, check_fields=False) - @classmethod - def check_duplicate_kwargs(cls, v: Optional[list[str]]) -> None: - if v is None: - return - - plural = '' if len(v) == 1 else 's' - keys = ', '.join(map(repr, v)) - raise TypeError(f'multiple values for argument{plural}: {keys}') - - model_config = config_wrapper.config_dict - - self.model = create_model(to_pascal(self.raw_function.__name__), __base__=DecoratorBaseModel, **fields) diff --git a/functions/python/data-processor/pydantic/deprecated/json.py b/functions/python/data-processor/pydantic/deprecated/json.py deleted file mode 100644 index 1e216a765..000000000 --- a/functions/python/data-processor/pydantic/deprecated/json.py +++ /dev/null @@ -1,141 +0,0 @@ -import datetime -import warnings -from collections import deque -from decimal import Decimal -from enum import Enum -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from pathlib import Path -from re import Pattern -from types import GeneratorType -from typing import TYPE_CHECKING, Any, Callable, Union -from uuid import UUID - -from typing_extensions import deprecated - -from .._internal._import_utils import import_cached_base_model -from ..color import Color -from ..networks import NameEmail -from ..types import SecretBytes, SecretStr -from ..warnings import PydanticDeprecatedSince20 - -if not TYPE_CHECKING: - # See PyCharm issues https://youtrack.jetbrains.com/issue/PY-21915 - # and https://youtrack.jetbrains.com/issue/PY-51428 - DeprecationWarning = PydanticDeprecatedSince20 - -__all__ = 'pydantic_encoder', 'custom_pydantic_encoder', 'timedelta_isoformat' - - -def isoformat(o: Union[datetime.date, datetime.time]) -> str: - return o.isoformat() - - -def decimal_encoder(dec_value: Decimal) -> Union[int, float]: - """Encodes a Decimal as int of there's no exponent, otherwise float. - - This is useful when we use ConstrainedDecimal to represent Numeric(x,0) - where a integer (but not int typed) is used. Encoding this as a float - results in failed round-tripping between encode and parse. - Our Id type is a prime example of this. - - >>> decimal_encoder(Decimal("1.0")) - 1.0 - - >>> decimal_encoder(Decimal("1")) - 1 - """ - exponent = dec_value.as_tuple().exponent - if isinstance(exponent, int) and exponent >= 0: - return int(dec_value) - else: - return float(dec_value) - - -ENCODERS_BY_TYPE: dict[type[Any], Callable[[Any], Any]] = { - bytes: lambda o: o.decode(), - Color: str, - datetime.date: isoformat, - datetime.datetime: isoformat, - datetime.time: isoformat, - datetime.timedelta: lambda td: td.total_seconds(), - Decimal: decimal_encoder, - Enum: lambda o: o.value, - frozenset: list, - deque: list, - GeneratorType: list, - IPv4Address: str, - IPv4Interface: str, - IPv4Network: str, - IPv6Address: str, - IPv6Interface: str, - IPv6Network: str, - NameEmail: str, - Path: str, - Pattern: lambda o: o.pattern, - SecretBytes: str, - SecretStr: str, - set: list, - UUID: str, -} - - -@deprecated( - '`pydantic_encoder` is deprecated, use `pydantic_core.to_jsonable_python` instead.', - category=None, -) -def pydantic_encoder(obj: Any) -> Any: - warnings.warn( - '`pydantic_encoder` is deprecated, use `pydantic_core.to_jsonable_python` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from dataclasses import asdict, is_dataclass - - BaseModel = import_cached_base_model() - - if isinstance(obj, BaseModel): - return obj.model_dump() - elif is_dataclass(obj): - return asdict(obj) # type: ignore - - # Check the class type and its superclasses for a matching encoder - for base in obj.__class__.__mro__[:-1]: - try: - encoder = ENCODERS_BY_TYPE[base] - except KeyError: - continue - return encoder(obj) - else: # We have exited the for loop without finding a suitable encoder - raise TypeError(f"Object of type '{obj.__class__.__name__}' is not JSON serializable") - - -# TODO: Add a suggested migration path once there is a way to use custom encoders -@deprecated( - '`custom_pydantic_encoder` is deprecated, use `BaseModel.model_dump` instead.', - category=None, -) -def custom_pydantic_encoder(type_encoders: dict[Any, Callable[[type[Any]], Any]], obj: Any) -> Any: - warnings.warn( - '`custom_pydantic_encoder` is deprecated, use `BaseModel.model_dump` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - # Check the class type and its superclasses for a matching encoder - for base in obj.__class__.__mro__[:-1]: - try: - encoder = type_encoders[base] - except KeyError: - continue - - return encoder(obj) - else: # We have exited the for loop without finding a suitable encoder - return pydantic_encoder(obj) - - -@deprecated('`timedelta_isoformat` is deprecated.', category=None) -def timedelta_isoformat(td: datetime.timedelta) -> str: - """ISO 8601 encoding for Python timedelta object.""" - warnings.warn('`timedelta_isoformat` is deprecated.', category=PydanticDeprecatedSince20, stacklevel=2) - minutes, seconds = divmod(td.seconds, 60) - hours, minutes = divmod(minutes, 60) - return f'{"-" if td.days < 0 else ""}P{abs(td.days)}DT{hours:d}H{minutes:d}M{seconds:d}.{td.microseconds:06d}S' diff --git a/functions/python/data-processor/pydantic/deprecated/parse.py b/functions/python/data-processor/pydantic/deprecated/parse.py deleted file mode 100644 index 2a92e62b7..000000000 --- a/functions/python/data-processor/pydantic/deprecated/parse.py +++ /dev/null @@ -1,80 +0,0 @@ -from __future__ import annotations - -import json -import pickle -import warnings -from enum import Enum -from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable - -from typing_extensions import deprecated - -from ..warnings import PydanticDeprecatedSince20 - -if not TYPE_CHECKING: - # See PyCharm issues https://youtrack.jetbrains.com/issue/PY-21915 - # and https://youtrack.jetbrains.com/issue/PY-51428 - DeprecationWarning = PydanticDeprecatedSince20 - - -class Protocol(str, Enum): - json = 'json' - pickle = 'pickle' - - -@deprecated('`load_str_bytes` is deprecated.', category=None) -def load_str_bytes( - b: str | bytes, - *, - content_type: str | None = None, - encoding: str = 'utf8', - proto: Protocol | None = None, - allow_pickle: bool = False, - json_loads: Callable[[str], Any] = json.loads, -) -> Any: - warnings.warn('`load_str_bytes` is deprecated.', category=PydanticDeprecatedSince20, stacklevel=2) - if proto is None and content_type: - if content_type.endswith(('json', 'javascript')): - pass - elif allow_pickle and content_type.endswith('pickle'): - proto = Protocol.pickle - else: - raise TypeError(f'Unknown content-type: {content_type}') - - proto = proto or Protocol.json - - if proto == Protocol.json: - if isinstance(b, bytes): - b = b.decode(encoding) - return json_loads(b) # type: ignore - elif proto == Protocol.pickle: - if not allow_pickle: - raise RuntimeError('Trying to decode with pickle with allow_pickle=False') - bb = b if isinstance(b, bytes) else b.encode() # type: ignore - return pickle.loads(bb) - else: - raise TypeError(f'Unknown protocol: {proto}') - - -@deprecated('`load_file` is deprecated.', category=None) -def load_file( - path: str | Path, - *, - content_type: str | None = None, - encoding: str = 'utf8', - proto: Protocol | None = None, - allow_pickle: bool = False, - json_loads: Callable[[str], Any] = json.loads, -) -> Any: - warnings.warn('`load_file` is deprecated.', category=PydanticDeprecatedSince20, stacklevel=2) - path = Path(path) - b = path.read_bytes() - if content_type is None: - if path.suffix in ('.js', '.json'): - proto = Protocol.json - elif path.suffix == '.pkl': - proto = Protocol.pickle - - return load_str_bytes( - b, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle, json_loads=json_loads - ) diff --git a/functions/python/data-processor/pydantic/deprecated/tools.py b/functions/python/data-processor/pydantic/deprecated/tools.py deleted file mode 100644 index 5ad7faef2..000000000 --- a/functions/python/data-processor/pydantic/deprecated/tools.py +++ /dev/null @@ -1,103 +0,0 @@ -from __future__ import annotations - -import json -import warnings -from typing import TYPE_CHECKING, Any, Callable, TypeVar, Union - -from typing_extensions import deprecated - -from ..json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema -from ..type_adapter import TypeAdapter -from ..warnings import PydanticDeprecatedSince20 - -if not TYPE_CHECKING: - # See PyCharm issues https://youtrack.jetbrains.com/issue/PY-21915 - # and https://youtrack.jetbrains.com/issue/PY-51428 - DeprecationWarning = PydanticDeprecatedSince20 - -__all__ = 'parse_obj_as', 'schema_of', 'schema_json_of' - -NameFactory = Union[str, Callable[[type[Any]], str]] - - -T = TypeVar('T') - - -@deprecated( - '`parse_obj_as` is deprecated. Use `pydantic.TypeAdapter.validate_python` instead.', - category=None, -) -def parse_obj_as(type_: type[T], obj: Any, type_name: NameFactory | None = None) -> T: - warnings.warn( - '`parse_obj_as` is deprecated. Use `pydantic.TypeAdapter.validate_python` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - if type_name is not None: # pragma: no cover - warnings.warn( - 'The type_name parameter is deprecated. parse_obj_as no longer creates temporary models', - DeprecationWarning, - stacklevel=2, - ) - return TypeAdapter(type_).validate_python(obj) - - -@deprecated( - '`schema_of` is deprecated. Use `pydantic.TypeAdapter.json_schema` instead.', - category=None, -) -def schema_of( - type_: Any, - *, - title: NameFactory | None = None, - by_alias: bool = True, - ref_template: str = DEFAULT_REF_TEMPLATE, - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, -) -> dict[str, Any]: - """Generate a JSON schema (as dict) for the passed model or dynamically generated one.""" - warnings.warn( - '`schema_of` is deprecated. Use `pydantic.TypeAdapter.json_schema` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - res = TypeAdapter(type_).json_schema( - by_alias=by_alias, - schema_generator=schema_generator, - ref_template=ref_template, - ) - if title is not None: - if isinstance(title, str): - res['title'] = title - else: - warnings.warn( - 'Passing a callable for the `title` parameter is deprecated and no longer supported', - DeprecationWarning, - stacklevel=2, - ) - res['title'] = title(type_) - return res - - -@deprecated( - '`schema_json_of` is deprecated. Use `pydantic.TypeAdapter.json_schema` instead.', - category=None, -) -def schema_json_of( - type_: Any, - *, - title: NameFactory | None = None, - by_alias: bool = True, - ref_template: str = DEFAULT_REF_TEMPLATE, - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, - **dumps_kwargs: Any, -) -> str: - """Generate a JSON schema (as JSON) for the passed model or dynamically generated one.""" - warnings.warn( - '`schema_json_of` is deprecated. Use `pydantic.TypeAdapter.json_schema` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return json.dumps( - schema_of(type_, title=title, by_alias=by_alias, ref_template=ref_template, schema_generator=schema_generator), - **dumps_kwargs, - ) diff --git a/functions/python/data-processor/pydantic/env_settings.py b/functions/python/data-processor/pydantic/env_settings.py deleted file mode 100644 index cd0b04e6a..000000000 --- a/functions/python/data-processor/pydantic/env_settings.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `env_settings` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/error_wrappers.py b/functions/python/data-processor/pydantic/error_wrappers.py deleted file mode 100644 index 2985419ab..000000000 --- a/functions/python/data-processor/pydantic/error_wrappers.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `error_wrappers` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/errors.py b/functions/python/data-processor/pydantic/errors.py deleted file mode 100644 index 33851bded..000000000 --- a/functions/python/data-processor/pydantic/errors.py +++ /dev/null @@ -1,189 +0,0 @@ -"""Pydantic-specific errors.""" - -from __future__ import annotations as _annotations - -import re -from typing import Any, ClassVar, Literal - -from typing_extensions import Self -from typing_inspection.introspection import Qualifier - -from pydantic._internal import _repr - -from ._migration import getattr_migration -from .version import version_short - -__all__ = ( - 'PydanticUserError', - 'PydanticUndefinedAnnotation', - 'PydanticImportError', - 'PydanticSchemaGenerationError', - 'PydanticInvalidForJsonSchema', - 'PydanticForbiddenQualifier', - 'PydanticErrorCodes', -) - -# We use this URL to allow for future flexibility about how we host the docs, while allowing for Pydantic -# code in the while with "old" URLs to still work. -# 'u' refers to "user errors" - e.g. errors caused by developers using pydantic, as opposed to validation errors. -DEV_ERROR_DOCS_URL = f'https://errors.pydantic.dev/{version_short()}/u/' -PydanticErrorCodes = Literal[ - 'class-not-fully-defined', - 'custom-json-schema', - 'decorator-invalid-fields', - 'decorator-missing-arguments', - 'decorator-missing-field', - 'discriminator-no-field', - 'discriminator-alias-type', - 'discriminator-needs-literal', - 'discriminator-alias', - 'discriminator-validator', - 'callable-discriminator-no-tag', - 'typed-dict-version', - 'model-field-overridden', - 'model-field-missing-annotation', - 'config-both', - 'removed-kwargs', - 'circular-reference-schema', - 'invalid-for-json-schema', - 'json-schema-already-used', - 'base-model-instantiated', - 'undefined-annotation', - 'schema-for-unknown-type', - 'import-error', - 'create-model-field-definitions', - 'validator-instance-method', - 'validator-input-type', - 'root-validator-pre-skip', - 'model-serializer-instance-method', - 'validator-field-config-info', - 'validator-v1-signature', - 'validator-signature', - 'field-serializer-signature', - 'model-serializer-signature', - 'multiple-field-serializers', - 'invalid-annotated-type', - 'type-adapter-config-unused', - 'root-model-extra', - 'unevaluable-type-annotation', - 'dataclass-init-false-extra-allow', - 'clashing-init-and-init-var', - 'model-config-invalid-field-name', - 'with-config-on-model', - 'dataclass-on-model', - 'validate-call-type', - 'unpack-typed-dict', - 'overlapping-unpack-typed-dict', - 'invalid-self-type', - 'validate-by-alias-and-name-false', -] - - -class PydanticErrorMixin: - """A mixin class for common functionality shared by all Pydantic-specific errors. - - Attributes: - message: A message describing the error. - code: An optional error code from PydanticErrorCodes enum. - """ - - def __init__(self, message: str, *, code: PydanticErrorCodes | None) -> None: - self.message = message - self.code = code - - def __str__(self) -> str: - if self.code is None: - return self.message - else: - return f'{self.message}\n\nFor further information visit {DEV_ERROR_DOCS_URL}{self.code}' - - -class PydanticUserError(PydanticErrorMixin, RuntimeError): - """An error raised due to incorrect use of Pydantic.""" - - -class PydanticUndefinedAnnotation(PydanticErrorMixin, NameError): - """A subclass of `NameError` raised when handling undefined annotations during `CoreSchema` generation. - - Attributes: - name: Name of the error. - message: Description of the error. - """ - - def __init__(self, name: str, message: str) -> None: - self.name = name - super().__init__(message=message, code='undefined-annotation') - - @classmethod - def from_name_error(cls, name_error: NameError) -> Self: - """Convert a `NameError` to a `PydanticUndefinedAnnotation` error. - - Args: - name_error: `NameError` to be converted. - - Returns: - Converted `PydanticUndefinedAnnotation` error. - """ - try: - name = name_error.name # type: ignore # python > 3.10 - except AttributeError: - name = re.search(r".*'(.+?)'", str(name_error)).group(1) # type: ignore[union-attr] - return cls(name=name, message=str(name_error)) - - -class PydanticImportError(PydanticErrorMixin, ImportError): - """An error raised when an import fails due to module changes between V1 and V2. - - Attributes: - message: Description of the error. - """ - - def __init__(self, message: str) -> None: - super().__init__(message, code='import-error') - - -class PydanticSchemaGenerationError(PydanticUserError): - """An error raised during failures to generate a `CoreSchema` for some type. - - Attributes: - message: Description of the error. - """ - - def __init__(self, message: str) -> None: - super().__init__(message, code='schema-for-unknown-type') - - -class PydanticInvalidForJsonSchema(PydanticUserError): - """An error raised during failures to generate a JSON schema for some `CoreSchema`. - - Attributes: - message: Description of the error. - """ - - def __init__(self, message: str) -> None: - super().__init__(message, code='invalid-for-json-schema') - - -class PydanticForbiddenQualifier(PydanticUserError): - """An error raised if a forbidden type qualifier is found in a type annotation.""" - - _qualifier_repr_map: ClassVar[dict[Qualifier, str]] = { - 'required': 'typing.Required', - 'not_required': 'typing.NotRequired', - 'read_only': 'typing.ReadOnly', - 'class_var': 'typing.ClassVar', - 'init_var': 'dataclasses.InitVar', - 'final': 'typing.Final', - } - - def __init__(self, qualifier: Qualifier, annotation: Any) -> None: - super().__init__( - message=( - f'The annotation {_repr.display_as_type(annotation)!r} contains the {self._qualifier_repr_map[qualifier]!r} ' - f'type qualifier, which is invalid in the context it is defined.' - ), - code=None, - ) - - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/experimental/__init__.py b/functions/python/data-processor/pydantic/experimental/__init__.py deleted file mode 100644 index 5b5add109..000000000 --- a/functions/python/data-processor/pydantic/experimental/__init__.py +++ /dev/null @@ -1 +0,0 @@ -"""The "experimental" module of pydantic contains potential new features that are subject to change.""" diff --git a/functions/python/data-processor/pydantic/experimental/arguments_schema.py b/functions/python/data-processor/pydantic/experimental/arguments_schema.py deleted file mode 100644 index af4a8f3be..000000000 --- a/functions/python/data-processor/pydantic/experimental/arguments_schema.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Experimental module exposing a function to generate a core schema that validates callable arguments.""" - -from __future__ import annotations - -from collections.abc import Callable -from typing import Any, Literal - -from pydantic_core import CoreSchema - -from pydantic import ConfigDict -from pydantic._internal import _config, _generate_schema, _namespace_utils - - -def generate_arguments_schema( - func: Callable[..., Any], - schema_type: Literal['arguments', 'arguments-v3'] = 'arguments-v3', - parameters_callback: Callable[[int, str, Any], Literal['skip'] | None] | None = None, - config: ConfigDict | None = None, -) -> CoreSchema: - """Generate the schema for the arguments of a function. - - Args: - func: The function to generate the schema for. - schema_type: The type of schema to generate. - parameters_callback: A callable that will be invoked for each parameter. The callback - should take three required arguments: the index, the name and the type annotation - (or [`Parameter.empty`][inspect.Parameter.empty] if not annotated) of the parameter. - The callback can optionally return `'skip'`, so that the parameter gets excluded - from the resulting schema. - config: The configuration to use. - - Returns: - The generated schema. - """ - generate_schema = _generate_schema.GenerateSchema( - _config.ConfigWrapper(config), - ns_resolver=_namespace_utils.NsResolver(namespaces_tuple=_namespace_utils.ns_for_function(func)), - ) - - if schema_type == 'arguments': - schema = generate_schema._arguments_schema(func, parameters_callback) # pyright: ignore[reportArgumentType] - else: - schema = generate_schema._arguments_v3_schema(func, parameters_callback) # pyright: ignore[reportArgumentType] - return generate_schema.clean_schema(schema) diff --git a/functions/python/data-processor/pydantic/experimental/missing_sentinel.py b/functions/python/data-processor/pydantic/experimental/missing_sentinel.py deleted file mode 100644 index 3e7f820ce..000000000 --- a/functions/python/data-processor/pydantic/experimental/missing_sentinel.py +++ /dev/null @@ -1,5 +0,0 @@ -"""Experimental module exposing a function a `MISSING` sentinel.""" - -from pydantic_core import MISSING - -__all__ = ('MISSING',) diff --git a/functions/python/data-processor/pydantic/experimental/pipeline.py b/functions/python/data-processor/pydantic/experimental/pipeline.py deleted file mode 100644 index 9ead658f0..000000000 --- a/functions/python/data-processor/pydantic/experimental/pipeline.py +++ /dev/null @@ -1,663 +0,0 @@ -"""Experimental pipeline API functionality. Be careful with this API, it's subject to change.""" - -from __future__ import annotations - -import datetime -import operator -import re -import sys -from collections import deque -from collections.abc import Container -from dataclasses import dataclass -from decimal import Decimal -from functools import cached_property, partial -from re import Pattern -from typing import TYPE_CHECKING, Annotated, Any, Callable, Generic, Protocol, TypeVar, Union, overload - -import annotated_types - -if TYPE_CHECKING: - from pydantic import GetCoreSchemaHandler - -from pydantic_core import PydanticCustomError -from pydantic_core import core_schema as cs - -from pydantic import Strict -from pydantic._internal._internal_dataclass import slots_true as _slots_true - -if sys.version_info < (3, 10): - EllipsisType = type(Ellipsis) -else: - from types import EllipsisType - -__all__ = ['validate_as', 'validate_as_deferred', 'transform'] - -_slots_frozen = {**_slots_true, 'frozen': True} - - -@dataclass(**_slots_frozen) -class _ValidateAs: - tp: type[Any] - strict: bool = False - - -@dataclass -class _ValidateAsDefer: - func: Callable[[], type[Any]] - - @cached_property - def tp(self) -> type[Any]: - return self.func() - - -@dataclass(**_slots_frozen) -class _Transform: - func: Callable[[Any], Any] - - -@dataclass(**_slots_frozen) -class _PipelineOr: - left: _Pipeline[Any, Any] - right: _Pipeline[Any, Any] - - -@dataclass(**_slots_frozen) -class _PipelineAnd: - left: _Pipeline[Any, Any] - right: _Pipeline[Any, Any] - - -@dataclass(**_slots_frozen) -class _Eq: - value: Any - - -@dataclass(**_slots_frozen) -class _NotEq: - value: Any - - -@dataclass(**_slots_frozen) -class _In: - values: Container[Any] - - -@dataclass(**_slots_frozen) -class _NotIn: - values: Container[Any] - - -_ConstraintAnnotation = Union[ - annotated_types.Le, - annotated_types.Ge, - annotated_types.Lt, - annotated_types.Gt, - annotated_types.Len, - annotated_types.MultipleOf, - annotated_types.Timezone, - annotated_types.Interval, - annotated_types.Predicate, - # common predicates not included in annotated_types - _Eq, - _NotEq, - _In, - _NotIn, - # regular expressions - Pattern[str], -] - - -@dataclass(**_slots_frozen) -class _Constraint: - constraint: _ConstraintAnnotation - - -_Step = Union[_ValidateAs, _ValidateAsDefer, _Transform, _PipelineOr, _PipelineAnd, _Constraint] - -_InT = TypeVar('_InT') -_OutT = TypeVar('_OutT') -_NewOutT = TypeVar('_NewOutT') - - -class _FieldTypeMarker: - pass - - -# TODO: ultimately, make this public, see https://github.com/pydantic/pydantic/pull/9459#discussion_r1628197626 -# Also, make this frozen eventually, but that doesn't work right now because of the generic base -# Which attempts to modify __orig_base__ and such. -# We could go with a manual freeze, but that seems overkill for now. -@dataclass(**_slots_true) -class _Pipeline(Generic[_InT, _OutT]): - """Abstract representation of a chain of validation, transformation, and parsing steps.""" - - _steps: tuple[_Step, ...] - - def transform( - self, - func: Callable[[_OutT], _NewOutT], - ) -> _Pipeline[_InT, _NewOutT]: - """Transform the output of the previous step. - - If used as the first step in a pipeline, the type of the field is used. - That is, the transformation is applied to after the value is parsed to the field's type. - """ - return _Pipeline[_InT, _NewOutT](self._steps + (_Transform(func),)) - - @overload - def validate_as(self, tp: type[_NewOutT], *, strict: bool = False) -> _Pipeline[_InT, _NewOutT]: ... - - @overload - def validate_as( - self, - tp: ellipsis, # noqa: F821 # TODO: use `_typing_extra.EllipsisType` when we drop Py3.9 - *, - strict: bool = False, - ) -> _Pipeline[_InT, Any]: ... - - # TODO PEP 747: use TypeForm to properly type Annotated aliases (e.g. NewPath, FilePath). - # This fallback accepts any type expression but loses generic type inference. - @overload - def validate_as(self, tp: Any, *, strict: bool = ...) -> _Pipeline[_InT, Any]: ... - - def validate_as(self, tp: type[_NewOutT] | EllipsisType | Any, *, strict: bool = False) -> _Pipeline[_InT, Any]: # type: ignore - """Validate / parse the input into a new type. - - If no type is provided, the type of the field is used. - - Types are parsed in Pydantic's `lax` mode by default, - but you can enable `strict` mode by passing `strict=True`. - """ - if isinstance(tp, EllipsisType): - return _Pipeline[_InT, Any](self._steps + (_ValidateAs(_FieldTypeMarker, strict=strict),)) - return _Pipeline[_InT, _NewOutT](self._steps + (_ValidateAs(tp, strict=strict),)) - - def validate_as_deferred(self, func: Callable[[], type[_NewOutT]]) -> _Pipeline[_InT, _NewOutT]: - """Parse the input into a new type, deferring resolution of the type until the current class - is fully defined. - - This is useful when you need to reference the class in it's own type annotations. - """ - return _Pipeline[_InT, _NewOutT](self._steps + (_ValidateAsDefer(func),)) - - # constraints - @overload - def constrain(self: _Pipeline[_InT, _NewOutGe], constraint: annotated_types.Ge) -> _Pipeline[_InT, _NewOutGe]: ... - - @overload - def constrain(self: _Pipeline[_InT, _NewOutGt], constraint: annotated_types.Gt) -> _Pipeline[_InT, _NewOutGt]: ... - - @overload - def constrain(self: _Pipeline[_InT, _NewOutLe], constraint: annotated_types.Le) -> _Pipeline[_InT, _NewOutLe]: ... - - @overload - def constrain(self: _Pipeline[_InT, _NewOutLt], constraint: annotated_types.Lt) -> _Pipeline[_InT, _NewOutLt]: ... - - @overload - def constrain( - self: _Pipeline[_InT, _NewOutLen], constraint: annotated_types.Len - ) -> _Pipeline[_InT, _NewOutLen]: ... - - @overload - def constrain( - self: _Pipeline[_InT, _NewOutT], constraint: annotated_types.MultipleOf - ) -> _Pipeline[_InT, _NewOutT]: ... - - @overload - def constrain( - self: _Pipeline[_InT, _NewOutDatetime], constraint: annotated_types.Timezone - ) -> _Pipeline[_InT, _NewOutDatetime]: ... - - @overload - def constrain(self: _Pipeline[_InT, _OutT], constraint: annotated_types.Predicate) -> _Pipeline[_InT, _OutT]: ... - - @overload - def constrain( - self: _Pipeline[_InT, _NewOutInterval], constraint: annotated_types.Interval - ) -> _Pipeline[_InT, _NewOutInterval]: ... - - @overload - def constrain(self: _Pipeline[_InT, _OutT], constraint: _Eq) -> _Pipeline[_InT, _OutT]: ... - - @overload - def constrain(self: _Pipeline[_InT, _OutT], constraint: _NotEq) -> _Pipeline[_InT, _OutT]: ... - - @overload - def constrain(self: _Pipeline[_InT, _OutT], constraint: _In) -> _Pipeline[_InT, _OutT]: ... - - @overload - def constrain(self: _Pipeline[_InT, _OutT], constraint: _NotIn) -> _Pipeline[_InT, _OutT]: ... - - @overload - def constrain(self: _Pipeline[_InT, _NewOutT], constraint: Pattern[str]) -> _Pipeline[_InT, _NewOutT]: ... - - def constrain(self, constraint: _ConstraintAnnotation) -> Any: - """Constrain a value to meet a certain condition. - - We support most conditions from `annotated_types`, as well as regular expressions. - - Most of the time you'll be calling a shortcut method like `gt`, `lt`, `len`, etc - so you don't need to call this directly. - """ - return _Pipeline[_InT, _OutT](self._steps + (_Constraint(constraint),)) - - def predicate(self: _Pipeline[_InT, _NewOutT], func: Callable[[_NewOutT], bool]) -> _Pipeline[_InT, _NewOutT]: - """Constrain a value to meet a certain predicate.""" - return self.constrain(annotated_types.Predicate(func)) - - def gt(self: _Pipeline[_InT, _NewOutGt], gt: _NewOutGt) -> _Pipeline[_InT, _NewOutGt]: - """Constrain a value to be greater than a certain value.""" - return self.constrain(annotated_types.Gt(gt)) - - def lt(self: _Pipeline[_InT, _NewOutLt], lt: _NewOutLt) -> _Pipeline[_InT, _NewOutLt]: - """Constrain a value to be less than a certain value.""" - return self.constrain(annotated_types.Lt(lt)) - - def ge(self: _Pipeline[_InT, _NewOutGe], ge: _NewOutGe) -> _Pipeline[_InT, _NewOutGe]: - """Constrain a value to be greater than or equal to a certain value.""" - return self.constrain(annotated_types.Ge(ge)) - - def le(self: _Pipeline[_InT, _NewOutLe], le: _NewOutLe) -> _Pipeline[_InT, _NewOutLe]: - """Constrain a value to be less than or equal to a certain value.""" - return self.constrain(annotated_types.Le(le)) - - def len(self: _Pipeline[_InT, _NewOutLen], min_len: int, max_len: int | None = None) -> _Pipeline[_InT, _NewOutLen]: - """Constrain a value to have a certain length.""" - return self.constrain(annotated_types.Len(min_len, max_len)) - - @overload - def multiple_of(self: _Pipeline[_InT, _NewOutDiv], multiple_of: _NewOutDiv) -> _Pipeline[_InT, _NewOutDiv]: ... - - @overload - def multiple_of(self: _Pipeline[_InT, _NewOutMod], multiple_of: _NewOutMod) -> _Pipeline[_InT, _NewOutMod]: ... - - def multiple_of(self: _Pipeline[_InT, Any], multiple_of: Any) -> _Pipeline[_InT, Any]: - """Constrain a value to be a multiple of a certain number.""" - return self.constrain(annotated_types.MultipleOf(multiple_of)) - - def eq(self: _Pipeline[_InT, _OutT], value: _OutT) -> _Pipeline[_InT, _OutT]: - """Constrain a value to be equal to a certain value.""" - return self.constrain(_Eq(value)) - - def not_eq(self: _Pipeline[_InT, _OutT], value: _OutT) -> _Pipeline[_InT, _OutT]: - """Constrain a value to not be equal to a certain value.""" - return self.constrain(_NotEq(value)) - - def in_(self: _Pipeline[_InT, _OutT], values: Container[_OutT]) -> _Pipeline[_InT, _OutT]: - """Constrain a value to be in a certain set.""" - return self.constrain(_In(values)) - - def not_in(self: _Pipeline[_InT, _OutT], values: Container[_OutT]) -> _Pipeline[_InT, _OutT]: - """Constrain a value to not be in a certain set.""" - return self.constrain(_NotIn(values)) - - # timezone methods - def datetime_tz_naive(self: _Pipeline[_InT, datetime.datetime]) -> _Pipeline[_InT, datetime.datetime]: - return self.constrain(annotated_types.Timezone(None)) - - def datetime_tz_aware(self: _Pipeline[_InT, datetime.datetime]) -> _Pipeline[_InT, datetime.datetime]: - return self.constrain(annotated_types.Timezone(...)) - - def datetime_tz( - self: _Pipeline[_InT, datetime.datetime], tz: datetime.tzinfo - ) -> _Pipeline[_InT, datetime.datetime]: - return self.constrain(annotated_types.Timezone(tz)) # type: ignore - - def datetime_with_tz( - self: _Pipeline[_InT, datetime.datetime], tz: datetime.tzinfo | None - ) -> _Pipeline[_InT, datetime.datetime]: - return self.transform(partial(datetime.datetime.replace, tzinfo=tz)) - - # string methods - def str_lower(self: _Pipeline[_InT, str]) -> _Pipeline[_InT, str]: - return self.transform(str.lower) - - def str_upper(self: _Pipeline[_InT, str]) -> _Pipeline[_InT, str]: - return self.transform(str.upper) - - def str_title(self: _Pipeline[_InT, str]) -> _Pipeline[_InT, str]: - return self.transform(str.title) - - def str_strip(self: _Pipeline[_InT, str]) -> _Pipeline[_InT, str]: - return self.transform(str.strip) - - def str_pattern(self: _Pipeline[_InT, str], pattern: str) -> _Pipeline[_InT, str]: - return self.constrain(re.compile(pattern)) - - def str_contains(self: _Pipeline[_InT, str], substring: str) -> _Pipeline[_InT, str]: - return self.predicate(lambda v: substring in v) - - def str_starts_with(self: _Pipeline[_InT, str], prefix: str) -> _Pipeline[_InT, str]: - return self.predicate(lambda v: v.startswith(prefix)) - - def str_ends_with(self: _Pipeline[_InT, str], suffix: str) -> _Pipeline[_InT, str]: - return self.predicate(lambda v: v.endswith(suffix)) - - # operators - def otherwise(self, other: _Pipeline[_OtherIn, _OtherOut]) -> _Pipeline[_InT | _OtherIn, _OutT | _OtherOut]: - """Combine two validation chains, returning the result of the first chain if it succeeds, and the second chain if it fails.""" - return _Pipeline((_PipelineOr(self, other),)) - - __or__ = otherwise - - def then(self, other: _Pipeline[_OutT, _OtherOut]) -> _Pipeline[_InT, _OtherOut]: - """Pipe the result of one validation chain into another.""" - return _Pipeline((_PipelineAnd(self, other),)) - - __and__ = then - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> cs.CoreSchema: - queue = deque(self._steps) - - s = None - - while queue: - step = queue.popleft() - s = _apply_step(step, s, handler, source_type) - - s = s or cs.any_schema() - return s - - def __supports_type__(self, _: _OutT) -> bool: - raise NotImplementedError - - -validate_as = _Pipeline[Any, Any](()).validate_as -validate_as_deferred = _Pipeline[Any, Any](()).validate_as_deferred -transform = _Pipeline[Any, Any]((_ValidateAs(_FieldTypeMarker),)).transform - - -def _check_func( - func: Callable[[Any], bool], predicate_err: str | Callable[[], str], s: cs.CoreSchema | None -) -> cs.CoreSchema: - def handler(v: Any) -> Any: - if func(v): - return v - raise ValueError(f'Expected {predicate_err if isinstance(predicate_err, str) else predicate_err()}') - - if s is None: - return cs.no_info_plain_validator_function(handler) - else: - return cs.no_info_after_validator_function(handler, s) - - -def _apply_step(step: _Step, s: cs.CoreSchema | None, handler: GetCoreSchemaHandler, source_type: Any) -> cs.CoreSchema: - if isinstance(step, _ValidateAs): - s = _apply_parse(s, step.tp, step.strict, handler, source_type) - elif isinstance(step, _ValidateAsDefer): - s = _apply_parse(s, step.tp, False, handler, source_type) - elif isinstance(step, _Transform): - s = _apply_transform(s, step.func, handler) - elif isinstance(step, _Constraint): - s = _apply_constraint(s, step.constraint) - elif isinstance(step, _PipelineOr): - s = cs.union_schema([handler(step.left), handler(step.right)]) - else: - assert isinstance(step, _PipelineAnd) - s = cs.chain_schema([handler(step.left), handler(step.right)]) - return s - - -def _apply_parse( - s: cs.CoreSchema | None, - tp: type[Any], - strict: bool, - handler: GetCoreSchemaHandler, - source_type: Any, -) -> cs.CoreSchema: - if tp is _FieldTypeMarker: - return cs.chain_schema([s, handler(source_type)]) if s else handler(source_type) - - if strict: - tp = Annotated[tp, Strict()] # type: ignore - - if s and s['type'] == 'any': - return handler(tp) - else: - return cs.chain_schema([s, handler(tp)]) if s else handler(tp) - - -def _apply_transform( - s: cs.CoreSchema | None, func: Callable[[Any], Any], handler: GetCoreSchemaHandler -) -> cs.CoreSchema: - if s is None: - return cs.no_info_plain_validator_function(func) - - if s['type'] == 'str': - if func is str.strip: - s = s.copy() - s['strip_whitespace'] = True - return s - elif func is str.lower: - s = s.copy() - s['to_lower'] = True - return s - elif func is str.upper: - s = s.copy() - s['to_upper'] = True - return s - - return cs.no_info_after_validator_function(func, s) - - -def _apply_constraint( # noqa: C901 - s: cs.CoreSchema | None, constraint: _ConstraintAnnotation -) -> cs.CoreSchema: - """Apply a single constraint to a schema.""" - if isinstance(constraint, annotated_types.Gt): - gt = constraint.gt - if s and s['type'] in {'int', 'float', 'decimal'}: - s = s.copy() - if s['type'] == 'int' and isinstance(gt, int): - s['gt'] = gt - elif s['type'] == 'float' and isinstance(gt, float): - s['gt'] = gt - elif s['type'] == 'decimal' and isinstance(gt, Decimal): - s['gt'] = gt - else: - - def check_gt(v: Any) -> bool: - return v > gt - - s = _check_func(check_gt, f'> {gt}', s) - elif isinstance(constraint, annotated_types.Ge): - ge = constraint.ge - if s and s['type'] in {'int', 'float', 'decimal'}: - s = s.copy() - if s['type'] == 'int' and isinstance(ge, int): - s['ge'] = ge - elif s['type'] == 'float' and isinstance(ge, float): - s['ge'] = ge - elif s['type'] == 'decimal' and isinstance(ge, Decimal): - s['ge'] = ge - - def check_ge(v: Any) -> bool: - return v >= ge - - s = _check_func(check_ge, f'>= {ge}', s) - elif isinstance(constraint, annotated_types.Lt): - lt = constraint.lt - if s and s['type'] in {'int', 'float', 'decimal'}: - s = s.copy() - if s['type'] == 'int' and isinstance(lt, int): - s['lt'] = lt - elif s['type'] == 'float' and isinstance(lt, float): - s['lt'] = lt - elif s['type'] == 'decimal' and isinstance(lt, Decimal): - s['lt'] = lt - - def check_lt(v: Any) -> bool: - return v < lt - - s = _check_func(check_lt, f'< {lt}', s) - elif isinstance(constraint, annotated_types.Le): - le = constraint.le - if s and s['type'] in {'int', 'float', 'decimal'}: - s = s.copy() - if s['type'] == 'int' and isinstance(le, int): - s['le'] = le - elif s['type'] == 'float' and isinstance(le, float): - s['le'] = le - elif s['type'] == 'decimal' and isinstance(le, Decimal): - s['le'] = le - - def check_le(v: Any) -> bool: - return v <= le - - s = _check_func(check_le, f'<= {le}', s) - elif isinstance(constraint, annotated_types.Len): - min_len = constraint.min_length - max_len = constraint.max_length - - if s and s['type'] in {'str', 'list', 'tuple', 'set', 'frozenset', 'dict'}: - assert ( - s['type'] == 'str' - or s['type'] == 'list' - or s['type'] == 'tuple' - or s['type'] == 'set' - or s['type'] == 'dict' - or s['type'] == 'frozenset' - ) - s = s.copy() - if min_len != 0: - s['min_length'] = min_len - if max_len is not None: - s['max_length'] = max_len - - def check_len(v: Any) -> bool: - if max_len is not None: - return (min_len <= len(v)) and (len(v) <= max_len) - return min_len <= len(v) - - s = _check_func(check_len, f'length >= {min_len} and length <= {max_len}', s) - elif isinstance(constraint, annotated_types.MultipleOf): - multiple_of = constraint.multiple_of - if s and s['type'] in {'int', 'float', 'decimal'}: - s = s.copy() - if s['type'] == 'int' and isinstance(multiple_of, int): - s['multiple_of'] = multiple_of - elif s['type'] == 'float' and isinstance(multiple_of, float): - s['multiple_of'] = multiple_of - elif s['type'] == 'decimal' and isinstance(multiple_of, Decimal): - s['multiple_of'] = multiple_of - - def check_multiple_of(v: Any) -> bool: - return v % multiple_of == 0 - - s = _check_func(check_multiple_of, f'% {multiple_of} == 0', s) - elif isinstance(constraint, annotated_types.Timezone): - tz = constraint.tz - - if tz is ...: - if s and s['type'] == 'datetime': - s = s.copy() - s['tz_constraint'] = 'aware' - else: - - def check_tz_aware(v: object) -> bool: - assert isinstance(v, datetime.datetime) - return v.tzinfo is not None - - s = _check_func(check_tz_aware, 'timezone aware', s) - elif tz is None: - if s and s['type'] == 'datetime': - s = s.copy() - s['tz_constraint'] = 'naive' - else: - - def check_tz_naive(v: object) -> bool: - assert isinstance(v, datetime.datetime) - return v.tzinfo is None - - s = _check_func(check_tz_naive, 'timezone naive', s) - else: - raise NotImplementedError('Constraining to a specific timezone is not yet supported') - elif isinstance(constraint, annotated_types.Interval): - if constraint.ge: - s = _apply_constraint(s, annotated_types.Ge(constraint.ge)) - if constraint.gt: - s = _apply_constraint(s, annotated_types.Gt(constraint.gt)) - if constraint.le: - s = _apply_constraint(s, annotated_types.Le(constraint.le)) - if constraint.lt: - s = _apply_constraint(s, annotated_types.Lt(constraint.lt)) - assert s is not None - elif isinstance(constraint, annotated_types.Predicate): - func = constraint.func - # Same logic as in `_known_annotated_metadata.apply_known_metadata()`: - predicate_name = f'{func.__qualname__!r} ' if hasattr(func, '__qualname__') else '' - - def predicate_func(v: Any) -> Any: - if not func(v): - raise PydanticCustomError( - 'predicate_failed', - f'Predicate {predicate_name}failed', # pyright: ignore[reportArgumentType] - ) - return v - - if s is None: - s = cs.no_info_plain_validator_function(predicate_func) - else: - s = cs.no_info_after_validator_function(predicate_func, s) - elif isinstance(constraint, _NotEq): - value = constraint.value - - def check_not_eq(v: Any) -> bool: - return operator.__ne__(v, value) - - s = _check_func(check_not_eq, f'!= {value}', s) - elif isinstance(constraint, _Eq): - value = constraint.value - - def check_eq(v: Any) -> bool: - return operator.__eq__(v, value) - - s = _check_func(check_eq, f'== {value}', s) - elif isinstance(constraint, _In): - values = constraint.values - - def check_in(v: Any) -> bool: - return operator.__contains__(values, v) - - s = _check_func(check_in, f'in {values}', s) - elif isinstance(constraint, _NotIn): - values = constraint.values - - def check_not_in(v: Any) -> bool: - return operator.__not__(operator.__contains__(values, v)) - - s = _check_func(check_not_in, f'not in {values}', s) - else: - assert isinstance(constraint, Pattern) - if s and s['type'] == 'str': - s = s.copy() - s['pattern'] = constraint.pattern - else: - - def check_pattern(v: object) -> bool: - assert isinstance(v, str) - return constraint.match(v) is not None - - s = _check_func(check_pattern, f'~ {constraint.pattern}', s) - return s - - -class _SupportsRange(annotated_types.SupportsLe, annotated_types.SupportsGe, Protocol): - pass - - -class _SupportsLen(Protocol): - def __len__(self) -> int: ... - - -_NewOutGt = TypeVar('_NewOutGt', bound=annotated_types.SupportsGt) -_NewOutGe = TypeVar('_NewOutGe', bound=annotated_types.SupportsGe) -_NewOutLt = TypeVar('_NewOutLt', bound=annotated_types.SupportsLt) -_NewOutLe = TypeVar('_NewOutLe', bound=annotated_types.SupportsLe) -_NewOutLen = TypeVar('_NewOutLen', bound=_SupportsLen) -_NewOutDiv = TypeVar('_NewOutDiv', bound=annotated_types.SupportsDiv) -_NewOutMod = TypeVar('_NewOutMod', bound=annotated_types.SupportsMod) -_NewOutDatetime = TypeVar('_NewOutDatetime', bound=datetime.datetime) -_NewOutInterval = TypeVar('_NewOutInterval', bound=_SupportsRange) -_OtherIn = TypeVar('_OtherIn') -_OtherOut = TypeVar('_OtherOut') diff --git a/functions/python/data-processor/pydantic/fields.py b/functions/python/data-processor/pydantic/fields.py deleted file mode 100644 index ede966fa8..000000000 --- a/functions/python/data-processor/pydantic/fields.py +++ /dev/null @@ -1,1892 +0,0 @@ -"""Defining fields on models.""" - -from __future__ import annotations as _annotations - -import dataclasses -import inspect -import re -import sys -from collections.abc import Callable, Mapping -from copy import copy -from dataclasses import Field as DataclassField -from functools import cached_property -from typing import TYPE_CHECKING, Annotated, Any, ClassVar, Literal, TypeVar, final, overload -from warnings import warn - -import annotated_types -import typing_extensions -from pydantic_core import MISSING, PydanticUndefined -from typing_extensions import Self, TypeAlias, TypedDict, Unpack, deprecated -from typing_inspection import typing_objects -from typing_inspection.introspection import UNKNOWN, AnnotationSource, ForbiddenQualifier, Qualifier, inspect_annotation - -from . import types -from ._internal import _decorators, _fields, _generics, _internal_dataclass, _repr, _typing_extra, _utils -from ._internal._namespace_utils import GlobalsNamespace, MappingNamespace -from .aliases import AliasChoices, AliasGenerator, AliasPath -from .config import JsonDict -from .errors import PydanticForbiddenQualifier, PydanticUserError -from .json_schema import PydanticJsonSchemaWarning -from .warnings import PydanticDeprecatedSince20 - -if TYPE_CHECKING: - from ._internal._config import ConfigWrapper - from ._internal._repr import ReprArgs - - -__all__ = 'Field', 'FieldInfo', 'PrivateAttr', 'computed_field' - - -_Unset: Any = PydanticUndefined - -if sys.version_info >= (3, 13): - import warnings - - Deprecated: TypeAlias = warnings.deprecated | deprecated -else: - Deprecated: TypeAlias = deprecated - - -class _FromFieldInfoInputs(TypedDict, total=False): - """This class exists solely to add type checking for the `**kwargs` in `FieldInfo.from_field`.""" - - # TODO PEP 747: use TypeForm: - annotation: type[Any] | None - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None - alias: str | None - alias_priority: int | None - validation_alias: str | AliasPath | AliasChoices | None - serialization_alias: str | None - title: str | None - field_title_generator: Callable[[str, FieldInfo], str] | None - description: str | None - examples: list[Any] | None - exclude: bool | None - exclude_if: Callable[[Any], bool] | None - gt: annotated_types.SupportsGt | None - ge: annotated_types.SupportsGe | None - lt: annotated_types.SupportsLt | None - le: annotated_types.SupportsLe | None - multiple_of: float | None - strict: bool | None - min_length: int | None - max_length: int | None - pattern: str | re.Pattern[str] | None - allow_inf_nan: bool | None - max_digits: int | None - decimal_places: int | None - union_mode: Literal['smart', 'left_to_right'] | None - discriminator: str | types.Discriminator | None - deprecated: Deprecated | str | bool | None - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None - frozen: bool | None - validate_default: bool | None - repr: bool - init: bool | None - init_var: bool | None - kw_only: bool | None - coerce_numbers_to_str: bool | None - fail_fast: bool | None - - -class _FieldInfoInputs(_FromFieldInfoInputs, total=False): - """This class exists solely to add type checking for the `**kwargs` in `FieldInfo.__init__`.""" - - default: Any - - -class _FieldInfoAsDict(TypedDict, closed=True): - # TODO PEP 747: use TypeForm: - annotation: Any - metadata: list[Any] - attributes: dict[str, Any] - - -@final -class FieldInfo(_repr.Representation): - """This class holds information about a field. - - `FieldInfo` is used for any field definition regardless of whether the [`Field()`][pydantic.fields.Field] - function is explicitly used. - - !!! warning - The `FieldInfo` class is meant to expose information about a field in a Pydantic model or dataclass. - `FieldInfo` instances shouldn't be instantiated directly, nor mutated. - - If you need to derive a new model from another one and are willing to alter `FieldInfo` instances, - refer to this [dynamic model example](../examples/dynamic_models.md). - - Attributes: - annotation: The type annotation of the field. - default: The default value of the field. - default_factory: A callable to generate the default value. The callable can either take 0 arguments - (in which case it is called as is) or a single argument containing the already validated data. - alias: The alias name of the field. - alias_priority: The priority of the field's alias. - validation_alias: The validation alias of the field. - serialization_alias: The serialization alias of the field. - title: The title of the field. - field_title_generator: A callable that takes a field name and returns title for it. - description: The description of the field. - examples: List of examples of the field. - exclude: Whether to exclude the field from the model serialization. - exclude_if: A callable that determines whether to exclude a field during serialization based on its value. - discriminator: Field name or Discriminator for discriminating the type in a tagged union. - deprecated: A deprecation message, an instance of `warnings.deprecated` or the `typing_extensions.deprecated` backport, - or a boolean. If `True`, a default deprecation message will be emitted when accessing the field. - json_schema_extra: A dict or callable to provide extra JSON schema properties. - frozen: Whether the field is frozen. - validate_default: Whether to validate the default value of the field. - repr: Whether to include the field in representation of the model. - init: Whether the field should be included in the constructor of the dataclass. - init_var: Whether the field should _only_ be included in the constructor of the dataclass, and not stored. - kw_only: Whether the field should be a keyword-only argument in the constructor of the dataclass. - metadata: The metadata list. Contains all the data that isn't expressed as direct `FieldInfo` attributes, including: - - * Type-specific constraints, such as `gt` or `min_length` (these are converted to metadata classes such as `annotated_types.Gt`). - * Any other arbitrary object used within [`Annotated`][typing.Annotated] metadata - (e.g. [custom types handlers](../concepts/types.md#as-an-annotation) or any object not recognized by Pydantic). - """ - - # TODO PEP 747: use TypeForm: - annotation: type[Any] | None - default: Any - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None - alias: str | None - alias_priority: int | None - validation_alias: str | AliasPath | AliasChoices | None - serialization_alias: str | None - title: str | None - field_title_generator: Callable[[str, FieldInfo], str] | None - description: str | None - examples: list[Any] | None - exclude: bool | None - exclude_if: Callable[[Any], bool] | None - discriminator: str | types.Discriminator | None - deprecated: Deprecated | str | bool | None - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None - frozen: bool | None - validate_default: bool | None - repr: bool - init: bool | None - init_var: bool | None - kw_only: bool | None - metadata: list[Any] - - __slots__ = ( - 'annotation', - 'default', - 'default_factory', - 'alias', - 'alias_priority', - 'validation_alias', - 'serialization_alias', - 'title', - 'field_title_generator', - 'description', - 'examples', - 'exclude', - 'exclude_if', - 'discriminator', - 'deprecated', - 'json_schema_extra', - 'frozen', - 'validate_default', - 'repr', - 'init', - 'init_var', - 'kw_only', - 'metadata', - '_attributes_set', - '_qualifiers', - '_complete', - '_original_assignment', - '_original_annotation', - '_final', - ) - - # used to convert kwargs to metadata/constraints, - # None has a special meaning - these items are collected into a `PydanticGeneralMetadata` - metadata_lookup: ClassVar[dict[str, Callable[[Any], Any] | None]] = { - 'strict': types.Strict, - 'gt': annotated_types.Gt, - 'ge': annotated_types.Ge, - 'lt': annotated_types.Lt, - 'le': annotated_types.Le, - 'multiple_of': annotated_types.MultipleOf, - 'min_length': annotated_types.MinLen, - 'max_length': annotated_types.MaxLen, - 'pattern': None, - 'allow_inf_nan': None, - 'max_digits': None, - 'decimal_places': None, - 'union_mode': None, - 'coerce_numbers_to_str': None, - 'fail_fast': types.FailFast, - } - - def __init__(self, **kwargs: Unpack[_FieldInfoInputs]) -> None: - """This class should generally not be initialized directly; instead, use the `pydantic.fields.Field` function - or one of the constructor classmethods. - - See the signature of `pydantic.fields.Field` for more details about the expected arguments. - """ - # Tracking the explicitly set attributes is necessary to correctly merge `Field()` functions - # (e.g. with `Annotated[int, Field(alias='a'), Field(alias=None)]`, even though `None` is the default value, - # we need to track that `alias=None` was explicitly set): - self._attributes_set = {k: v for k, v in kwargs.items() if v is not _Unset and k not in self.metadata_lookup} - kwargs = {k: _DefaultValues.get(k) if v is _Unset else v for k, v in kwargs.items()} # type: ignore - self.annotation = kwargs.get('annotation') - - # Note: in theory, the second `pop()` arguments are not required below, as defaults are already set from `_DefaultsValues`. - default = kwargs.pop('default', PydanticUndefined) - if default is Ellipsis: - self.default = PydanticUndefined - self._attributes_set.pop('default', None) - else: - self.default = default - - self.default_factory = kwargs.pop('default_factory', None) - - if self.default is not PydanticUndefined and self.default_factory is not None: - raise TypeError('cannot specify both default and default_factory') - - self.alias = kwargs.pop('alias', None) - self.validation_alias = kwargs.pop('validation_alias', None) - self.serialization_alias = kwargs.pop('serialization_alias', None) - alias_is_set = any(alias is not None for alias in (self.alias, self.validation_alias, self.serialization_alias)) - self.alias_priority = kwargs.pop('alias_priority', None) or 2 if alias_is_set else None - self.title = kwargs.pop('title', None) - self.field_title_generator = kwargs.pop('field_title_generator', None) - self.description = kwargs.pop('description', None) - self.examples = kwargs.pop('examples', None) - self.exclude = kwargs.pop('exclude', None) - self.exclude_if = kwargs.pop('exclude_if', None) - self.discriminator = kwargs.pop('discriminator', None) - # For compatibility with FastAPI<=0.110.0, we preserve the existing value if it is not overridden - self.deprecated = kwargs.pop('deprecated', getattr(self, 'deprecated', None)) - self.repr = kwargs.pop('repr', True) - self.json_schema_extra = kwargs.pop('json_schema_extra', None) - self.validate_default = kwargs.pop('validate_default', None) - self.frozen = kwargs.pop('frozen', None) - # currently only used on dataclasses - self.init = kwargs.pop('init', None) - self.init_var = kwargs.pop('init_var', None) - self.kw_only = kwargs.pop('kw_only', None) - - self.metadata = self._collect_metadata(kwargs) # type: ignore - - # Private attributes: - self._qualifiers: set[Qualifier] = set() - # Used to rebuild FieldInfo instances: - self._complete = True - self._original_annotation: Any = PydanticUndefined - self._original_assignment: Any = PydanticUndefined - # Used to track whether the `FieldInfo` instance represents the data about a field (and is exposed in `model_fields`/`__pydantic_fields__`), - # or if it is the result of the `Field()` function being used as metadata in an `Annotated` type/as an assignment - # (not an ideal pattern, see https://github.com/pydantic/pydantic/issues/11122): - self._final = False - - @staticmethod - def from_field(default: Any = PydanticUndefined, **kwargs: Unpack[_FromFieldInfoInputs]) -> FieldInfo: - """Create a new `FieldInfo` object with the `Field` function. - - Args: - default: The default value for the field. Defaults to Undefined. - **kwargs: Additional arguments dictionary. - - Raises: - TypeError: If 'annotation' is passed as a keyword argument. - - Returns: - A new FieldInfo object with the given parameters. - - Example: - This is how you can create a field with default value like this: - - ```python - import pydantic - - class MyModel(pydantic.BaseModel): - foo: int = pydantic.Field(4) - ``` - """ - if 'annotation' in kwargs: - raise TypeError('"annotation" is not permitted as a Field keyword argument') - return FieldInfo(default=default, **kwargs) - - @staticmethod - def from_annotation(annotation: type[Any], *, _source: AnnotationSource = AnnotationSource.ANY) -> FieldInfo: - """Creates a `FieldInfo` instance from a bare annotation. - - This function is used internally to create a `FieldInfo` from a bare annotation like this: - - ```python - import pydantic - - class MyModel(pydantic.BaseModel): - foo: int # <-- like this - ``` - - We also account for the case where the annotation can be an instance of `Annotated` and where - one of the (not first) arguments in `Annotated` is an instance of `FieldInfo`, e.g.: - - ```python - from typing import Annotated - - import annotated_types - - import pydantic - - class MyModel(pydantic.BaseModel): - foo: Annotated[int, annotated_types.Gt(42)] - bar: Annotated[int, pydantic.Field(gt=42)] - ``` - - Args: - annotation: An annotation object. - - Returns: - An instance of the field metadata. - """ - try: - inspected_ann = inspect_annotation( - annotation, - annotation_source=_source, - unpack_type_aliases='skip', - ) - except ForbiddenQualifier as e: - raise PydanticForbiddenQualifier(e.qualifier, annotation) - - # TODO check for classvar and error? - - # No assigned value, this happens when using a bare `Final` qualifier (also for other - # qualifiers, but they shouldn't appear here). In this case we infer the type as `Any` - # because we don't have any assigned value. - type_expr: Any = Any if inspected_ann.type is UNKNOWN else inspected_ann.type - final = 'final' in inspected_ann.qualifiers - metadata = inspected_ann.metadata - - attr_overrides = {'annotation': type_expr} - if final: - attr_overrides['frozen'] = True - field_info = FieldInfo._construct(metadata, **attr_overrides) - field_info._qualifiers = inspected_ann.qualifiers - field_info._final = True - return field_info - - @staticmethod - def from_annotated_attribute( - annotation: type[Any], default: Any, *, _source: AnnotationSource = AnnotationSource.ANY - ) -> FieldInfo: - """Create `FieldInfo` from an annotation with a default value. - - This is used in cases like the following: - - ```python - from typing import Annotated - - import annotated_types - - import pydantic - - class MyModel(pydantic.BaseModel): - foo: int = 4 # <-- like this - bar: Annotated[int, annotated_types.Gt(4)] = 4 # <-- or this - spam: Annotated[int, pydantic.Field(gt=4)] = 4 # <-- or this - ``` - - Args: - annotation: The type annotation of the field. - default: The default value of the field. - - Returns: - A field object with the passed values. - """ - if annotation is not MISSING and annotation is default: - raise PydanticUserError( - 'Error when building FieldInfo from annotated attribute. ' - "Make sure you don't have any field name clashing with a type annotation.", - code='unevaluable-type-annotation', - ) - - try: - inspected_ann = inspect_annotation( - annotation, - annotation_source=_source, - unpack_type_aliases='skip', - ) - except ForbiddenQualifier as e: - raise PydanticForbiddenQualifier(e.qualifier, annotation) - - # TODO check for classvar and error? - - # TODO infer from the default, this can be done in v3 once we treat final fields with - # a default as proper fields and not class variables: - type_expr: Any = Any if inspected_ann.type is UNKNOWN else inspected_ann.type - final = 'final' in inspected_ann.qualifiers - metadata = inspected_ann.metadata - - # HACK 1: the order in which the metadata is merged is inconsistent; we need to prepend - # metadata from the assignment at the beginning of the metadata. Changing this is only - # possible in v3 (at least). See https://github.com/pydantic/pydantic/issues/10507 - prepend_metadata: list[Any] | None = None - attr_overrides = {'annotation': type_expr} - if final: - attr_overrides['frozen'] = True - - # HACK 2: FastAPI is subclassing `FieldInfo` and historically expected the actual - # instance's type to be preserved when constructing new models with its subclasses as assignments. - # This code is never reached by Pydantic itself, and in an ideal world this shouldn't be necessary. - if not metadata and isinstance(default, FieldInfo) and type(default) is not FieldInfo: - field_info = default._copy() - field_info._attributes_set.update(attr_overrides) - for k, v in attr_overrides.items(): - setattr(field_info, k, v) - return field_info - - if isinstance(default, FieldInfo): - default_copy = default._copy() # Copy unnecessary when we remove HACK 1. - prepend_metadata = default_copy.metadata - default_copy.metadata = [] - metadata = metadata + [default_copy] - if 'init_var' in inspected_ann.qualifiers: - # Only relevant for dataclasses, when `f: InitVar[] = Field(...)` - # is used: - attr_overrides['init_var'] = True - elif isinstance(default, dataclasses.Field): - from_field = FieldInfo._from_dataclass_field(default) - prepend_metadata = from_field.metadata # Unnecessary when we remove HACK 1. - from_field.metadata = [] - metadata = metadata + [from_field] - if 'init_var' in inspected_ann.qualifiers: - attr_overrides['init_var'] = True - if (init := getattr(default, 'init', None)) is not None: - attr_overrides['init'] = init - if (kw_only := getattr(default, 'kw_only', None)) is not None: - attr_overrides['kw_only'] = kw_only - else: - # `default` is the actual default value - attr_overrides['default'] = default - - field_info = FieldInfo._construct( - prepend_metadata + metadata if prepend_metadata is not None else metadata, **attr_overrides - ) - field_info._qualifiers = inspected_ann.qualifiers - field_info._final = True - return field_info - - @classmethod - def _construct(cls, metadata: list[Any], **attr_overrides: Any) -> Self: - """Construct the final `FieldInfo` instance, by merging the possibly existing `FieldInfo` instances from the metadata. - - With the following example: - - ```python {test="skip" lint="skip"} - class Model(BaseModel): - f: Annotated[int, Gt(1), Field(description='desc', lt=2)] - ``` - - `metadata` refers to the metadata elements of the `Annotated` form. This metadata is iterated over from left to right: - - - If the element is a `Field()` function (which is itself a `FieldInfo` instance), the field attributes (such as - `description`) are saved to be set on the final `FieldInfo` instance. - On the other hand, some kwargs (such as `lt`) are stored as `metadata` (see `FieldInfo.__init__()`, calling - `FieldInfo._collect_metadata()`). In this case, the final metadata list is extended with the one from this instance. - - Else, the element is considered as a single metadata object, and is appended to the final metadata list. - - Args: - metadata: The list of metadata elements to merge together. If the `FieldInfo` instance to be constructed is for - a field with an assigned `Field()`, this `Field()` assignment should be added as the last element of the - provided metadata. - **attr_overrides: Extra attributes that should be set on the final merged `FieldInfo` instance. - - Returns: - The final merged `FieldInfo` instance. - """ - merged_metadata: list[Any] = [] - merged_kwargs: dict[str, Any] = {} - - for meta in metadata: - if isinstance(meta, FieldInfo): - merged_metadata.extend(meta.metadata) - - new_js_extra: JsonDict | None = None - current_js_extra = meta.json_schema_extra - if current_js_extra is not None and 'json_schema_extra' in merged_kwargs: - # We need to merge `json_schema_extra`'s: - existing_js_extra = merged_kwargs['json_schema_extra'] - if isinstance(existing_js_extra, dict): - if isinstance(current_js_extra, dict): - new_js_extra = { - **existing_js_extra, - **current_js_extra, - } - elif callable(current_js_extra): - warn( - 'Composing `dict` and `callable` type `json_schema_extra` is not supported. ' - 'The `callable` type is being ignored. ' - "If you'd like support for this behavior, please open an issue on pydantic.", - UserWarning, - ) - elif callable(existing_js_extra) and isinstance(current_js_extra, dict): - warn( - 'Composing `dict` and `callable` type `json_schema_extra` is not supported. ' - 'The `callable` type is being ignored. ' - "If you'd like support for this behavior, please open an issue on pydantic.", - UserWarning, - ) - - # HACK: It is common for users to define "make model partial" (or similar) utilities, that - # convert all model fields to be optional (i.e. have a default value). To do so, they mutate - # each `FieldInfo` instance from `model_fields` to set a `default`, and use `create_model()` - # with `Annotated[ | None, mutated_field_info]`` as an annotation. However, such - # mutations (by doing simple assignments) are only accidentally working, because we also - # need to track attributes explicitly set in `_attributes_set` (relying on default values for - # each attribute is *not* enough, for instance with `Annotated[int, Field(alias='a'), Field(alias=None)]` - # the resulting `FieldInfo` should have `alias=None`). - # To mitigate this, we add a special case when a "final" `FieldInfo` instance (that is an instance coming - # from `model_fields`) is used in annotated metadata (or assignment). In this case, we assume *all* attributes - # were explicitly set, and as such we use all of them (and this will correctly pick up the mutations). - # In theory, this shouldn't really be supported, you are only supposed to use the `Field()` function, not - # a `FieldInfo` instance directly (granted, `Field()` returns a `FieldInfo`, see - # https://github.com/pydantic/pydantic/issues/11122): - if meta._final: - merged_kwargs.update({attr: getattr(meta, attr) for attr in _Attrs}) - else: - merged_kwargs.update(meta._attributes_set) - - if new_js_extra is not None: - merged_kwargs['json_schema_extra'] = new_js_extra - elif typing_objects.is_deprecated(meta): - merged_kwargs['deprecated'] = meta - else: - merged_metadata.append(meta) - - merged_kwargs.update(attr_overrides) - merged_field_info = cls(**merged_kwargs) - merged_field_info.metadata = merged_metadata - return merged_field_info - - @staticmethod - @typing_extensions.deprecated( - "The 'merge_field_infos()' method is deprecated and will be removed in a future version. " - 'If you relied on this method, please open an issue in the Pydantic issue tracker.', - category=None, - ) - def merge_field_infos(*field_infos: FieldInfo, **overrides: Any) -> FieldInfo: - """Merge `FieldInfo` instances keeping only explicitly set attributes. - - Later `FieldInfo` instances override earlier ones. - - Returns: - FieldInfo: A merged FieldInfo instance. - """ - if len(field_infos) == 1: - # No merging necessary, but we still need to make a copy and apply the overrides - field_info = field_infos[0]._copy() - field_info._attributes_set.update(overrides) - - default_override = overrides.pop('default', PydanticUndefined) - if default_override is Ellipsis: - default_override = PydanticUndefined - if default_override is not PydanticUndefined: - field_info.default = default_override - - for k, v in overrides.items(): - setattr(field_info, k, v) - return field_info # type: ignore - - merged_field_info_kwargs: dict[str, Any] = {} - metadata = {} - for field_info in field_infos: - attributes_set = field_info._attributes_set.copy() - - try: - json_schema_extra = attributes_set.pop('json_schema_extra') - existing_json_schema_extra = merged_field_info_kwargs.get('json_schema_extra') - - if existing_json_schema_extra is None: - merged_field_info_kwargs['json_schema_extra'] = json_schema_extra - if isinstance(existing_json_schema_extra, dict): - if isinstance(json_schema_extra, dict): - merged_field_info_kwargs['json_schema_extra'] = { - **existing_json_schema_extra, - **json_schema_extra, - } - if callable(json_schema_extra): - warn( - 'Composing `dict` and `callable` type `json_schema_extra` is not supported.' - 'The `callable` type is being ignored.' - "If you'd like support for this behavior, please open an issue on pydantic.", - PydanticJsonSchemaWarning, - ) - elif callable(json_schema_extra): - # if ever there's a case of a callable, we'll just keep the last json schema extra spec - merged_field_info_kwargs['json_schema_extra'] = json_schema_extra - except KeyError: - pass - - # later FieldInfo instances override everything except json_schema_extra from earlier FieldInfo instances - merged_field_info_kwargs.update(attributes_set) - - for x in field_info.metadata: - if not isinstance(x, FieldInfo): - metadata[type(x)] = x - - merged_field_info_kwargs.update(overrides) - field_info = FieldInfo(**merged_field_info_kwargs) - field_info.metadata = list(metadata.values()) - return field_info - - @staticmethod - def _from_dataclass_field(dc_field: DataclassField[Any]) -> FieldInfo: - """Return a new `FieldInfo` instance from a `dataclasses.Field` instance. - - Args: - dc_field: The `dataclasses.Field` instance to convert. - - Returns: - The corresponding `FieldInfo` instance. - - Raises: - TypeError: If any of the `FieldInfo` kwargs does not match the `dataclass.Field` kwargs. - """ - default = dc_field.default - if default is dataclasses.MISSING: - default = _Unset - - if dc_field.default_factory is dataclasses.MISSING: - default_factory = _Unset - else: - default_factory = dc_field.default_factory - - # use the `Field` function so in correct kwargs raise the correct `TypeError` - dc_field_metadata = {k: v for k, v in dc_field.metadata.items() if k in _FIELD_ARG_NAMES} - if sys.version_info >= (3, 14) and dc_field.doc is not None: - dc_field_metadata['description'] = dc_field.doc - return Field(default=default, default_factory=default_factory, repr=dc_field.repr, **dc_field_metadata) # pyright: ignore[reportCallIssue] - - @staticmethod - def _collect_metadata(kwargs: dict[str, Any]) -> list[Any]: - """Collect annotations from kwargs. - - Args: - kwargs: Keyword arguments passed to the function. - - Returns: - A list of metadata objects - a combination of `annotated_types.BaseMetadata` and - `PydanticMetadata`. - """ - metadata: list[Any] = [] - general_metadata = {} - for key, value in list(kwargs.items()): - try: - marker = FieldInfo.metadata_lookup[key] - except KeyError: - continue - - del kwargs[key] - if value is not None: - if marker is None: - general_metadata[key] = value - else: - metadata.append(marker(value)) - if general_metadata: - metadata.append(_fields.pydantic_general_metadata(**general_metadata)) - return metadata - - @property - def deprecation_message(self) -> str | None: - """The deprecation message to be emitted, or `None` if not set.""" - if self.deprecated is None: - return None - if isinstance(self.deprecated, bool): - return 'deprecated' if self.deprecated else None - return self.deprecated if isinstance(self.deprecated, str) else self.deprecated.message - - @property - def default_factory_takes_validated_data(self) -> bool | None: - """Whether the provided default factory callable has a validated data parameter. - - Returns `None` if no default factory is set. - """ - if self.default_factory is not None: - return _fields.takes_validated_data_argument(self.default_factory) - - @overload - def get_default( - self, *, call_default_factory: Literal[True], validated_data: dict[str, Any] | None = None - ) -> Any: ... - - @overload - def get_default(self, *, call_default_factory: Literal[False] = ...) -> Any: ... - - def get_default(self, *, call_default_factory: bool = False, validated_data: dict[str, Any] | None = None) -> Any: - """Get the default value. - - We expose an option for whether to call the default_factory (if present), as calling it may - result in side effects that we want to avoid. However, there are times when it really should - be called (namely, when instantiating a model via `model_construct`). - - Args: - call_default_factory: Whether to call the default factory or not. - validated_data: The already validated data to be passed to the default factory. - - Returns: - The default value, calling the default factory if requested or `PydanticUndefined` if not set. - """ - return _fields.resolve_default_value( - default=self.default, - default_factory=self.default_factory, - validated_data=validated_data, - call_default_factory=call_default_factory, - ) - - def is_required(self) -> bool: - """Check if the field is required (i.e., does not have a default value or factory). - - Returns: - `True` if the field is required, `False` otherwise. - """ - return self.default is PydanticUndefined and self.default_factory is None - - def rebuild_annotation(self) -> Any: - """Attempts to rebuild the original annotation for use in function signatures. - - If metadata is present, it adds it to the original annotation using - `Annotated`. Otherwise, it returns the original annotation as-is. - - Note that because the metadata has been flattened, the original annotation - may not be reconstructed exactly as originally provided, e.g. if the original - type had unrecognized annotations, or was annotated with a call to `pydantic.Field`. - - Returns: - The rebuilt annotation. - """ - if not self.metadata: - return self.annotation - else: - # Annotated arguments must be a tuple - return Annotated[(self.annotation, *self.metadata)] # type: ignore - - def apply_typevars_map( - self, - typevars_map: Mapping[TypeVar, Any] | None, - globalns: GlobalsNamespace | None = None, - localns: MappingNamespace | None = None, - ) -> None: - """Apply a `typevars_map` to the annotation. - - This method is used when analyzing parametrized generic types to replace typevars with their concrete types. - - This method applies the `typevars_map` to the annotation in place. - - Args: - typevars_map: A dictionary mapping type variables to their concrete types. - globalns: The globals namespace to use during type annotation evaluation. - localns: The locals namespace to use during type annotation evaluation. - - See Also: - pydantic._internal._generics.replace_types is used for replacing the typevars with - their concrete types. - """ - annotation = _generics.replace_types(self.annotation, typevars_map) - annotation, evaluated = _typing_extra.try_eval_type(annotation, globalns, localns) - self.annotation = annotation - if not evaluated: - self._complete = False - self._original_annotation = self.annotation - - def asdict(self) -> _FieldInfoAsDict: - """Return a dictionary representation of the `FieldInfo` instance. - - The returned value is a dictionary with three items: - - * `annotation`: The type annotation of the field. - * `metadata`: The metadata list. - * `attributes`: A mapping of the remaining `FieldInfo` attributes to their values (e.g. `alias`, `title`). - """ - return { - 'annotation': self.annotation, - 'metadata': self.metadata, - 'attributes': {attr: getattr(self, attr) for attr in _Attrs}, - } - - def _copy(self) -> Self: - """Return a copy of the `FieldInfo` instance.""" - # Note: we can't define a custom `__copy__()`, as `FieldInfo` is being subclassed - # by some third-party libraries with extra attributes defined (and as `FieldInfo` - # is slotted, we can't make a copy of the `__dict__`). - if type(self) is FieldInfo: - # Fast-path if the instance isn't a subclass (`copy.copy()` relies on pickling which is slower): - copied = FieldInfo.__new__(FieldInfo) - for attr_name in FieldInfo.__slots__: - setattr(copied, attr_name, getattr(self, attr_name)) - else: - copied = copy(self) - - for attr_name in ('metadata', '_attributes_set', '_qualifiers'): - # Apply "deep-copy" behavior on collections attributes: - setattr(copied, attr_name, getattr(copied, attr_name).copy()) - - return copied # pyright: ignore[reportReturnType] - - def __repr_args__(self) -> ReprArgs: - yield 'annotation', _repr.PlainRepr(_repr.display_as_type(self.annotation)) - yield 'required', self.is_required() - - for s in self.__slots__: - # TODO: properly make use of the protocol (https://rich.readthedocs.io/en/stable/pretty.html#rich-repr-protocol) - # By yielding a three-tuple: - if s in ( - 'annotation', - '_attributes_set', - '_qualifiers', - '_complete', - '_original_assignment', - '_original_annotation', - '_final', - ): - continue - elif s == 'metadata' and not self.metadata: - continue - elif s == 'repr' and self.repr is True: - continue - if s == 'frozen' and self.frozen is False: - continue - if s == 'validation_alias' and self.validation_alias == self.alias: - continue - if s == 'serialization_alias' and self.serialization_alias == self.alias: - continue - if s == 'default' and self.default is not PydanticUndefined: - yield 'default', self.default - elif s == 'default_factory' and self.default_factory is not None: - yield 'default_factory', _repr.PlainRepr(_repr.display_as_type(self.default_factory)) - else: - value = getattr(self, s) - if value is not None and value is not PydanticUndefined: - yield s, value - - -class _EmptyKwargs(TypedDict): - """This class exists solely to ensure that type checking warns about passing `**extra` in `Field`.""" - - -_Attrs = { - 'default': ..., - 'default_factory': None, - 'alias': None, - 'alias_priority': None, - 'validation_alias': None, - 'serialization_alias': None, - 'title': None, - 'field_title_generator': None, - 'description': None, - 'examples': None, - 'exclude': None, - 'exclude_if': None, - 'discriminator': None, - 'deprecated': None, - 'json_schema_extra': None, - 'frozen': None, - 'validate_default': None, - 'repr': True, - 'init': None, - 'init_var': None, - 'kw_only': None, -} - -_DefaultValues = { - **_Attrs, - 'kw_only': None, - 'pattern': None, - 'strict': None, - 'gt': None, - 'ge': None, - 'lt': None, - 'le': None, - 'multiple_of': None, - 'allow_inf_nan': None, - 'max_digits': None, - 'decimal_places': None, - 'min_length': None, - 'max_length': None, - 'coerce_numbers_to_str': None, -} - - -_T = TypeVar('_T') - - -# NOTE: Actual return type is 'FieldInfo', but we want to help type checkers -# to understand the magic that happens at runtime with the following overloads: -@overload # type hint the return value as `Any` to avoid type checking regressions when using `...`. -def Field( - default: ellipsis, # noqa: F821 # TODO: use `_typing_extra.EllipsisType` when we drop Py3.9 - *, - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: bool | None = _Unset, - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> Any: ... -@overload # `default` argument set, validate_default=True (no type checking on the default value) -def Field( - default: Any, - *, - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: Literal[True], - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> Any: ... -@overload # `default` argument set, validate_default=False or unset -def Field( - default: _T, - *, - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - # NOTE: to get proper type checking on `exclude_if`'s argument, we could use `_T` instead of `Any`. However, - # this requires (at least for pyright) adding an additional overload where `exclude_if` is required (otherwise - # `a: int = Field(default_factory=str)` results in a false negative). - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: Literal[False] = ..., - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> _T: ... -@overload # `default_factory` argument set, validate_default=True (no type checking on the default value) -def Field( # pyright: ignore[reportOverlappingOverload] - *, - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any], - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: Literal[True], - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> Any: ... -@overload # `default_factory` argument set, validate_default=False or unset -def Field( - *, - default_factory: Callable[[], _T] | Callable[[dict[str, Any]], _T], - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - # NOTE: to get proper type checking on `exclude_if`'s argument, we could use `_T` instead of `Any`. However, - # this requires (at least for pyright) adding an additional overload where `exclude_if` is required (otherwise - # `a: int = Field(default_factory=str)` results in a false negative). - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: Literal[False] | None = _Unset, - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> _T: ... -@overload -def Field( # No default set - *, - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: bool | None = _Unset, - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> Any: ... -def Field( # noqa: C901 - default: Any = PydanticUndefined, - *, - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None = _Unset, - alias: str | None = _Unset, - alias_priority: int | None = _Unset, - validation_alias: str | AliasPath | AliasChoices | None = _Unset, - serialization_alias: str | None = _Unset, - title: str | None = _Unset, - field_title_generator: Callable[[str, FieldInfo], str] | None = _Unset, - description: str | None = _Unset, - examples: list[Any] | None = _Unset, - exclude: bool | None = _Unset, - exclude_if: Callable[[Any], bool] | None = _Unset, - discriminator: str | types.Discriminator | None = _Unset, - deprecated: Deprecated | str | bool | None = _Unset, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = _Unset, - frozen: bool | None = _Unset, - validate_default: bool | None = _Unset, - repr: bool = _Unset, - init: bool | None = _Unset, - init_var: bool | None = _Unset, - kw_only: bool | None = _Unset, - pattern: str | re.Pattern[str] | None = _Unset, - strict: bool | None = _Unset, - coerce_numbers_to_str: bool | None = _Unset, - gt: annotated_types.SupportsGt | None = _Unset, - ge: annotated_types.SupportsGe | None = _Unset, - lt: annotated_types.SupportsLt | None = _Unset, - le: annotated_types.SupportsLe | None = _Unset, - multiple_of: float | None = _Unset, - allow_inf_nan: bool | None = _Unset, - max_digits: int | None = _Unset, - decimal_places: int | None = _Unset, - min_length: int | None = _Unset, - max_length: int | None = _Unset, - union_mode: Literal['smart', 'left_to_right'] = _Unset, - fail_fast: bool | None = _Unset, - **extra: Unpack[_EmptyKwargs], -) -> Any: - """!!! abstract "Usage Documentation" - [Fields](../concepts/fields.md) - - Create a field for objects that can be configured. - - Used to provide extra information about a field, either for the model schema or complex validation. Some arguments - apply only to number fields (`int`, `float`, `Decimal`) and some apply only to `str`. - - Note: - - Any `_Unset` objects will be replaced by the corresponding value defined in the `_DefaultValues` dictionary. If a key for the `_Unset` object is not found in the `_DefaultValues` dictionary, it will default to `None` - - Args: - default: Default value if the field is not set. - default_factory: A callable to generate the default value. The callable can either take 0 arguments - (in which case it is called as is) or a single argument containing the already validated data. - alias: The name to use for the attribute when validating or serializing by alias. - This is often used for things like converting between snake and camel case. - alias_priority: Priority of the alias. This affects whether an alias generator is used. - validation_alias: Like `alias`, but only affects validation, not serialization. - serialization_alias: Like `alias`, but only affects serialization, not validation. - title: Human-readable title. - field_title_generator: A callable that takes a field name and returns title for it. - description: Human-readable description. - examples: Example values for this field. - exclude: Whether to exclude the field from the model serialization. - exclude_if: A callable that determines whether to exclude a field during serialization based on its value. - discriminator: Field name or Discriminator for discriminating the type in a tagged union. - deprecated: A deprecation message, an instance of `warnings.deprecated` or the `typing_extensions.deprecated` backport, - or a boolean. If `True`, a default deprecation message will be emitted when accessing the field. - json_schema_extra: A dict or callable to provide extra JSON schema properties. - frozen: Whether the field is frozen. If true, attempts to change the value on an instance will raise an error. - validate_default: If `True`, apply validation to the default value every time you create an instance. - Otherwise, for performance reasons, the default value of the field is trusted and not validated. - repr: A boolean indicating whether to include the field in the `__repr__` output. - init: Whether the field should be included in the constructor of the dataclass. - (Only applies to dataclasses.) - init_var: Whether the field should _only_ be included in the constructor of the dataclass. - (Only applies to dataclasses.) - kw_only: Whether the field should be a keyword-only argument in the constructor of the dataclass. - (Only applies to dataclasses.) - coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode). - strict: If `True`, strict validation is applied to the field. - See [Strict Mode](../concepts/strict_mode.md) for details. - gt: Greater than. If set, value must be greater than this. Only applicable to numbers. - ge: Greater than or equal. If set, value must be greater than or equal to this. Only applicable to numbers. - lt: Less than. If set, value must be less than this. Only applicable to numbers. - le: Less than or equal. If set, value must be less than or equal to this. Only applicable to numbers. - multiple_of: Value must be a multiple of this. Only applicable to numbers. - min_length: Minimum length for iterables. - max_length: Maximum length for iterables. - pattern: Pattern for strings (a regular expression). - allow_inf_nan: Allow `inf`, `-inf`, `nan`. Only applicable to float and [`Decimal`][decimal.Decimal] numbers. - max_digits: Maximum number of allow digits for strings. - decimal_places: Maximum number of decimal places allowed for numbers. - union_mode: The strategy to apply when validating a union. Can be `smart` (the default), or `left_to_right`. - See [Union Mode](../concepts/unions.md#union-modes) for details. - fail_fast: If `True`, validation will stop on the first error. If `False`, all validation errors will be collected. - This option can be applied only to iterable types (list, tuple, set, and frozenset). - extra: (Deprecated) Extra fields that will be included in the JSON schema. - - !!! warning Deprecated - The `extra` kwargs is deprecated. Use `json_schema_extra` instead. - - Returns: - A new [`FieldInfo`][pydantic.fields.FieldInfo]. The return annotation is `Any` so `Field` can be used on - type-annotated fields without causing a type error. - """ - # Check deprecated and removed params from V1. This logic should eventually be removed. - const = extra.pop('const', None) # type: ignore - if const is not None: - raise PydanticUserError('`const` is removed, use `Literal` instead', code='removed-kwargs') - - min_items = extra.pop('min_items', None) # type: ignore - if min_items is not None: - warn( - '`min_items` is deprecated and will be removed, use `min_length` instead', - PydanticDeprecatedSince20, - stacklevel=2, - ) - if min_length in (None, _Unset): - min_length = min_items # type: ignore - - max_items = extra.pop('max_items', None) # type: ignore - if max_items is not None: - warn( - '`max_items` is deprecated and will be removed, use `max_length` instead', - PydanticDeprecatedSince20, - stacklevel=2, - ) - if max_length in (None, _Unset): - max_length = max_items # type: ignore - - unique_items = extra.pop('unique_items', None) # type: ignore - if unique_items is not None: - raise PydanticUserError( - ( - '`unique_items` is removed, use `Set` instead' - '(this feature is discussed in https://github.com/pydantic/pydantic-core/issues/296)' - ), - code='removed-kwargs', - ) - - allow_mutation = extra.pop('allow_mutation', None) # type: ignore - if allow_mutation is not None: - warn( - '`allow_mutation` is deprecated and will be removed. use `frozen` instead', - PydanticDeprecatedSince20, - stacklevel=2, - ) - if allow_mutation is False: - frozen = True - - regex = extra.pop('regex', None) # type: ignore - if regex is not None: - raise PydanticUserError('`regex` is removed. use `pattern` instead', code='removed-kwargs') - - if extra: - warn( - 'Using extra keyword arguments on `Field` is deprecated and will be removed.' - ' Use `json_schema_extra` instead.' - f' (Extra keys: {", ".join(k.__repr__() for k in extra.keys())})', - PydanticDeprecatedSince20, - stacklevel=2, - ) - if not json_schema_extra or json_schema_extra is _Unset: - json_schema_extra = extra # type: ignore - - if ( - validation_alias - and validation_alias is not _Unset - and not isinstance(validation_alias, (str, AliasChoices, AliasPath)) - ): - raise TypeError('Invalid `validation_alias` type. it should be `str`, `AliasChoices`, or `AliasPath`') - - if serialization_alias in (_Unset, None) and isinstance(alias, str): - serialization_alias = alias - - if validation_alias in (_Unset, None): - validation_alias = alias - - include = extra.pop('include', None) # type: ignore - if include is not None: - warn( - '`include` is deprecated and does nothing. It will be removed, use `exclude` instead', - PydanticDeprecatedSince20, - stacklevel=2, - ) - - return FieldInfo.from_field( - default, - default_factory=default_factory, - alias=alias, - alias_priority=alias_priority, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - title=title, - field_title_generator=field_title_generator, - description=description, - examples=examples, - exclude=exclude, - exclude_if=exclude_if, - discriminator=discriminator, - deprecated=deprecated, - json_schema_extra=json_schema_extra, - frozen=frozen, - pattern=pattern, - validate_default=validate_default, - repr=repr, - init=init, - init_var=init_var, - kw_only=kw_only, - coerce_numbers_to_str=coerce_numbers_to_str, - strict=strict, - gt=gt, - ge=ge, - lt=lt, - le=le, - multiple_of=multiple_of, - min_length=min_length, - max_length=max_length, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - union_mode=union_mode, - fail_fast=fail_fast, - ) - - -_FIELD_ARG_NAMES = set(inspect.signature(Field).parameters) -_FIELD_ARG_NAMES.remove('extra') # do not include the varkwargs parameter - - -class ModelPrivateAttr(_repr.Representation): - """A descriptor for private attributes in class models. - - !!! warning - You generally shouldn't be creating `ModelPrivateAttr` instances directly, instead use - `pydantic.fields.PrivateAttr`. (This is similar to `FieldInfo` vs. `Field`.) - - Attributes: - default: The default value of the attribute if not provided. - default_factory: A callable to generate the default value. The callable can either take 0 arguments - (in which case it is called as is) or a single argument containing the validated data (the model's - [`__dict__`][object.__dict__]) and the already initialized private attributes. - """ - - __slots__ = ('default', 'default_factory') - - def __init__( - self, - default: Any = PydanticUndefined, - *, - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None = None, - ) -> None: - if default is Ellipsis: - self.default = PydanticUndefined - else: - self.default = default - self.default_factory = default_factory - - if not TYPE_CHECKING: - # We put `__getattr__` in a non-TYPE_CHECKING block because otherwise, mypy allows arbitrary attribute access - - def __getattr__(self, item: str) -> Any: - """This function improves compatibility with custom descriptors by ensuring delegation happens - as expected when the default value of a private attribute is a descriptor. - """ - if item in {'__get__', '__set__', '__delete__'}: - if hasattr(self.default, item): - return getattr(self.default, item) - raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}') - - def __set_name__(self, cls: type[Any], name: str) -> None: - """Preserve `__set_name__` protocol defined in https://peps.python.org/pep-0487.""" - default = self.default - if default is PydanticUndefined: - return - set_name = getattr(default, '__set_name__', None) - if callable(set_name): - set_name(cls, name) - - @property - def default_factory_takes_validated_data(self) -> bool | None: - """Whether the provided default factory callable has a validated data parameter. - - Returns `None` if no default factory is set. - """ - if self.default_factory is not None: - return _fields.takes_validated_data_argument(self.default_factory) - - @overload - def get_default( - self, *, call_default_factory: Literal[True], validated_data: dict[str, Any] | None = None - ) -> Any: ... - - @overload - def get_default(self, *, call_default_factory: Literal[False] = ...) -> Any: ... - - def get_default(self, *, call_default_factory: bool = False, validated_data: dict[str, Any] | None = None) -> Any: - """Get the default value. - - We expose an option for whether to call the default_factory (if present), as calling it may - result in side effects that we want to avoid. However, there are times when it really should - be called (namely, when instantiating a model via `model_construct`). - - Args: - call_default_factory: Whether to call the default factory or not. - validated_data: The already validated data to be passed to the default factory. - - Returns: - The default value, calling the default factory if requested or `None` if not set. - """ - return _fields.resolve_default_value( - default=self.default, - default_factory=self.default_factory, - validated_data=validated_data, - call_default_factory=call_default_factory, - ) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and (self.default, self.default_factory) == ( - other.default, - other.default_factory, - ) - - -# NOTE: Actual return type is 'ModelPrivateAttr', but we want to help type checkers -# to understand the magic that happens at runtime. -@overload # `default` argument set -def PrivateAttr( - default: _T, - *, - init: Literal[False] = False, -) -> _T: ... -@overload # `default_factory` argument set -def PrivateAttr( - *, - default_factory: Callable[[], _T] | Callable[[dict[str, Any]], _T], - init: Literal[False] = False, -) -> _T: ... -@overload # No default set -def PrivateAttr( - *, - init: Literal[False] = False, -) -> Any: ... -def PrivateAttr( - default: Any = PydanticUndefined, - *, - default_factory: Callable[[], Any] | Callable[[dict[str, Any]], Any] | None = None, - init: Literal[False] = False, -) -> Any: - """!!! abstract "Usage Documentation" - [Private Model Attributes](../concepts/models.md#private-model-attributes) - - Indicates that an attribute is intended for private use and not handled during normal validation/serialization. - - Private attributes are not validated by Pydantic, so it's up to you to ensure they are used in a type-safe manner. - - Private attributes are stored in `__private_attributes__` on the model. - - Args: - default: The attribute's default value. Defaults to Undefined. - default_factory: A callable to generate the default value. The callable can either take 0 arguments - (in which case it is called as is) or a single argument containing the validated data (the model's - [`__dict__`][object.__dict__]) and the already initialized private attributes. - If both `default` and `default_factory` are set, an error will be raised. - init: Whether the attribute should be included in the constructor of the dataclass. Always `False`. - - Returns: - An instance of [`ModelPrivateAttr`][pydantic.fields.ModelPrivateAttr] class. - - Raises: - ValueError: If both `default` and `default_factory` are set. - """ - if default is not PydanticUndefined and default_factory is not None: - raise TypeError('cannot specify both default and default_factory') - - return ModelPrivateAttr( - default, - default_factory=default_factory, - ) - - -@dataclasses.dataclass(**_internal_dataclass.slots_true) -class ComputedFieldInfo: - """A container for data from `@computed_field` so that we can access it while building the pydantic-core schema. - - Attributes: - decorator_repr: A class variable representing the decorator string, '@computed_field'. - wrapped_property: The wrapped computed field property. - return_type: The type of the computed field property's return value. - alias: The alias of the property to be used during serialization. - alias_priority: The priority of the alias. This affects whether an alias generator is used. - title: Title of the computed field to include in the serialization JSON schema. - field_title_generator: A callable that takes a field name and returns title for it. - description: Description of the computed field to include in the serialization JSON schema. - deprecated: A deprecation message, an instance of `warnings.deprecated` or the `typing_extensions.deprecated` backport, - or a boolean. If `True`, a default deprecation message will be emitted when accessing the field. - examples: Example values of the computed field to include in the serialization JSON schema. - json_schema_extra: A dict or callable to provide extra JSON schema properties. - repr: A boolean indicating whether to include the field in the __repr__ output. - """ - - decorator_repr: ClassVar[str] = '@computed_field' - wrapped_property: property - return_type: Any - alias: str | None - alias_priority: int | None - exclude_if: Callable[[Any], bool] | None - title: str | None - field_title_generator: Callable[[str, ComputedFieldInfo], str] | None - description: str | None - deprecated: Deprecated | str | bool | None - examples: list[Any] | None - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None - repr: bool - # NOTE: if you add a new field, add it to the `__copy__()` implementation. - - def __copy__(self) -> Self: - return type(self)( - wrapped_property=self.wrapped_property, - return_type=self.return_type, - alias=self.alias, - alias_priority=self.alias_priority, - exclude_if=self.exclude_if, - title=self.title, - field_title_generator=self.field_title_generator, - description=self.description, - deprecated=self.deprecated, - examples=self.examples.copy() if isinstance(self.examples, list) else self.examples, - json_schema_extra=self.json_schema_extra.copy() - if isinstance(self.json_schema_extra, dict) - else self.json_schema_extra, - repr=self.repr, - ) - - @property - def deprecation_message(self) -> str | None: - """The deprecation message to be emitted, or `None` if not set.""" - if self.deprecated is None: - return None - if isinstance(self.deprecated, bool): - return 'deprecated' if self.deprecated else None - return self.deprecated if isinstance(self.deprecated, str) else self.deprecated.message - - def _update_from_config(self, config_wrapper: ConfigWrapper, name: str) -> None: - """Update the instance from the configuration set on the class this computed field belongs to.""" - title_generator = self.field_title_generator or config_wrapper.field_title_generator - if title_generator is not None and self.title is None: - self.title = title_generator(name, self) - if config_wrapper.alias_generator is not None: - self._apply_alias_generator(config_wrapper.alias_generator, name) - - def _apply_alias_generator(self, alias_generator: Callable[[str], str] | AliasGenerator, name: str) -> None: - """Apply an alias generator to aliases if appropriate. - - Args: - alias_generator: A callable that takes a string and returns a string, or an `AliasGenerator` instance. - name: The name of the computed field from which to generate the alias. - """ - # Apply an alias_generator if - # 1. An alias is not specified - # 2. An alias is specified, but the priority is <= 1 - - if self.alias_priority is None or self.alias_priority <= 1 or self.alias is None: - alias, _, serialization_alias = None, None, None - - if isinstance(alias_generator, AliasGenerator): - alias, _, serialization_alias = alias_generator.generate_aliases(name) - elif callable(alias_generator): - alias = alias_generator(name) - - # if priority is not set, we set to 1 - # which supports the case where the alias_generator from a child class is used - # to generate an alias for a field in a parent class - if self.alias_priority is None or self.alias_priority <= 1: - self.alias_priority = 1 - - # if the priority is 1, then we set the aliases to the generated alias - # note that we use the serialization_alias with priority over alias, as computed_field - # aliases are used for serialization only (not validation) - if self.alias_priority == 1: - self.alias = _utils.get_first_not_none(serialization_alias, alias) - - -def _wrapped_property_is_private(property_: cached_property | property) -> bool: # type: ignore - """Returns true if provided property is private, False otherwise.""" - wrapped_name: str = '' - - if isinstance(property_, property): - wrapped_name = getattr(property_.fget, '__name__', '') - elif isinstance(property_, cached_property): # type: ignore - wrapped_name = getattr(property_.func, '__name__', '') # type: ignore - - return wrapped_name.startswith('_') and not wrapped_name.startswith('__') - - -# this should really be `property[T], cached_property[T]` but property is not generic unlike cached_property -# See https://github.com/python/typing/issues/985 and linked issues -PropertyT = TypeVar('PropertyT') - - -@overload -def computed_field(func: PropertyT, /) -> PropertyT: ... - - -@overload -def computed_field( - *, - alias: str | None = None, - alias_priority: int | None = None, - exclude_if: Callable[[Any], bool] | None = None, - title: str | None = None, - field_title_generator: Callable[[str, ComputedFieldInfo], str] | None = None, - description: str | None = None, - deprecated: Deprecated | str | bool | None = None, - examples: list[Any] | None = None, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = None, - repr: bool = True, - return_type: Any = PydanticUndefined, -) -> Callable[[PropertyT], PropertyT]: ... - - -def computed_field( - func: PropertyT | None = None, - /, - *, - alias: str | None = None, - alias_priority: int | None = None, - exclude_if: Callable[[Any], bool] | None = None, - title: str | None = None, - field_title_generator: Callable[[str, ComputedFieldInfo], str] | None = None, - description: str | None = None, - deprecated: Deprecated | str | bool | None = None, - examples: list[Any] | None = None, - json_schema_extra: JsonDict | Callable[[JsonDict], None] | None = None, - repr: bool | None = None, - return_type: Any = PydanticUndefined, -) -> PropertyT | Callable[[PropertyT], PropertyT]: - """!!! abstract "Usage Documentation" - [The `computed_field` decorator](../concepts/fields.md#the-computed_field-decorator) - - Decorator to include `property` and `cached_property` when serializing models or dataclasses. - - This is useful for fields that are computed from other fields, or for fields that are expensive to compute and should be cached. - - ```python - from pydantic import BaseModel, computed_field - - class Rectangle(BaseModel): - width: int - length: int - - @computed_field - @property - def area(self) -> int: - return self.width * self.length - - print(Rectangle(width=3, length=2).model_dump()) - #> {'width': 3, 'length': 2, 'area': 6} - ``` - - If applied to functions not yet decorated with `@property` or `@cached_property`, the function is - automatically wrapped with `property`. Although this is more concise, you will lose IntelliSense in your IDE, - and confuse static type checkers, thus explicit use of `@property` is recommended. - - !!! warning "Mypy Warning" - Even with the `@property` or `@cached_property` applied to your function before `@computed_field`, - mypy may throw a `Decorated property not supported` error. - See [mypy issue #1362](https://github.com/python/mypy/issues/1362), for more information. - To avoid this error message, add `# type: ignore[prop-decorator]` to the `@computed_field` line. - - [pyright](https://github.com/microsoft/pyright) supports `@computed_field` without error. - - ```python - import random - - from pydantic import BaseModel, computed_field - - class Square(BaseModel): - width: float - - @computed_field - def area(self) -> float: # converted to a `property` by `computed_field` - return round(self.width**2, 2) - - @area.setter - def area(self, new_area: float) -> None: - self.width = new_area**0.5 - - @computed_field(alias='the magic number', repr=False) - def random_number(self) -> int: - return random.randint(0, 1_000) - - square = Square(width=1.3) - - # `random_number` does not appear in representation - print(repr(square)) - #> Square(width=1.3, area=1.69) - - print(square.random_number) - #> 3 - - square.area = 4 - - print(square.model_dump_json(by_alias=True)) - #> {"width":2.0,"area":4.0,"the magic number":3} - ``` - - !!! warning "Overriding with `computed_field`" - You can't override a field from a parent class with a `computed_field` in the child class. - `mypy` complains about this behavior if allowed, and `dataclasses` doesn't allow this pattern either. - See the example below: - - ```python - from pydantic import BaseModel, computed_field - - class Parent(BaseModel): - a: str - - try: - - class Child(Parent): - @computed_field - @property - def a(self) -> str: - return 'new a' - - except TypeError as e: - print(e) - ''' - Field 'a' of class 'Child' overrides symbol of same name in a parent class. This override with a computed_field is incompatible. - ''' - ``` - - Private properties decorated with `@computed_field` have `repr=False` by default. - - ```python - from functools import cached_property - - from pydantic import BaseModel, computed_field - - class Model(BaseModel): - foo: int - - @computed_field - @cached_property - def _private_cached_property(self) -> int: - return -self.foo - - @computed_field - @property - def _private_property(self) -> int: - return -self.foo - - m = Model(foo=1) - print(repr(m)) - #> Model(foo=1) - ``` - - Args: - func: the function to wrap. - alias: alias to use when serializing this computed field, only used when `by_alias=True` - alias_priority: priority of the alias. This affects whether an alias generator is used - exclude_if: A callable that determines whether to exclude this computed field during serialization based on its value. - title: Title to use when including this computed field in JSON Schema - field_title_generator: A callable that takes a field name and returns title for it. - description: Description to use when including this computed field in JSON Schema, defaults to the function's - docstring - deprecated: A deprecation message (or an instance of `warnings.deprecated` or the `typing_extensions.deprecated` backport). - to be emitted when accessing the field. Or a boolean. This will automatically be set if the property is decorated with the - `deprecated` decorator. - examples: Example values to use when including this computed field in JSON Schema - json_schema_extra: A dict or callable to provide extra JSON schema properties. - repr: whether to include this computed field in model repr. - Default is `False` for private properties and `True` for public properties. - return_type: optional return for serialization logic to expect when serializing to JSON, if included - this must be correct, otherwise a `TypeError` is raised. - If you don't include a return type Any is used, which does runtime introspection to handle arbitrary - objects. - - Returns: - A proxy wrapper for the property. - """ - - def dec(f: Any) -> Any: - nonlocal description, deprecated, return_type, alias_priority - unwrapped = _decorators.unwrap_wrapped_function(f) - - if description is None and unwrapped.__doc__: - description = inspect.cleandoc(unwrapped.__doc__) - - if deprecated is None and hasattr(unwrapped, '__deprecated__'): - deprecated = unwrapped.__deprecated__ - - # if the function isn't already decorated with `@property` (or another descriptor), then we wrap it now - f = _decorators.ensure_property(f) - alias_priority = (alias_priority or 2) if alias is not None else None - - if repr is None: - repr_: bool = not _wrapped_property_is_private(property_=f) - else: - repr_ = repr - - dec_info = ComputedFieldInfo( - f, - return_type, - alias, - alias_priority, - exclude_if, - title, - field_title_generator, - description, - deprecated, - examples, - json_schema_extra, - repr_, - ) - return _decorators.PydanticDescriptorProxy(f, dec_info) - - if func is None: - return dec - else: - return dec(func) diff --git a/functions/python/data-processor/pydantic/functional_serializers.py b/functions/python/data-processor/pydantic/functional_serializers.py deleted file mode 100644 index ed6333694..000000000 --- a/functions/python/data-processor/pydantic/functional_serializers.py +++ /dev/null @@ -1,470 +0,0 @@ -"""This module contains related classes and functions for serialization.""" - -from __future__ import annotations - -import dataclasses -from functools import partial, partialmethod -from typing import TYPE_CHECKING, Annotated, Any, Callable, Literal, TypeVar, overload - -from pydantic_core import PydanticUndefined, core_schema -from pydantic_core.core_schema import SerializationInfo, SerializerFunctionWrapHandler, WhenUsed -from typing_extensions import TypeAlias - -from . import PydanticUndefinedAnnotation -from ._internal import _decorators, _internal_dataclass -from .annotated_handlers import GetCoreSchemaHandler -from .errors import PydanticUserError - - -@dataclasses.dataclass(**_internal_dataclass.slots_true, frozen=True) -class PlainSerializer: - """Plain serializers use a function to modify the output of serialization. - - This is particularly helpful when you want to customize the serialization for annotated types. - Consider an input of `list`, which will be serialized into a space-delimited string. - - ```python - from typing import Annotated - - from pydantic import BaseModel, PlainSerializer - - CustomStr = Annotated[ - list, PlainSerializer(lambda x: ' '.join(x), return_type=str) - ] - - class StudentModel(BaseModel): - courses: CustomStr - - student = StudentModel(courses=['Math', 'Chemistry', 'English']) - print(student.model_dump()) - #> {'courses': 'Math Chemistry English'} - ``` - - Attributes: - func: The serializer function. - return_type: The return type for the function. If omitted it will be inferred from the type annotation. - when_used: Determines when this serializer should be used. Accepts a string with values `'always'`, - `'unless-none'`, `'json'`, and `'json-unless-none'`. Defaults to 'always'. - """ - - func: core_schema.SerializerFunction - return_type: Any = PydanticUndefined - when_used: WhenUsed = 'always' - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - """Gets the Pydantic core schema. - - Args: - source_type: The source type. - handler: The `GetCoreSchemaHandler` instance. - - Returns: - The Pydantic core schema. - """ - schema = handler(source_type) - if self.return_type is not PydanticUndefined: - return_type = self.return_type - else: - try: - # Do not pass in globals as the function could be defined in a different module. - # Instead, let `get_callable_return_type` infer the globals to use, but still pass - # in locals that may contain a parent/rebuild namespace: - return_type = _decorators.get_callable_return_type( - self.func, - localns=handler._get_types_namespace().locals, - ) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - return_schema = None if return_type is PydanticUndefined else handler.generate_schema(return_type) - schema['serialization'] = core_schema.plain_serializer_function_ser_schema( - function=self.func, - info_arg=_decorators.inspect_annotated_serializer(self.func, 'plain'), - return_schema=return_schema, - when_used=self.when_used, - ) - return schema - - -@dataclasses.dataclass(**_internal_dataclass.slots_true, frozen=True) -class WrapSerializer: - """Wrap serializers receive the raw inputs along with a handler function that applies the standard serialization - logic, and can modify the resulting value before returning it as the final output of serialization. - - For example, here's a scenario in which a wrap serializer transforms timezones to UTC **and** utilizes the existing `datetime` serialization logic. - - ```python - from datetime import datetime, timezone - from typing import Annotated, Any - - from pydantic import BaseModel, WrapSerializer - - class EventDatetime(BaseModel): - start: datetime - end: datetime - - def convert_to_utc(value: Any, handler, info) -> dict[str, datetime]: - # Note that `handler` can actually help serialize the `value` for - # further custom serialization in case it's a subclass. - partial_result = handler(value, info) - if info.mode == 'json': - return { - k: datetime.fromisoformat(v).astimezone(timezone.utc) - for k, v in partial_result.items() - } - return {k: v.astimezone(timezone.utc) for k, v in partial_result.items()} - - UTCEventDatetime = Annotated[EventDatetime, WrapSerializer(convert_to_utc)] - - class EventModel(BaseModel): - event_datetime: UTCEventDatetime - - dt = EventDatetime( - start='2024-01-01T07:00:00-08:00', end='2024-01-03T20:00:00+06:00' - ) - event = EventModel(event_datetime=dt) - print(event.model_dump()) - ''' - { - 'event_datetime': { - 'start': datetime.datetime( - 2024, 1, 1, 15, 0, tzinfo=datetime.timezone.utc - ), - 'end': datetime.datetime( - 2024, 1, 3, 14, 0, tzinfo=datetime.timezone.utc - ), - } - } - ''' - - print(event.model_dump_json()) - ''' - {"event_datetime":{"start":"2024-01-01T15:00:00Z","end":"2024-01-03T14:00:00Z"}} - ''' - ``` - - Attributes: - func: The serializer function to be wrapped. - return_type: The return type for the function. If omitted it will be inferred from the type annotation. - when_used: Determines when this serializer should be used. Accepts a string with values `'always'`, - `'unless-none'`, `'json'`, and `'json-unless-none'`. Defaults to 'always'. - """ - - func: core_schema.WrapSerializerFunction - return_type: Any = PydanticUndefined - when_used: WhenUsed = 'always' - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - """This method is used to get the Pydantic core schema of the class. - - Args: - source_type: Source type. - handler: Core schema handler. - - Returns: - The generated core schema of the class. - """ - schema = handler(source_type) - if self.return_type is not PydanticUndefined: - return_type = self.return_type - else: - try: - # Do not pass in globals as the function could be defined in a different module. - # Instead, let `get_callable_return_type` infer the globals to use, but still pass - # in locals that may contain a parent/rebuild namespace: - return_type = _decorators.get_callable_return_type( - self.func, - localns=handler._get_types_namespace().locals, - ) - except NameError as e: - raise PydanticUndefinedAnnotation.from_name_error(e) from e - - return_schema = None if return_type is PydanticUndefined else handler.generate_schema(return_type) - schema['serialization'] = core_schema.wrap_serializer_function_ser_schema( - function=self.func, - info_arg=_decorators.inspect_annotated_serializer(self.func, 'wrap'), - return_schema=return_schema, - when_used=self.when_used, - ) - return schema - - -if TYPE_CHECKING: - _Partial: TypeAlias = 'partial[Any] | partialmethod[Any]' - - FieldPlainSerializer: TypeAlias = 'core_schema.SerializerFunction | _Partial' - """A field serializer method or function in `plain` mode.""" - - FieldWrapSerializer: TypeAlias = 'core_schema.WrapSerializerFunction | _Partial' - """A field serializer method or function in `wrap` mode.""" - - FieldSerializer: TypeAlias = 'FieldPlainSerializer | FieldWrapSerializer' - """A field serializer method or function.""" - - _FieldPlainSerializerT = TypeVar('_FieldPlainSerializerT', bound=FieldPlainSerializer) - _FieldWrapSerializerT = TypeVar('_FieldWrapSerializerT', bound=FieldWrapSerializer) - - -@overload -def field_serializer( - field: str, - /, - *fields: str, - mode: Literal['wrap'], - return_type: Any = ..., - when_used: WhenUsed = ..., - check_fields: bool | None = ..., -) -> Callable[[_FieldWrapSerializerT], _FieldWrapSerializerT]: ... - - -@overload -def field_serializer( - field: str, - /, - *fields: str, - mode: Literal['plain'] = ..., - return_type: Any = ..., - when_used: WhenUsed = ..., - check_fields: bool | None = ..., -) -> Callable[[_FieldPlainSerializerT], _FieldPlainSerializerT]: ... - - -def field_serializer( # noqa: D417 - field: str, - /, - *fields: str, - mode: Literal['plain', 'wrap'] = 'plain', - # TODO PEP 747 (grep for 'return_type' on the whole code base): - return_type: Any = PydanticUndefined, - when_used: WhenUsed = 'always', - check_fields: bool | None = None, -) -> ( - Callable[[_FieldWrapSerializerT], _FieldWrapSerializerT] - | Callable[[_FieldPlainSerializerT], _FieldPlainSerializerT] -): - """Decorator that enables custom field serialization. - - In the below example, a field of type `set` is used to mitigate duplication. A `field_serializer` is used to serialize the data as a sorted list. - - ```python - from pydantic import BaseModel, field_serializer - - class StudentModel(BaseModel): - name: str = 'Jane' - courses: set[str] - - @field_serializer('courses', when_used='json') - def serialize_courses_in_order(self, courses: set[str]): - return sorted(courses) - - student = StudentModel(courses={'Math', 'Chemistry', 'English'}) - print(student.model_dump_json()) - #> {"name":"Jane","courses":["Chemistry","English","Math"]} - ``` - - See [the usage documentation](../concepts/serialization.md#serializers) for more information. - - Four signatures are supported for the decorated serializer: - - - `(self, value: Any, info: FieldSerializationInfo)` - - `(self, value: Any, nxt: SerializerFunctionWrapHandler, info: FieldSerializationInfo)` - - `(value: Any, info: SerializationInfo)` - - `(value: Any, nxt: SerializerFunctionWrapHandler, info: SerializationInfo)` - - Args: - *fields: The field names the serializer should apply to. - mode: The serialization mode. - - - `plain` means the function will be called instead of the default serialization logic, - - `wrap` means the function will be called with an argument to optionally call the - default serialization logic. - return_type: Optional return type for the function, if omitted it will be inferred from the type annotation. - when_used: Determines the serializer will be used for serialization. - check_fields: Whether to check that the fields actually exist on the model. - - Raises: - PydanticUserError: - - If the decorator is used without any arguments (at least one field name must be provided). - - If the provided field names are not strings. - """ - if callable(field) or isinstance(field, classmethod): - raise PydanticUserError( - 'The `@field_serializer` decorator cannot be used without arguments, at least one field must be provided. ' - "For example: `@field_serializer('', ...)`.", - code='decorator-missing-arguments', - ) - - fields = field, *fields - if not all(isinstance(field, str) for field in fields): - raise PydanticUserError( - 'The provided field names to the `@field_serializer` decorator should be strings. ' - "For example: `@field_serializer('', '', ...).`", - code='decorator-invalid-fields', - ) - - def dec(f: FieldSerializer) -> _decorators.PydanticDescriptorProxy[Any]: - dec_info = _decorators.FieldSerializerDecoratorInfo( - fields=fields, - mode=mode, - return_type=return_type, - when_used=when_used, - check_fields=check_fields, - ) - return _decorators.PydanticDescriptorProxy(f, dec_info) # pyright: ignore[reportArgumentType] - - return dec # pyright: ignore[reportReturnType] - - -if TYPE_CHECKING: - # The first argument in the following callables represent the `self` type: - - ModelPlainSerializerWithInfo: TypeAlias = Callable[[Any, SerializationInfo[Any]], Any] - """A model serializer method with the `info` argument, in `plain` mode.""" - - ModelPlainSerializerWithoutInfo: TypeAlias = Callable[[Any], Any] - """A model serializer method without the `info` argument, in `plain` mode.""" - - ModelPlainSerializer: TypeAlias = 'ModelPlainSerializerWithInfo | ModelPlainSerializerWithoutInfo' - """A model serializer method in `plain` mode.""" - - ModelWrapSerializerWithInfo: TypeAlias = Callable[[Any, SerializerFunctionWrapHandler, SerializationInfo[Any]], Any] - """A model serializer method with the `info` argument, in `wrap` mode.""" - - ModelWrapSerializerWithoutInfo: TypeAlias = Callable[[Any, SerializerFunctionWrapHandler], Any] - """A model serializer method without the `info` argument, in `wrap` mode.""" - - ModelWrapSerializer: TypeAlias = 'ModelWrapSerializerWithInfo | ModelWrapSerializerWithoutInfo' - """A model serializer method in `wrap` mode.""" - - ModelSerializer: TypeAlias = 'ModelPlainSerializer | ModelWrapSerializer' - - _ModelPlainSerializerT = TypeVar('_ModelPlainSerializerT', bound=ModelPlainSerializer) - _ModelWrapSerializerT = TypeVar('_ModelWrapSerializerT', bound=ModelWrapSerializer) - - -@overload -def model_serializer(f: _ModelPlainSerializerT, /) -> _ModelPlainSerializerT: ... - - -@overload -def model_serializer( - *, mode: Literal['wrap'], when_used: WhenUsed = 'always', return_type: Any = ... -) -> Callable[[_ModelWrapSerializerT], _ModelWrapSerializerT]: ... - - -@overload -def model_serializer( - *, - mode: Literal['plain'] = ..., - when_used: WhenUsed = 'always', - return_type: Any = ..., -) -> Callable[[_ModelPlainSerializerT], _ModelPlainSerializerT]: ... - - -def model_serializer( - f: _ModelPlainSerializerT | _ModelWrapSerializerT | None = None, - /, - *, - mode: Literal['plain', 'wrap'] = 'plain', - when_used: WhenUsed = 'always', - return_type: Any = PydanticUndefined, -) -> ( - _ModelPlainSerializerT - | Callable[[_ModelWrapSerializerT], _ModelWrapSerializerT] - | Callable[[_ModelPlainSerializerT], _ModelPlainSerializerT] -): - """Decorator that enables custom model serialization. - - This is useful when a model need to be serialized in a customized manner, allowing for flexibility beyond just specific fields. - - An example would be to serialize temperature to the same temperature scale, such as degrees Celsius. - - ```python - from typing import Literal - - from pydantic import BaseModel, model_serializer - - class TemperatureModel(BaseModel): - unit: Literal['C', 'F'] - value: int - - @model_serializer() - def serialize_model(self): - if self.unit == 'F': - return {'unit': 'C', 'value': int((self.value - 32) / 1.8)} - return {'unit': self.unit, 'value': self.value} - - temperature = TemperatureModel(unit='F', value=212) - print(temperature.model_dump()) - #> {'unit': 'C', 'value': 100} - ``` - - Two signatures are supported for `mode='plain'`, which is the default: - - - `(self)` - - `(self, info: SerializationInfo)` - - And two other signatures for `mode='wrap'`: - - - `(self, nxt: SerializerFunctionWrapHandler)` - - `(self, nxt: SerializerFunctionWrapHandler, info: SerializationInfo)` - - See [the usage documentation](../concepts/serialization.md#serializers) for more information. - - Args: - f: The function to be decorated. - mode: The serialization mode. - - - `'plain'` means the function will be called instead of the default serialization logic - - `'wrap'` means the function will be called with an argument to optionally call the default - serialization logic. - when_used: Determines when this serializer should be used. - return_type: The return type for the function. If omitted it will be inferred from the type annotation. - - Returns: - The decorator function. - """ - - def dec(f: ModelSerializer) -> _decorators.PydanticDescriptorProxy[Any]: - dec_info = _decorators.ModelSerializerDecoratorInfo(mode=mode, return_type=return_type, when_used=when_used) - return _decorators.PydanticDescriptorProxy(f, dec_info) - - if f is None: - return dec # pyright: ignore[reportReturnType] - else: - return dec(f) # pyright: ignore[reportReturnType] - - -AnyType = TypeVar('AnyType') - - -if TYPE_CHECKING: - SerializeAsAny = Annotated[AnyType, ...] # SerializeAsAny[list[str]] will be treated by type checkers as list[str] - """Annotation used to mark a type as having duck-typing serialization behavior. - - See [usage documentation](../concepts/serialization.md#serializing-with-duck-typing) for more details. - """ -else: - - @dataclasses.dataclass(**_internal_dataclass.slots_true) - class SerializeAsAny: - """Annotation used to mark a type as having duck-typing serialization behavior. - - See [usage documentation](../concepts/serialization.md#serializing-with-duck-typing) for more details. - """ - - def __class_getitem__(cls, item: Any) -> Any: - return Annotated[item, SerializeAsAny()] - - def __get_pydantic_core_schema__( - self, source_type: Any, handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - schema = handler(source_type) - schema_to_update = schema - while schema_to_update['type'] == 'definitions': - schema_to_update = schema_to_update.copy() - schema_to_update = schema_to_update['schema'] - schema_to_update['serialization'] = core_schema.simple_ser_schema('any') - return schema - - __hash__ = object.__hash__ diff --git a/functions/python/data-processor/pydantic/functional_validators.py b/functions/python/data-processor/pydantic/functional_validators.py deleted file mode 100644 index 558e99c1e..000000000 --- a/functions/python/data-processor/pydantic/functional_validators.py +++ /dev/null @@ -1,889 +0,0 @@ -"""This module contains related classes and functions for validation.""" - -from __future__ import annotations as _annotations - -import dataclasses -import sys -import warnings -from functools import partialmethod -from typing import TYPE_CHECKING, Annotated, Any, Callable, Literal, TypeVar, Union, cast, overload - -from pydantic_core import PydanticUndefined, core_schema -from typing_extensions import Self, TypeAlias - -from ._internal import _decorators, _generics, _internal_dataclass -from .annotated_handlers import GetCoreSchemaHandler -from .errors import PydanticUserError -from .version import version_short -from .warnings import ArbitraryTypeWarning, PydanticDeprecatedSince212 - -if sys.version_info < (3, 11): - from typing_extensions import Protocol -else: - from typing import Protocol - -_inspect_validator = _decorators.inspect_validator - - -@dataclasses.dataclass(frozen=True, **_internal_dataclass.slots_true) -class AfterValidator: - """!!! abstract "Usage Documentation" - [field *after* validators](../concepts/validators.md#field-after-validator) - - A metadata class that indicates that a validation should be applied **after** the inner validation logic. - - Attributes: - func: The validator function. - - Example: - ```python - from typing import Annotated - - from pydantic import AfterValidator, BaseModel, ValidationError - - MyInt = Annotated[int, AfterValidator(lambda v: v + 1)] - - class Model(BaseModel): - a: MyInt - - print(Model(a=1).a) - #> 2 - - try: - Model(a='a') - except ValidationError as e: - print(e.json(indent=2)) - ''' - [ - { - "type": "int_parsing", - "loc": [ - "a" - ], - "msg": "Input should be a valid integer, unable to parse string as an integer", - "input": "a", - "url": "https://errors.pydantic.dev/2/v/int_parsing" - } - ] - ''' - ``` - """ - - func: core_schema.NoInfoValidatorFunction | core_schema.WithInfoValidatorFunction - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source_type) - info_arg = _inspect_validator(self.func, mode='after', type='field') - if info_arg: - func = cast(core_schema.WithInfoValidatorFunction, self.func) - return core_schema.with_info_after_validator_function(func, schema=schema) - else: - func = cast(core_schema.NoInfoValidatorFunction, self.func) - return core_schema.no_info_after_validator_function(func, schema=schema) - - @classmethod - def _from_decorator(cls, decorator: _decorators.Decorator[_decorators.FieldValidatorDecoratorInfo]) -> Self: - return cls(func=decorator.func) - - -@dataclasses.dataclass(frozen=True, **_internal_dataclass.slots_true) -class BeforeValidator: - """!!! abstract "Usage Documentation" - [field *before* validators](../concepts/validators.md#field-before-validator) - - A metadata class that indicates that a validation should be applied **before** the inner validation logic. - - Attributes: - func: The validator function. - json_schema_input_type: The input type used to generate the appropriate - JSON Schema (in validation mode). The actual input type is `Any`. - - Example: - ```python - from typing import Annotated - - from pydantic import BaseModel, BeforeValidator - - MyInt = Annotated[int, BeforeValidator(lambda v: v + 1)] - - class Model(BaseModel): - a: MyInt - - print(Model(a=1).a) - #> 2 - - try: - Model(a='a') - except TypeError as e: - print(e) - #> can only concatenate str (not "int") to str - ``` - """ - - func: core_schema.NoInfoValidatorFunction | core_schema.WithInfoValidatorFunction - json_schema_input_type: Any = PydanticUndefined - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source_type) - input_schema = ( - None - if self.json_schema_input_type is PydanticUndefined - else handler.generate_schema(self.json_schema_input_type) - ) - - info_arg = _inspect_validator(self.func, mode='before', type='field') - if info_arg: - func = cast(core_schema.WithInfoValidatorFunction, self.func) - return core_schema.with_info_before_validator_function( - func, - schema=schema, - json_schema_input_schema=input_schema, - ) - else: - func = cast(core_schema.NoInfoValidatorFunction, self.func) - return core_schema.no_info_before_validator_function( - func, schema=schema, json_schema_input_schema=input_schema - ) - - @classmethod - def _from_decorator(cls, decorator: _decorators.Decorator[_decorators.FieldValidatorDecoratorInfo]) -> Self: - return cls( - func=decorator.func, - json_schema_input_type=decorator.info.json_schema_input_type, - ) - - -@dataclasses.dataclass(frozen=True, **_internal_dataclass.slots_true) -class PlainValidator: - """!!! abstract "Usage Documentation" - [field *plain* validators](../concepts/validators.md#field-plain-validator) - - A metadata class that indicates that a validation should be applied **instead** of the inner validation logic. - - !!! note - Before v2.9, `PlainValidator` wasn't always compatible with JSON Schema generation for `mode='validation'`. - You can now use the `json_schema_input_type` argument to specify the input type of the function - to be used in the JSON schema when `mode='validation'` (the default). See the example below for more details. - - Attributes: - func: The validator function. - json_schema_input_type: The input type used to generate the appropriate - JSON Schema (in validation mode). The actual input type is `Any`. - - Example: - ```python - from typing import Annotated, Union - - from pydantic import BaseModel, PlainValidator - - def validate(v: object) -> int: - if not isinstance(v, (int, str)): - raise ValueError(f'Expected int or str, got {type(v)}') - - return int(v) + 1 - - MyInt = Annotated[ - int, - PlainValidator(validate, json_schema_input_type=Union[str, int]), # (1)! - ] - - class Model(BaseModel): - a: MyInt - - print(Model(a='1').a) - #> 2 - - print(Model(a=1).a) - #> 2 - ``` - - 1. In this example, we've specified the `json_schema_input_type` as `Union[str, int]` which indicates to the JSON schema - generator that in validation mode, the input type for the `a` field can be either a [`str`][] or an [`int`][]. - """ - - func: core_schema.NoInfoValidatorFunction | core_schema.WithInfoValidatorFunction - json_schema_input_type: Any = Any - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - # Note that for some valid uses of PlainValidator, it is not possible to generate a core schema for the - # source_type, so calling `handler(source_type)` will error, which prevents us from generating a proper - # serialization schema. To work around this for use cases that will not involve serialization, we simply - # catch any PydanticSchemaGenerationError that may be raised while attempting to build the serialization schema - # and abort any attempts to handle special serialization. - from pydantic import PydanticSchemaGenerationError - - try: - schema = handler(source_type) - # TODO if `schema['serialization']` is one of `'include-exclude-dict/sequence', - # schema validation will fail. That's why we use 'type ignore' comments below. - serialization = schema.get( - 'serialization', - core_schema.wrap_serializer_function_ser_schema( - function=lambda v, h: h(v), - schema=schema, - return_schema=handler.generate_schema(source_type), - ), - ) - except PydanticSchemaGenerationError: - serialization = None - - input_schema = handler.generate_schema(self.json_schema_input_type) - - info_arg = _inspect_validator(self.func, mode='plain', type='field') - if info_arg: - func = cast(core_schema.WithInfoValidatorFunction, self.func) - return core_schema.with_info_plain_validator_function( - func, - serialization=serialization, # pyright: ignore[reportArgumentType] - json_schema_input_schema=input_schema, - ) - else: - func = cast(core_schema.NoInfoValidatorFunction, self.func) - return core_schema.no_info_plain_validator_function( - func, - serialization=serialization, # pyright: ignore[reportArgumentType] - json_schema_input_schema=input_schema, - ) - - @classmethod - def _from_decorator(cls, decorator: _decorators.Decorator[_decorators.FieldValidatorDecoratorInfo]) -> Self: - return cls( - func=decorator.func, - json_schema_input_type=decorator.info.json_schema_input_type, - ) - - -@dataclasses.dataclass(frozen=True, **_internal_dataclass.slots_true) -class WrapValidator: - """!!! abstract "Usage Documentation" - [field *wrap* validators](../concepts/validators.md#field-wrap-validator) - - A metadata class that indicates that a validation should be applied **around** the inner validation logic. - - Attributes: - func: The validator function. - json_schema_input_type: The input type used to generate the appropriate - JSON Schema (in validation mode). The actual input type is `Any`. - - ```python - from datetime import datetime - from typing import Annotated - - from pydantic import BaseModel, ValidationError, WrapValidator - - def validate_timestamp(v, handler): - if v == 'now': - # we don't want to bother with further validation, just return the new value - return datetime.now() - try: - return handler(v) - except ValidationError: - # validation failed, in this case we want to return a default value - return datetime(2000, 1, 1) - - MyTimestamp = Annotated[datetime, WrapValidator(validate_timestamp)] - - class Model(BaseModel): - a: MyTimestamp - - print(Model(a='now').a) - #> 2032-01-02 03:04:05.000006 - print(Model(a='invalid').a) - #> 2000-01-01 00:00:00 - ``` - """ - - func: core_schema.NoInfoWrapValidatorFunction | core_schema.WithInfoWrapValidatorFunction - json_schema_input_type: Any = PydanticUndefined - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source_type) - input_schema = ( - None - if self.json_schema_input_type is PydanticUndefined - else handler.generate_schema(self.json_schema_input_type) - ) - - info_arg = _inspect_validator(self.func, mode='wrap', type='field') - if info_arg: - func = cast(core_schema.WithInfoWrapValidatorFunction, self.func) - return core_schema.with_info_wrap_validator_function( - func, - schema=schema, - json_schema_input_schema=input_schema, - ) - else: - func = cast(core_schema.NoInfoWrapValidatorFunction, self.func) - return core_schema.no_info_wrap_validator_function( - func, - schema=schema, - json_schema_input_schema=input_schema, - ) - - @classmethod - def _from_decorator(cls, decorator: _decorators.Decorator[_decorators.FieldValidatorDecoratorInfo]) -> Self: - return cls( - func=decorator.func, - json_schema_input_type=decorator.info.json_schema_input_type, - ) - - -if TYPE_CHECKING: - - class _OnlyValueValidatorClsMethod(Protocol): - def __call__(self, cls: Any, value: Any, /) -> Any: ... - - class _V2ValidatorClsMethod(Protocol): - def __call__(self, cls: Any, value: Any, info: core_schema.ValidationInfo[Any], /) -> Any: ... - - class _OnlyValueWrapValidatorClsMethod(Protocol): - def __call__(self, cls: Any, value: Any, handler: core_schema.ValidatorFunctionWrapHandler, /) -> Any: ... - - class _V2WrapValidatorClsMethod(Protocol): - def __call__( - self, - cls: Any, - value: Any, - handler: core_schema.ValidatorFunctionWrapHandler, - info: core_schema.ValidationInfo[Any], - /, - ) -> Any: ... - - _V2Validator = Union[ - _V2ValidatorClsMethod, - core_schema.WithInfoValidatorFunction, - _OnlyValueValidatorClsMethod, - core_schema.NoInfoValidatorFunction, - ] - - _V2WrapValidator = Union[ - _V2WrapValidatorClsMethod, - core_schema.WithInfoWrapValidatorFunction, - _OnlyValueWrapValidatorClsMethod, - core_schema.NoInfoWrapValidatorFunction, - ] - - _PartialClsOrStaticMethod: TypeAlias = Union[classmethod[Any, Any, Any], staticmethod[Any, Any], partialmethod[Any]] - - _V2BeforeAfterOrPlainValidatorType = TypeVar( - '_V2BeforeAfterOrPlainValidatorType', - bound=Union[_V2Validator, _PartialClsOrStaticMethod], - ) - _V2WrapValidatorType = TypeVar('_V2WrapValidatorType', bound=Union[_V2WrapValidator, _PartialClsOrStaticMethod]) - -FieldValidatorModes: TypeAlias = Literal['before', 'after', 'wrap', 'plain'] - - -@overload -def field_validator( - field: str, - /, - *fields: str, - mode: Literal['wrap'], - check_fields: bool | None = ..., - json_schema_input_type: Any = ..., -) -> Callable[[_V2WrapValidatorType], _V2WrapValidatorType]: ... - - -@overload -def field_validator( - field: str, - /, - *fields: str, - mode: Literal['before', 'plain'], - check_fields: bool | None = ..., - json_schema_input_type: Any = ..., -) -> Callable[[_V2BeforeAfterOrPlainValidatorType], _V2BeforeAfterOrPlainValidatorType]: ... - - -@overload -def field_validator( - field: str, - /, - *fields: str, - mode: Literal['after'] = ..., - check_fields: bool | None = ..., -) -> Callable[[_V2BeforeAfterOrPlainValidatorType], _V2BeforeAfterOrPlainValidatorType]: ... - - -def field_validator( # noqa: D417 - field: str, - /, - *fields: str, - mode: FieldValidatorModes = 'after', - check_fields: bool | None = None, - json_schema_input_type: Any = PydanticUndefined, -) -> Callable[[Any], Any]: - """!!! abstract "Usage Documentation" - [field validators](../concepts/validators.md#field-validators) - - Decorate methods on the class indicating that they should be used to validate fields. - - Example usage: - ```python - from typing import Any - - from pydantic import ( - BaseModel, - ValidationError, - field_validator, - ) - - class Model(BaseModel): - a: str - - @field_validator('a') - @classmethod - def ensure_foobar(cls, v: Any): - if 'foobar' not in v: - raise ValueError('"foobar" not found in a') - return v - - print(repr(Model(a='this is foobar good'))) - #> Model(a='this is foobar good') - - try: - Model(a='snap') - except ValidationError as exc_info: - print(exc_info) - ''' - 1 validation error for Model - a - Value error, "foobar" not found in a [type=value_error, input_value='snap', input_type=str] - ''' - ``` - - For more in depth examples, see [Field Validators](../concepts/validators.md#field-validators). - - Args: - *fields: The field names the validator should apply to. - mode: Specifies whether to validate the fields before or after validation. - check_fields: Whether to check that the fields actually exist on the model. - json_schema_input_type: The input type of the function. This is only used to generate - the appropriate JSON Schema (in validation mode) and can only specified - when `mode` is either `'before'`, `'plain'` or `'wrap'`. - - Raises: - PydanticUserError: - - If the decorator is used without any arguments (at least one field name must be provided). - - If the provided field names are not strings. - - If `json_schema_input_type` is provided with an unsupported `mode`. - - If the decorator is applied to an instance method. - """ - if callable(field) or isinstance(field, classmethod): - raise PydanticUserError( - 'The `@field_validator` decorator cannot be used without arguments, at least one field must be provided. ' - "For example: `@field_validator('', ...)`.", - code='decorator-missing-arguments', - ) - - if mode not in ('before', 'plain', 'wrap') and json_schema_input_type is not PydanticUndefined: - raise PydanticUserError( - f"`json_schema_input_type` can't be used when mode is set to {mode!r}", - code='validator-input-type', - ) - - if json_schema_input_type is PydanticUndefined and mode == 'plain': - json_schema_input_type = Any - - fields = field, *fields - if not all(isinstance(field, str) for field in fields): - raise PydanticUserError( - 'The provided field names to the `@field_validator` decorator should be strings. ' - "For example: `@field_validator('', '', ...).`", - code='decorator-invalid-fields', - ) - - def dec( - f: Callable[..., Any] | staticmethod[Any, Any] | classmethod[Any, Any, Any], - ) -> _decorators.PydanticDescriptorProxy[Any]: - if _decorators.is_instance_method_from_sig(f): - raise PydanticUserError( - 'The `@field_validator` decorator cannot be applied to instance methods', - code='validator-instance-method', - ) - - # auto apply the @classmethod decorator - f = _decorators.ensure_classmethod_based_on_signature(f) - - dec_info = _decorators.FieldValidatorDecoratorInfo( - fields=fields, mode=mode, check_fields=check_fields, json_schema_input_type=json_schema_input_type - ) - return _decorators.PydanticDescriptorProxy(f, dec_info) - - return dec - - -_ModelType = TypeVar('_ModelType') -_ModelTypeCo = TypeVar('_ModelTypeCo', covariant=True) - - -class ModelWrapValidatorHandler(core_schema.ValidatorFunctionWrapHandler, Protocol[_ModelTypeCo]): - """`@model_validator` decorated function handler argument type. This is used when `mode='wrap'`.""" - - def __call__( # noqa: D102 - self, - value: Any, - outer_location: str | int | None = None, - /, - ) -> _ModelTypeCo: # pragma: no cover - ... - - -class ModelWrapValidatorWithoutInfo(Protocol[_ModelType]): - """A `@model_validator` decorated function signature. - This is used when `mode='wrap'` and the function does not have info argument. - """ - - def __call__( # noqa: D102 - self, - cls: type[_ModelType], - # this can be a dict, a model instance - # or anything else that gets passed to validate_python - # thus validators _must_ handle all cases - value: Any, - handler: ModelWrapValidatorHandler[_ModelType], - /, - ) -> _ModelType: ... - - -class ModelWrapValidator(Protocol[_ModelType]): - """A `@model_validator` decorated function signature. This is used when `mode='wrap'`.""" - - def __call__( # noqa: D102 - self, - cls: type[_ModelType], - # this can be a dict, a model instance - # or anything else that gets passed to validate_python - # thus validators _must_ handle all cases - value: Any, - handler: ModelWrapValidatorHandler[_ModelType], - info: core_schema.ValidationInfo, - /, - ) -> _ModelType: ... - - -class FreeModelBeforeValidatorWithoutInfo(Protocol): - """A `@model_validator` decorated function signature. - This is used when `mode='before'` and the function does not have info argument. - """ - - def __call__( # noqa: D102 - self, - # this can be a dict, a model instance - # or anything else that gets passed to validate_python - # thus validators _must_ handle all cases - value: Any, - /, - ) -> Any: ... - - -class ModelBeforeValidatorWithoutInfo(Protocol): - """A `@model_validator` decorated function signature. - This is used when `mode='before'` and the function does not have info argument. - """ - - def __call__( # noqa: D102 - self, - cls: Any, - # this can be a dict, a model instance - # or anything else that gets passed to validate_python - # thus validators _must_ handle all cases - value: Any, - /, - ) -> Any: ... - - -class FreeModelBeforeValidator(Protocol): - """A `@model_validator` decorated function signature. This is used when `mode='before'`.""" - - def __call__( # noqa: D102 - self, - # this can be a dict, a model instance - # or anything else that gets passed to validate_python - # thus validators _must_ handle all cases - value: Any, - info: core_schema.ValidationInfo[Any], - /, - ) -> Any: ... - - -class ModelBeforeValidator(Protocol): - """A `@model_validator` decorated function signature. This is used when `mode='before'`.""" - - def __call__( # noqa: D102 - self, - cls: Any, - # this can be a dict, a model instance - # or anything else that gets passed to validate_python - # thus validators _must_ handle all cases - value: Any, - info: core_schema.ValidationInfo[Any], - /, - ) -> Any: ... - - -ModelAfterValidatorWithoutInfo = Callable[[_ModelType], _ModelType] -"""A `@model_validator` decorated function signature. This is used when `mode='after'` and the function does not -have info argument. -""" - -ModelAfterValidator = Callable[[_ModelType, core_schema.ValidationInfo[Any]], _ModelType] -"""A `@model_validator` decorated function signature. This is used when `mode='after'`.""" - -_AnyModelWrapValidator = Union[ModelWrapValidator[_ModelType], ModelWrapValidatorWithoutInfo[_ModelType]] -_AnyModelBeforeValidator = Union[ - FreeModelBeforeValidator, ModelBeforeValidator, FreeModelBeforeValidatorWithoutInfo, ModelBeforeValidatorWithoutInfo -] -_AnyModelAfterValidator = Union[ModelAfterValidator[_ModelType], ModelAfterValidatorWithoutInfo[_ModelType]] - - -@overload -def model_validator( - *, - mode: Literal['wrap'], -) -> Callable[ - [_AnyModelWrapValidator[_ModelType]], _decorators.PydanticDescriptorProxy[_decorators.ModelValidatorDecoratorInfo] -]: ... - - -@overload -def model_validator( - *, - mode: Literal['before'], -) -> Callable[ - [_AnyModelBeforeValidator], _decorators.PydanticDescriptorProxy[_decorators.ModelValidatorDecoratorInfo] -]: ... - - -@overload -def model_validator( - *, - mode: Literal['after'], -) -> Callable[ - [_AnyModelAfterValidator[_ModelType]], _decorators.PydanticDescriptorProxy[_decorators.ModelValidatorDecoratorInfo] -]: ... - - -def model_validator( - *, - mode: Literal['wrap', 'before', 'after'], -) -> Any: - """!!! abstract "Usage Documentation" - [Model Validators](../concepts/validators.md#model-validators) - - Decorate model methods for validation purposes. - - Example usage: - ```python - from typing_extensions import Self - - from pydantic import BaseModel, ValidationError, model_validator - - class Square(BaseModel): - width: float - height: float - - @model_validator(mode='after') - def verify_square(self) -> Self: - if self.width != self.height: - raise ValueError('width and height do not match') - return self - - s = Square(width=1, height=1) - print(repr(s)) - #> Square(width=1.0, height=1.0) - - try: - Square(width=1, height=2) - except ValidationError as e: - print(e) - ''' - 1 validation error for Square - Value error, width and height do not match [type=value_error, input_value={'width': 1, 'height': 2}, input_type=dict] - ''' - ``` - - For more in depth examples, see [Model Validators](../concepts/validators.md#model-validators). - - Args: - mode: A required string literal that specifies the validation mode. - It can be one of the following: 'wrap', 'before', or 'after'. - - Returns: - A decorator that can be used to decorate a function to be used as a model validator. - """ - - def dec(f: Any) -> _decorators.PydanticDescriptorProxy[Any]: - # auto apply the @classmethod decorator. NOTE: in V3, do not apply the conversion for 'after' validators: - f = _decorators.ensure_classmethod_based_on_signature(f) - if mode == 'after' and isinstance(f, classmethod): - warnings.warn( - category=PydanticDeprecatedSince212, - message=( - "Using `@model_validator` with mode='after' on a classmethod is deprecated. Instead, use an instance method. " - f'See the documentation at https://docs.pydantic.dev/{version_short()}/concepts/validators/#model-after-validator.' - ), - stacklevel=2, - ) - - dec_info = _decorators.ModelValidatorDecoratorInfo(mode=mode) - return _decorators.PydanticDescriptorProxy(f, dec_info) - - return dec - - -AnyType = TypeVar('AnyType') - - -if TYPE_CHECKING: - # If we add configurable attributes to IsInstance, we'd probably need to stop hiding it from type checkers like this - InstanceOf = Annotated[AnyType, ...] # `IsInstance[Sequence]` will be recognized by type checkers as `Sequence` - -else: - - @dataclasses.dataclass(**_internal_dataclass.slots_true) - class InstanceOf: - '''Generic type for annotating a type that is an instance of a given class. - - Example: - ```python - from pydantic import BaseModel, InstanceOf - - class Foo: - ... - - class Bar(BaseModel): - foo: InstanceOf[Foo] - - Bar(foo=Foo()) - try: - Bar(foo=42) - except ValidationError as e: - print(e) - """ - [ - │ { - │ │ 'type': 'is_instance_of', - │ │ 'loc': ('foo',), - │ │ 'msg': 'Input should be an instance of Foo', - │ │ 'input': 42, - │ │ 'ctx': {'class': 'Foo'}, - │ │ 'url': 'https://errors.pydantic.dev/0.38.0/v/is_instance_of' - │ } - ] - """ - ``` - ''' - - @classmethod - def __class_getitem__(cls, item: AnyType) -> AnyType: - return Annotated[item, cls()] - - @classmethod - def __get_pydantic_core_schema__(cls, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - from pydantic._internal._generate_schema import GENERATE_SCHEMA_ERRORS - - # use the generic _origin_ as the second argument to isinstance when appropriate - instance_of_schema = core_schema.is_instance_schema(_generics.get_origin(source) or source) - - try: - # Try to generate the "standard" schema, which will be used when loading from JSON - original_schema = handler(source) - except GENERATE_SCHEMA_ERRORS: - # If that fails, just produce a schema that can validate from python - return instance_of_schema - else: - # Use the "original" approach to serialization - instance_of_schema['serialization'] = core_schema.wrap_serializer_function_ser_schema( - function=lambda v, h: h(v), schema=original_schema - ) - return core_schema.json_or_python_schema(python_schema=instance_of_schema, json_schema=original_schema) - - __hash__ = object.__hash__ - - -if TYPE_CHECKING: - SkipValidation = Annotated[AnyType, ...] # SkipValidation[list[str]] will be treated by type checkers as list[str] -else: - - @dataclasses.dataclass(**_internal_dataclass.slots_true) - class SkipValidation: - """If this is applied as an annotation (e.g., via `x: Annotated[int, SkipValidation]`), validation will be - skipped. You can also use `SkipValidation[int]` as a shorthand for `Annotated[int, SkipValidation]`. - - This can be useful if you want to use a type annotation for documentation/IDE/type-checking purposes, - and know that it is safe to skip validation for one or more of the fields. - - Because this converts the validation schema to `any_schema`, subsequent annotation-applied transformations - may not have the expected effects. Therefore, when used, this annotation should generally be the final - annotation applied to a type. - """ - - def __class_getitem__(cls, item: Any) -> Any: - return Annotated[item, SkipValidation()] - - @classmethod - def __get_pydantic_core_schema__(cls, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - with warnings.catch_warnings(): - warnings.simplefilter('ignore', ArbitraryTypeWarning) - original_schema = handler(source) - metadata = {'pydantic_js_annotation_functions': [lambda _c, h: h(original_schema)]} - return core_schema.any_schema( - metadata=metadata, - serialization=core_schema.wrap_serializer_function_ser_schema( - function=lambda v, h: h(v), schema=original_schema - ), - ) - - __hash__ = object.__hash__ - - -_FromTypeT = TypeVar('_FromTypeT') - - -class ValidateAs: - """A helper class to validate a custom type from a type that is natively supported by Pydantic. - - Args: - from_type: The type natively supported by Pydantic to use to perform validation. - instantiation_hook: A callable taking the validated type as an argument, and returning - the populated custom type. - - Example: - ```python {lint="skip"} - from typing import Annotated - - from pydantic import BaseModel, TypeAdapter, ValidateAs - - class MyCls: - def __init__(self, a: int) -> None: - self.a = a - - def __repr__(self) -> str: - return f"MyCls(a={self.a})" - - class Model(BaseModel): - a: int - - - ta = TypeAdapter( - Annotated[MyCls, ValidateAs(Model, lambda v: MyCls(a=v.a))] - ) - - print(ta.validate_python({'a': 1})) - #> MyCls(a=1) - ``` - """ - - # TODO: make use of PEP 747 - def __init__(self, from_type: type[_FromTypeT], /, instantiation_hook: Callable[[_FromTypeT], Any]) -> None: - self.from_type = from_type - self.instantiation_hook = instantiation_hook - - def __get_pydantic_core_schema__(self, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(self.from_type) - return core_schema.no_info_after_validator_function( - self.instantiation_hook, - schema=schema, - ) diff --git a/functions/python/data-processor/pydantic/generics.py b/functions/python/data-processor/pydantic/generics.py deleted file mode 100644 index 3f1070d08..000000000 --- a/functions/python/data-processor/pydantic/generics.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `generics` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/json.py b/functions/python/data-processor/pydantic/json.py deleted file mode 100644 index bcaff9f57..000000000 --- a/functions/python/data-processor/pydantic/json.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `json` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/json_schema.py b/functions/python/data-processor/pydantic/json_schema.py deleted file mode 100644 index b9b21fd52..000000000 --- a/functions/python/data-processor/pydantic/json_schema.py +++ /dev/null @@ -1,2911 +0,0 @@ -"""!!! abstract "Usage Documentation" - [JSON Schema](../concepts/json_schema.md) - -The `json_schema` module contains classes and functions to allow the way [JSON Schema](https://json-schema.org/) -is generated to be customized. - -In general you shouldn't need to use this module directly; instead, you can use -[`BaseModel.model_json_schema`][pydantic.BaseModel.model_json_schema] and -[`TypeAdapter.json_schema`][pydantic.TypeAdapter.json_schema]. -""" - -from __future__ import annotations as _annotations - -import collections.abc -import dataclasses -import inspect -import math -import os -import re -import warnings -from collections import Counter, defaultdict -from collections.abc import Hashable, Iterable, Sequence -from copy import deepcopy -from enum import Enum -from re import Pattern -from typing import ( - TYPE_CHECKING, - Annotated, - Any, - Callable, - Literal, - NewType, - TypeVar, - Union, - cast, - overload, -) - -import pydantic_core -from pydantic_core import MISSING, CoreSchema, PydanticOmit, core_schema, to_jsonable_python -from pydantic_core.core_schema import ComputedField -from typing_extensions import TypeAlias, assert_never, deprecated, final -from typing_inspection.introspection import get_literal_values - -from pydantic.warnings import PydanticDeprecatedSince26, PydanticDeprecatedSince29 - -from ._internal import ( - _config, - _core_metadata, - _core_utils, - _decorators, - _internal_dataclass, - _mock_val_ser, - _schema_generation_shared, - _typing_extra, -) -from .annotated_handlers import GetJsonSchemaHandler -from .config import JsonDict, JsonValue -from .errors import PydanticInvalidForJsonSchema, PydanticSchemaGenerationError, PydanticUserError - -if TYPE_CHECKING: - from . import ConfigDict - from ._internal._core_utils import CoreSchemaField, CoreSchemaOrField - from ._internal._dataclasses import PydanticDataclass - from ._internal._schema_generation_shared import GetJsonSchemaFunction - from .main import BaseModel - - -CoreSchemaOrFieldType = Literal[core_schema.CoreSchemaType, core_schema.CoreSchemaFieldType] -""" -A type alias for defined schema types that represents a union of -`core_schema.CoreSchemaType` and -`core_schema.CoreSchemaFieldType`. -""" - -JsonSchemaValue = dict[str, Any] -""" -A type alias for a JSON schema value. This is a dictionary of string keys to arbitrary JSON values. -""" - -JsonSchemaMode = Literal['validation', 'serialization'] -""" -A type alias that represents the mode of a JSON schema; either 'validation' or 'serialization'. - -For some types, the inputs to validation differ from the outputs of serialization. For example, -computed fields will only be present when serializing, and should not be provided when -validating. This flag provides a way to indicate whether you want the JSON schema required -for validation inputs, or that will be matched by serialization outputs. -""" - -_MODE_TITLE_MAPPING: dict[JsonSchemaMode, str] = {'validation': 'Input', 'serialization': 'Output'} - - -JsonSchemaWarningKind = Literal['skipped-choice', 'non-serializable-default', 'skipped-discriminator'] -""" -A type alias representing the kinds of warnings that can be emitted during JSON schema generation. - -See [`GenerateJsonSchema.render_warning_message`][pydantic.json_schema.GenerateJsonSchema.render_warning_message] -for more details. -""" - - -class PydanticJsonSchemaWarning(UserWarning): - """This class is used to emit warnings produced during JSON schema generation. - See the [`GenerateJsonSchema.emit_warning`][pydantic.json_schema.GenerateJsonSchema.emit_warning] and - [`GenerateJsonSchema.render_warning_message`][pydantic.json_schema.GenerateJsonSchema.render_warning_message] - methods for more details; these can be overridden to control warning behavior. - """ - - -NoDefault = object() -"""A sentinel value used to indicate that no default value should be used when generating a JSON Schema -for a core schema with a default value. -""" - - -# ##### JSON Schema Generation ##### -DEFAULT_REF_TEMPLATE = '#/$defs/{model}' -"""The default format string used to generate reference names.""" - -# There are three types of references relevant to building JSON schemas: -# 1. core_schema "ref" values; these are not exposed as part of the JSON schema -# * these might look like the fully qualified path of a model, its id, or something similar -CoreRef = NewType('CoreRef', str) -# 2. keys of the "definitions" object that will eventually go into the JSON schema -# * by default, these look like "MyModel", though may change in the presence of collisions -# * eventually, we may want to make it easier to modify the way these names are generated -DefsRef = NewType('DefsRef', str) -# 3. the values corresponding to the "$ref" key in the schema -# * By default, these look like "#/$defs/MyModel", as in {"$ref": "#/$defs/MyModel"} -JsonRef = NewType('JsonRef', str) - -CoreModeRef = tuple[CoreRef, JsonSchemaMode] -JsonSchemaKeyT = TypeVar('JsonSchemaKeyT', bound=Hashable) - -_PRIMITIVE_JSON_SCHEMA_TYPES = ('string', 'boolean', 'null', 'integer', 'number') - - -@dataclasses.dataclass(**_internal_dataclass.slots_true) -class _DefinitionsRemapping: - defs_remapping: dict[DefsRef, DefsRef] - json_remapping: dict[JsonRef, JsonRef] - - @staticmethod - def from_prioritized_choices( - prioritized_choices: dict[DefsRef, list[DefsRef]], - defs_to_json: dict[DefsRef, JsonRef], - definitions: dict[DefsRef, JsonSchemaValue], - ) -> _DefinitionsRemapping: - """ - This function should produce a remapping that replaces complex DefsRef with the simpler ones from the - prioritized_choices such that applying the name remapping would result in an equivalent JSON schema. - """ - # We need to iteratively simplify the definitions until we reach a fixed point. - # The reason for this is that outer definitions may reference inner definitions that get simplified - # into an equivalent reference, and the outer definitions won't be equivalent until we've simplified - # the inner definitions. - copied_definitions = deepcopy(definitions) - definitions_schema = {'$defs': copied_definitions} - for _iter in range(100): # prevent an infinite loop in the case of a bug, 100 iterations should be enough - # For every possible remapped DefsRef, collect all schemas that DefsRef might be used for: - schemas_for_alternatives: dict[DefsRef, list[JsonSchemaValue]] = defaultdict(list) - for defs_ref in copied_definitions: - alternatives = prioritized_choices[defs_ref] - for alternative in alternatives: - schemas_for_alternatives[alternative].append(copied_definitions[defs_ref]) - - # Deduplicate the schemas for each alternative; the idea is that we only want to remap to a new DefsRef - # if it introduces no ambiguity, i.e., there is only one distinct schema for that DefsRef. - for defs_ref in schemas_for_alternatives: - schemas_for_alternatives[defs_ref] = _deduplicate_schemas(schemas_for_alternatives[defs_ref]) - - # Build the remapping - defs_remapping: dict[DefsRef, DefsRef] = {} - json_remapping: dict[JsonRef, JsonRef] = {} - for original_defs_ref in definitions: - alternatives = prioritized_choices[original_defs_ref] - # Pick the first alternative that has only one schema, since that means there is no collision - remapped_defs_ref = next(x for x in alternatives if len(schemas_for_alternatives[x]) == 1) - defs_remapping[original_defs_ref] = remapped_defs_ref - - # Map all alternatives after the remapped one to the remapped one - # This ensures that intermediate simplifications are also remapped - remapped_index = alternatives.index(remapped_defs_ref) - for alt in alternatives[remapped_index:]: - json_remapping[defs_to_json[alt]] = defs_to_json[remapped_defs_ref] - remapping = _DefinitionsRemapping(defs_remapping, json_remapping) - new_definitions_schema = remapping.remap_json_schema({'$defs': copied_definitions}) - if definitions_schema == new_definitions_schema: - # We've reached the fixed point - return remapping - definitions_schema = new_definitions_schema - - raise PydanticInvalidForJsonSchema('Failed to simplify the JSON schema definitions') - - def remap_defs_ref(self, ref: DefsRef) -> DefsRef: - return self.defs_remapping.get(ref, ref) - - def remap_json_ref(self, ref: JsonRef) -> JsonRef: - return self.json_remapping.get(ref, ref) - - def remap_json_schema(self, schema: Any) -> Any: - """ - Recursively update the JSON schema replacing all $refs - """ - if isinstance(schema, str): - # Note: this may not really be a JsonRef; we rely on having no collisions between JsonRefs and other strings - return self.remap_json_ref(JsonRef(schema)) - elif isinstance(schema, list): - return [self.remap_json_schema(item) for item in schema] - elif isinstance(schema, dict): - for key, value in schema.items(): - if key == '$ref' and isinstance(value, str): - schema['$ref'] = self.remap_json_ref(JsonRef(value)) - elif key == '$defs': - schema['$defs'] = { - self.remap_defs_ref(DefsRef(key)): self.remap_json_schema(value) - for key, value in schema['$defs'].items() - } - else: - schema[key] = self.remap_json_schema(value) - return schema - - -class GenerateJsonSchema: - """!!! abstract "Usage Documentation" - [Customizing the JSON Schema Generation Process](../concepts/json_schema.md#customizing-the-json-schema-generation-process) - - A class for generating JSON schemas. - - This class generates JSON schemas based on configured parameters. The default schema dialect - is [https://json-schema.org/draft/2020-12/schema](https://json-schema.org/draft/2020-12/schema). - The class uses `by_alias` to configure how fields with - multiple names are handled and `ref_template` to format reference names. - - Attributes: - schema_dialect: The JSON schema dialect used to generate the schema. See - [Declaring a Dialect](https://json-schema.org/understanding-json-schema/reference/schema.html#id4) - in the JSON Schema documentation for more information about dialects. - ignored_warning_kinds: Warnings to ignore when generating the schema. `self.render_warning_message` will - do nothing if its argument `kind` is in `ignored_warning_kinds`; - this value can be modified on subclasses to easily control which warnings are emitted. - by_alias: Whether to use field aliases when generating the schema. - ref_template: The format string used when generating reference names. - core_to_json_refs: A mapping of core refs to JSON refs. - core_to_defs_refs: A mapping of core refs to definition refs. - defs_to_core_refs: A mapping of definition refs to core refs. - json_to_defs_refs: A mapping of JSON refs to definition refs. - definitions: Definitions in the schema. - - Args: - by_alias: Whether to use field aliases in the generated schemas. - ref_template: The format string to use when generating reference names. - union_format: The format to use when combining schemas from unions together. Can be one of: - - - `'any_of'`: Use the [`anyOf`](https://json-schema.org/understanding-json-schema/reference/combining#anyOf) - keyword to combine schemas (the default). - - `'primitive_type_array'`: Use the [`type`](https://json-schema.org/understanding-json-schema/reference/type) - keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive - type (`string`, `boolean`, `null`, `integer` or `number`) or contains constraints/metadata, falls back to - `any_of`. - - Raises: - JsonSchemaError: If the instance of the class is inadvertently reused after generating a schema. - """ - - schema_dialect = 'https://json-schema.org/draft/2020-12/schema' - - # `self.render_warning_message` will do nothing if its argument `kind` is in `ignored_warning_kinds`; - # this value can be modified on subclasses to easily control which warnings are emitted - ignored_warning_kinds: set[JsonSchemaWarningKind] = {'skipped-choice'} - - def __init__( - self, - by_alias: bool = True, - ref_template: str = DEFAULT_REF_TEMPLATE, - union_format: Literal['any_of', 'primitive_type_array'] = 'any_of', - ) -> None: - self.by_alias = by_alias - self.ref_template = ref_template - self.union_format: Literal['any_of', 'primitive_type_array'] = union_format - - self.core_to_json_refs: dict[CoreModeRef, JsonRef] = {} - self.core_to_defs_refs: dict[CoreModeRef, DefsRef] = {} - self.defs_to_core_refs: dict[DefsRef, CoreModeRef] = {} - self.json_to_defs_refs: dict[JsonRef, DefsRef] = {} - - self.definitions: dict[DefsRef, JsonSchemaValue] = {} - self._config_wrapper_stack = _config.ConfigWrapperStack(_config.ConfigWrapper({})) - - self._mode: JsonSchemaMode = 'validation' - - # The following includes a mapping of a fully-unique defs ref choice to a list of preferred - # alternatives, which are generally simpler, such as only including the class name. - # At the end of schema generation, we use these to produce a JSON schema with more human-readable - # definitions, which would also work better in a generated OpenAPI client, etc. - self._prioritized_defsref_choices: dict[DefsRef, list[DefsRef]] = {} - self._collision_counter: dict[str, int] = defaultdict(int) - self._collision_index: dict[str, int] = {} - - self._schema_type_to_method = self.build_schema_type_to_method() - - # When we encounter definitions we need to try to build them immediately - # so that they are available schemas that reference them - # But it's possible that CoreSchema was never going to be used - # (e.g. because the CoreSchema that references short circuits is JSON schema generation without needing - # the reference) so instead of failing altogether if we can't build a definition we - # store the error raised and re-throw it if we end up needing that def - self._core_defs_invalid_for_json_schema: dict[DefsRef, PydanticInvalidForJsonSchema] = {} - - # This changes to True after generating a schema, to prevent issues caused by accidental reuse - # of a single instance of a schema generator - self._used = False - - @property - def _config(self) -> _config.ConfigWrapper: - return self._config_wrapper_stack.tail - - @property - def mode(self) -> JsonSchemaMode: - if self._config.json_schema_mode_override is not None: - return self._config.json_schema_mode_override - else: - return self._mode - - def build_schema_type_to_method( - self, - ) -> dict[CoreSchemaOrFieldType, Callable[[CoreSchemaOrField], JsonSchemaValue]]: - """Builds a dictionary mapping fields to methods for generating JSON schemas. - - Returns: - A dictionary containing the mapping of `CoreSchemaOrFieldType` to a handler method. - - Raises: - TypeError: If no method has been defined for generating a JSON schema for a given pydantic core schema type. - """ - mapping: dict[CoreSchemaOrFieldType, Callable[[CoreSchemaOrField], JsonSchemaValue]] = {} - core_schema_types: list[CoreSchemaOrFieldType] = list(get_literal_values(CoreSchemaOrFieldType)) - for key in core_schema_types: - method_name = f'{key.replace("-", "_")}_schema' - try: - mapping[key] = getattr(self, method_name) - except AttributeError as e: # pragma: no cover - if os.getenv('PYDANTIC_PRIVATE_ALLOW_UNHANDLED_SCHEMA_TYPES'): - continue - raise TypeError( - f'No method for generating JsonSchema for core_schema.type={key!r} ' - f'(expected: {type(self).__name__}.{method_name})' - ) from e - return mapping - - def generate_definitions( - self, inputs: Sequence[tuple[JsonSchemaKeyT, JsonSchemaMode, core_schema.CoreSchema]] - ) -> tuple[dict[tuple[JsonSchemaKeyT, JsonSchemaMode], JsonSchemaValue], dict[DefsRef, JsonSchemaValue]]: - """Generates JSON schema definitions from a list of core schemas, pairing the generated definitions with a - mapping that links the input keys to the definition references. - - Args: - inputs: A sequence of tuples, where: - - - The first element is a JSON schema key type. - - The second element is the JSON mode: either 'validation' or 'serialization'. - - The third element is a core schema. - - Returns: - A tuple where: - - - The first element is a dictionary whose keys are tuples of JSON schema key type and JSON mode, and - whose values are the JSON schema corresponding to that pair of inputs. (These schemas may have - JsonRef references to definitions that are defined in the second returned element.) - - The second element is a dictionary whose keys are definition references for the JSON schemas - from the first returned element, and whose values are the actual JSON schema definitions. - - Raises: - PydanticUserError: Raised if the JSON schema generator has already been used to generate a JSON schema. - """ - if self._used: - raise PydanticUserError( - 'This JSON schema generator has already been used to generate a JSON schema. ' - f'You must create a new instance of {type(self).__name__} to generate a new JSON schema.', - code='json-schema-already-used', - ) - - for _, mode, schema in inputs: - self._mode = mode - self.generate_inner(schema) - - definitions_remapping = self._build_definitions_remapping() - - json_schemas_map: dict[tuple[JsonSchemaKeyT, JsonSchemaMode], DefsRef] = {} - for key, mode, schema in inputs: - self._mode = mode - json_schema = self.generate_inner(schema) - json_schemas_map[(key, mode)] = definitions_remapping.remap_json_schema(json_schema) - - json_schema = {'$defs': self.definitions} - json_schema = definitions_remapping.remap_json_schema(json_schema) - self._used = True - return json_schemas_map, self.sort(json_schema['$defs']) # type: ignore - - def generate(self, schema: CoreSchema, mode: JsonSchemaMode = 'validation') -> JsonSchemaValue: - """Generates a JSON schema for a specified schema in a specified mode. - - Args: - schema: A Pydantic model. - mode: The mode in which to generate the schema. Defaults to 'validation'. - - Returns: - A JSON schema representing the specified schema. - - Raises: - PydanticUserError: If the JSON schema generator has already been used to generate a JSON schema. - """ - self._mode = mode - if self._used: - raise PydanticUserError( - 'This JSON schema generator has already been used to generate a JSON schema. ' - f'You must create a new instance of {type(self).__name__} to generate a new JSON schema.', - code='json-schema-already-used', - ) - - json_schema: JsonSchemaValue = self.generate_inner(schema) - json_ref_counts = self.get_json_ref_counts(json_schema) - - ref = cast(JsonRef, json_schema.get('$ref')) - while ref is not None: # may need to unpack multiple levels - ref_json_schema = self.get_schema_from_definitions(ref) - if json_ref_counts[ref] == 1 and ref_json_schema is not None and len(json_schema) == 1: - # "Unpack" the ref since this is the only reference and there are no sibling keys - json_schema = ref_json_schema.copy() # copy to prevent recursive dict reference - json_ref_counts[ref] -= 1 - ref = cast(JsonRef, json_schema.get('$ref')) - ref = None - - self._garbage_collect_definitions(json_schema) - definitions_remapping = self._build_definitions_remapping() - - if self.definitions: - json_schema['$defs'] = self.definitions - - json_schema = definitions_remapping.remap_json_schema(json_schema) - - # For now, we will not set the $schema key. However, if desired, this can be easily added by overriding - # this method and adding the following line after a call to super().generate(schema): - # json_schema['$schema'] = self.schema_dialect - - self._used = True - return self.sort(json_schema) - - def generate_inner(self, schema: CoreSchemaOrField) -> JsonSchemaValue: # noqa: C901 - """Generates a JSON schema for a given core schema. - - Args: - schema: The given core schema. - - Returns: - The generated JSON schema. - - TODO: the nested function definitions here seem like bad practice, I'd like to unpack these - in a future PR. It'd be great if we could shorten the call stack a bit for JSON schema generation, - and I think there's potential for that here. - """ - # If a schema with the same CoreRef has been handled, just return a reference to it - # Note that this assumes that it will _never_ be the case that the same CoreRef is used - # on types that should have different JSON schemas - if 'ref' in schema: - core_ref = CoreRef(schema['ref']) # type: ignore[typeddict-item] - core_mode_ref = (core_ref, self.mode) - if core_mode_ref in self.core_to_defs_refs and self.core_to_defs_refs[core_mode_ref] in self.definitions: - return {'$ref': self.core_to_json_refs[core_mode_ref]} - - def populate_defs(core_schema: CoreSchema, json_schema: JsonSchemaValue) -> JsonSchemaValue: - if 'ref' in core_schema: - core_ref = CoreRef(core_schema['ref']) # type: ignore[typeddict-item] - defs_ref, ref_json_schema = self.get_cache_defs_ref_schema(core_ref) - json_ref = JsonRef(ref_json_schema['$ref']) - # Replace the schema if it's not a reference to itself - # What we want to avoid is having the def be just a ref to itself - # which is what would happen if we blindly assigned any - if json_schema.get('$ref', None) != json_ref: - self.definitions[defs_ref] = json_schema - self._core_defs_invalid_for_json_schema.pop(defs_ref, None) - json_schema = ref_json_schema - return json_schema - - def handler_func(schema_or_field: CoreSchemaOrField) -> JsonSchemaValue: - """Generate a JSON schema based on the input schema. - - Args: - schema_or_field: The core schema to generate a JSON schema from. - - Returns: - The generated JSON schema. - - Raises: - TypeError: If an unexpected schema type is encountered. - """ - # Generate the core-schema-type-specific bits of the schema generation: - json_schema: JsonSchemaValue | None = None - if self.mode == 'serialization' and 'serialization' in schema_or_field: - # In this case, we skip the JSON Schema generation of the schema - # and use the `'serialization'` schema instead (canonical example: - # `Annotated[int, PlainSerializer(str)]`). - ser_schema = schema_or_field['serialization'] # type: ignore - json_schema = self.ser_schema(ser_schema) - - # It might be that the 'serialization'` is skipped depending on `when_used`. - # This is only relevant for `nullable` schemas though, so we special case here. - if ( - json_schema is not None - and ser_schema.get('when_used') in ('unless-none', 'json-unless-none') - and schema_or_field['type'] == 'nullable' - ): - json_schema = self.get_union_of_schemas([{'type': 'null'}, json_schema]) - if json_schema is None: - if _core_utils.is_core_schema(schema_or_field) or _core_utils.is_core_schema_field(schema_or_field): - generate_for_schema_type = self._schema_type_to_method[schema_or_field['type']] - json_schema = generate_for_schema_type(schema_or_field) - else: - raise TypeError(f'Unexpected schema type: schema={schema_or_field}') - return json_schema - - current_handler = _schema_generation_shared.GenerateJsonSchemaHandler(self, handler_func) - - metadata = cast(_core_metadata.CoreMetadata, schema.get('metadata', {})) - - # TODO: I dislike that we have to wrap these basic dict updates in callables, is there any way around this? - - if js_updates := metadata.get('pydantic_js_updates'): - - def js_updates_handler_func( - schema_or_field: CoreSchemaOrField, - current_handler: GetJsonSchemaHandler = current_handler, - ) -> JsonSchemaValue: - json_schema = {**current_handler(schema_or_field), **js_updates} - return json_schema - - current_handler = _schema_generation_shared.GenerateJsonSchemaHandler(self, js_updates_handler_func) - - if js_extra := metadata.get('pydantic_js_extra'): - - def js_extra_handler_func( - schema_or_field: CoreSchemaOrField, - current_handler: GetJsonSchemaHandler = current_handler, - ) -> JsonSchemaValue: - json_schema = current_handler(schema_or_field) - if isinstance(js_extra, dict): - json_schema.update(to_jsonable_python(js_extra)) - elif callable(js_extra): - # similar to typing issue in _update_class_schema when we're working with callable js extra - js_extra(json_schema) # type: ignore - return json_schema - - current_handler = _schema_generation_shared.GenerateJsonSchemaHandler(self, js_extra_handler_func) - - for js_modify_function in metadata.get('pydantic_js_functions', ()): - - def new_handler_func( - schema_or_field: CoreSchemaOrField, - current_handler: GetJsonSchemaHandler = current_handler, - js_modify_function: GetJsonSchemaFunction = js_modify_function, - ) -> JsonSchemaValue: - json_schema = js_modify_function(schema_or_field, current_handler) - if _core_utils.is_core_schema(schema_or_field): - json_schema = populate_defs(schema_or_field, json_schema) - original_schema = current_handler.resolve_ref_schema(json_schema) - ref = json_schema.pop('$ref', None) - if ref and json_schema: - original_schema.update(json_schema) - return original_schema - - current_handler = _schema_generation_shared.GenerateJsonSchemaHandler(self, new_handler_func) - - for js_modify_function in metadata.get('pydantic_js_annotation_functions', ()): - - def new_handler_func( - schema_or_field: CoreSchemaOrField, - current_handler: GetJsonSchemaHandler = current_handler, - js_modify_function: GetJsonSchemaFunction = js_modify_function, - ) -> JsonSchemaValue: - return js_modify_function(schema_or_field, current_handler) - - current_handler = _schema_generation_shared.GenerateJsonSchemaHandler(self, new_handler_func) - - json_schema = current_handler(schema) - if _core_utils.is_core_schema(schema): - json_schema = populate_defs(schema, json_schema) - return json_schema - - def sort(self, value: JsonSchemaValue, parent_key: str | None = None) -> JsonSchemaValue: - """Override this method to customize the sorting of the JSON schema (e.g., don't sort at all, sort all keys unconditionally, etc.) - - By default, alphabetically sort the keys in the JSON schema, skipping the 'properties' and 'default' keys to preserve field definition order. - This sort is recursive, so it will sort all nested dictionaries as well. - """ - sorted_dict: dict[str, JsonSchemaValue] = {} - keys = value.keys() - if parent_key not in ('properties', 'default'): - keys = sorted(keys) - for key in keys: - sorted_dict[key] = self._sort_recursive(value[key], parent_key=key) - return sorted_dict - - def _sort_recursive(self, value: Any, parent_key: str | None = None) -> Any: - """Recursively sort a JSON schema value.""" - if isinstance(value, dict): - sorted_dict: dict[str, JsonSchemaValue] = {} - keys = value.keys() - if parent_key not in ('properties', 'default'): - keys = sorted(keys) - for key in keys: - sorted_dict[key] = self._sort_recursive(value[key], parent_key=key) - return sorted_dict - elif isinstance(value, list): - sorted_list: list[JsonSchemaValue] = [self._sort_recursive(item, parent_key) for item in value] - return sorted_list - else: - return value - - # ### Schema generation methods - - def invalid_schema(self, schema: core_schema.InvalidSchema) -> JsonSchemaValue: - """Placeholder - should never be called.""" - - raise RuntimeError('Cannot generate schema for invalid_schema. This is a bug! Please report it.') - - def any_schema(self, schema: core_schema.AnySchema) -> JsonSchemaValue: - """Generates a JSON schema that matches any value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {} - - def none_schema(self, schema: core_schema.NoneSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches `None`. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'null'} - - def bool_schema(self, schema: core_schema.BoolSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a bool value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'boolean'} - - def int_schema(self, schema: core_schema.IntSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches an int value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema: dict[str, Any] = {'type': 'integer'} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.numeric) - json_schema = {k: v for k, v in json_schema.items() if v not in {math.inf, -math.inf}} - return json_schema - - def float_schema(self, schema: core_schema.FloatSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a float value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema: dict[str, Any] = {'type': 'number'} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.numeric) - json_schema = {k: v for k, v in json_schema.items() if v not in {math.inf, -math.inf}} - return json_schema - - def decimal_schema(self, schema: core_schema.DecimalSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a decimal value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - - def get_decimal_pattern(schema: core_schema.DecimalSchema) -> str: - max_digits = schema.get('max_digits') - decimal_places = schema.get('decimal_places') - - pattern = ( - r'^(?!^[-+.]*$)[+-]?0*' # check it is not empty string and not one or sequence of ".+-" characters. - ) - - # Case 1: Both max_digits and decimal_places are set - if max_digits is not None and decimal_places is not None: - integer_places = max(0, max_digits - decimal_places) - pattern += ( - rf'(?:' - rf'\d{{0,{integer_places}}}' - rf'|' - rf'(?=[\d.]{{1,{max_digits + 1}}}0*$)' - rf'\d{{0,{integer_places}}}\.\d{{0,{decimal_places}}}0*$' - rf')' - ) - - # Case 2: Only max_digits is set - elif max_digits is not None and decimal_places is None: - pattern += ( - rf'(?:' - rf'\d{{0,{max_digits}}}' - rf'|' - rf'(?=[\d.]{{1,{max_digits + 1}}}0*$)' - rf'\d*\.\d*0*$' - rf')' - ) - - # Case 3: Only decimal_places is set - elif max_digits is None and decimal_places is not None: - pattern += rf'\d*\.?\d{{0,{decimal_places}}}0*$' - - # Case 4: Both are None (no restrictions) - else: - pattern += r'\d*\.?\d*$' # look for arbitrary integer or decimal - - return pattern - - json_schema = self.str_schema(core_schema.str_schema(pattern=get_decimal_pattern(schema))) - if self.mode == 'validation': - multiple_of = schema.get('multiple_of') - le = schema.get('le') - ge = schema.get('ge') - lt = schema.get('lt') - gt = schema.get('gt') - json_schema = { - 'anyOf': [ - self.float_schema( - core_schema.float_schema( - allow_inf_nan=schema.get('allow_inf_nan'), - multiple_of=None if multiple_of is None else float(multiple_of), - le=None if le is None else float(le), - ge=None if ge is None else float(ge), - lt=None if lt is None else float(lt), - gt=None if gt is None else float(gt), - ) - ), - json_schema, - ], - } - return json_schema - - def str_schema(self, schema: core_schema.StringSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a string value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema = {'type': 'string'} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.string) - if isinstance(json_schema.get('pattern'), Pattern): - # TODO: should we add regex flags to the pattern? - json_schema['pattern'] = json_schema.get('pattern').pattern # type: ignore - return json_schema - - def bytes_schema(self, schema: core_schema.BytesSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a bytes value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema = {'type': 'string', 'format': 'base64url' if self._config.ser_json_bytes == 'base64' else 'binary'} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.bytes) - return json_schema - - def date_schema(self, schema: core_schema.DateSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a date value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'string', 'format': 'date'} - - def time_schema(self, schema: core_schema.TimeSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a time value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'string', 'format': 'time'} - - def datetime_schema(self, schema: core_schema.DatetimeSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a datetime value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'string', 'format': 'date-time'} - - def timedelta_schema(self, schema: core_schema.TimedeltaSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a timedelta value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - if self._config.ser_json_timedelta == 'float': - return {'type': 'number'} - return {'type': 'string', 'format': 'duration'} - - def literal_schema(self, schema: core_schema.LiteralSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a literal value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - expected = [to_jsonable_python(v.value if isinstance(v, Enum) else v) for v in schema['expected']] - - result: dict[str, Any] = {} - if len(expected) == 1: - result['const'] = expected[0] - else: - result['enum'] = expected - - types = {type(e) for e in expected} - if types == {str}: - result['type'] = 'string' - elif types == {int}: - result['type'] = 'integer' - elif types == {float}: - result['type'] = 'number' - elif types == {bool}: - result['type'] = 'boolean' - elif types == {list}: - result['type'] = 'array' - elif types == {type(None)}: - result['type'] = 'null' - return result - - def missing_sentinel_schema(self, schema: core_schema.MissingSentinelSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches the `MISSING` sentinel value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - raise PydanticOmit - - def enum_schema(self, schema: core_schema.EnumSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches an Enum value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - enum_type = schema['cls'] - description = None if not enum_type.__doc__ else inspect.cleandoc(enum_type.__doc__) - if ( - description == 'An enumeration.' - ): # This is the default value provided by enum.EnumMeta.__new__; don't use it - description = None - result: dict[str, Any] = {'title': enum_type.__name__, 'description': description} - result = {k: v for k, v in result.items() if v is not None} - - expected = [to_jsonable_python(v.value) for v in schema['members']] - - result['enum'] = expected - - types = {type(e) for e in expected} - if isinstance(enum_type, str) or types == {str}: - result['type'] = 'string' - elif isinstance(enum_type, int) or types == {int}: - result['type'] = 'integer' - elif isinstance(enum_type, float) or types == {float}: - result['type'] = 'number' - elif types == {bool}: - result['type'] = 'boolean' - elif types == {list}: - result['type'] = 'array' - - return result - - def is_instance_schema(self, schema: core_schema.IsInstanceSchema) -> JsonSchemaValue: - """Handles JSON schema generation for a core schema that checks if a value is an instance of a class. - - Unless overridden in a subclass, this raises an error. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.handle_invalid_for_json_schema(schema, f'core_schema.IsInstanceSchema ({schema["cls"]})') - - def is_subclass_schema(self, schema: core_schema.IsSubclassSchema) -> JsonSchemaValue: - """Handles JSON schema generation for a core schema that checks if a value is a subclass of a class. - - For backwards compatibility with v1, this does not raise an error, but can be overridden to change this. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - # Note: This is for compatibility with V1; you can override if you want different behavior. - return {} - - def callable_schema(self, schema: core_schema.CallableSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a callable value. - - Unless overridden in a subclass, this raises an error. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.handle_invalid_for_json_schema(schema, 'core_schema.CallableSchema') - - def list_schema(self, schema: core_schema.ListSchema) -> JsonSchemaValue: - """Returns a schema that matches a list schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - items_schema = {} if 'items_schema' not in schema else self.generate_inner(schema['items_schema']) - json_schema = {'type': 'array', 'items': items_schema} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.array) - return json_schema - - @deprecated('`tuple_positional_schema` is deprecated. Use `tuple_schema` instead.', category=None) - @final - def tuple_positional_schema(self, schema: core_schema.TupleSchema) -> JsonSchemaValue: - """Replaced by `tuple_schema`.""" - warnings.warn( - '`tuple_positional_schema` is deprecated. Use `tuple_schema` instead.', - PydanticDeprecatedSince26, - stacklevel=2, - ) - return self.tuple_schema(schema) - - @deprecated('`tuple_variable_schema` is deprecated. Use `tuple_schema` instead.', category=None) - @final - def tuple_variable_schema(self, schema: core_schema.TupleSchema) -> JsonSchemaValue: - """Replaced by `tuple_schema`.""" - warnings.warn( - '`tuple_variable_schema` is deprecated. Use `tuple_schema` instead.', - PydanticDeprecatedSince26, - stacklevel=2, - ) - return self.tuple_schema(schema) - - def tuple_schema(self, schema: core_schema.TupleSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a tuple schema e.g. `tuple[int, - str, bool]` or `tuple[int, ...]`. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema: JsonSchemaValue = {'type': 'array'} - if 'variadic_item_index' in schema: - variadic_item_index = schema['variadic_item_index'] - if variadic_item_index > 0: - json_schema['minItems'] = variadic_item_index - json_schema['prefixItems'] = [ - self.generate_inner(item) for item in schema['items_schema'][:variadic_item_index] - ] - if variadic_item_index + 1 == len(schema['items_schema']): - # if the variadic item is the last item, then represent it faithfully - json_schema['items'] = self.generate_inner(schema['items_schema'][variadic_item_index]) - else: - # otherwise, 'items' represents the schema for the variadic - # item plus the suffix, so just allow anything for simplicity - # for now - json_schema['items'] = True - else: - prefixItems = [self.generate_inner(item) for item in schema['items_schema']] - if prefixItems: - json_schema['prefixItems'] = prefixItems - json_schema['minItems'] = len(prefixItems) - json_schema['maxItems'] = len(prefixItems) - self.update_with_validations(json_schema, schema, self.ValidationsMapping.array) - return json_schema - - def set_schema(self, schema: core_schema.SetSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a set schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self._common_set_schema(schema) - - def frozenset_schema(self, schema: core_schema.FrozenSetSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a frozenset schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self._common_set_schema(schema) - - def _common_set_schema(self, schema: core_schema.SetSchema | core_schema.FrozenSetSchema) -> JsonSchemaValue: - items_schema = {} if 'items_schema' not in schema else self.generate_inner(schema['items_schema']) - json_schema = {'type': 'array', 'uniqueItems': True, 'items': items_schema} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.array) - return json_schema - - def generator_schema(self, schema: core_schema.GeneratorSchema) -> JsonSchemaValue: - """Returns a JSON schema that represents the provided GeneratorSchema. - - Args: - schema: The schema. - - Returns: - The generated JSON schema. - """ - items_schema = {} if 'items_schema' not in schema else self.generate_inner(schema['items_schema']) - json_schema = {'type': 'array', 'items': items_schema} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.array) - return json_schema - - def dict_schema(self, schema: core_schema.DictSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a dict schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema: JsonSchemaValue = {'type': 'object'} - - keys_schema = self.generate_inner(schema['keys_schema']).copy() if 'keys_schema' in schema else {} - if '$ref' not in keys_schema: - keys_pattern = keys_schema.pop('pattern', None) - # Don't give a title to patternProperties/propertyNames: - keys_schema.pop('title', None) - else: - # Here, we assume that if the keys schema is a definition reference, - # it can't be a simple string core schema (and thus no pattern can exist). - # However, this is only in practice (in theory, a definition reference core - # schema could be generated for a simple string schema). - # Note that we avoid calling `self.resolve_ref_schema`, as it might not exist yet. - keys_pattern = None - - values_schema = self.generate_inner(schema['values_schema']).copy() if 'values_schema' in schema else {} - # don't give a title to additionalProperties: - values_schema.pop('title', None) - - if values_schema or keys_pattern is not None: - if keys_pattern is None: - json_schema['additionalProperties'] = values_schema - else: - json_schema['patternProperties'] = {keys_pattern: values_schema} - else: # for `dict[str, Any]`, we allow any key and any value, since `str` is the default key type - json_schema['additionalProperties'] = True - - if ( - # The len check indicates that constraints are probably present: - (keys_schema.get('type') == 'string' and len(keys_schema) > 1) - # If this is a definition reference schema, it most likely has constraints: - or '$ref' in keys_schema - ): - keys_schema.pop('type', None) - json_schema['propertyNames'] = keys_schema - - self.update_with_validations(json_schema, schema, self.ValidationsMapping.object) - return json_schema - - def function_before_schema(self, schema: core_schema.BeforeValidatorFunctionSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a function-before schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - if self.mode == 'validation' and (input_schema := schema.get('json_schema_input_schema')): - return self.generate_inner(input_schema) - - return self.generate_inner(schema['schema']) - - def function_after_schema(self, schema: core_schema.AfterValidatorFunctionSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a function-after schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['schema']) - - def function_plain_schema(self, schema: core_schema.PlainValidatorFunctionSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a function-plain schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - if self.mode == 'validation' and (input_schema := schema.get('json_schema_input_schema')): - return self.generate_inner(input_schema) - - return self.handle_invalid_for_json_schema( - schema, f'core_schema.PlainValidatorFunctionSchema ({schema["function"]})' - ) - - def function_wrap_schema(self, schema: core_schema.WrapValidatorFunctionSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a function-wrap schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - if self.mode == 'validation' and (input_schema := schema.get('json_schema_input_schema')): - return self.generate_inner(input_schema) - - return self.generate_inner(schema['schema']) - - def default_schema(self, schema: core_schema.WithDefaultSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema with a default value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema = self.generate_inner(schema['schema']) - - default = self.get_default_value(schema) - if default is NoDefault or default is MISSING: - return json_schema - - # we reflect the application of custom plain, no-info serializers to defaults for - # JSON Schemas viewed in serialization mode: - # TODO: improvements along with https://github.com/pydantic/pydantic/issues/8208 - if self.mode == 'serialization': - # `_get_ser_schema_for_default_value()` is used to unpack potentially nested validator schemas: - ser_schema = _get_ser_schema_for_default_value(schema['schema']) - if ( - ser_schema is not None - and (ser_func := ser_schema.get('function')) - and not (default is None and ser_schema.get('when_used') in ('unless-none', 'json-unless-none')) - ): - try: - default = ser_func(default) # type: ignore - except Exception: - # It might be that the provided default needs to be validated (read: parsed) first - # (assuming `validate_default` is enabled). However, we can't perform - # such validation during JSON Schema generation so we don't support - # this pattern for now. - # (One example is when using `foo: ByteSize = '1MB'`, which validates and - # serializes as an int. In this case, `ser_func` is `int` and `int('1MB')` fails). - self.emit_warning( - 'non-serializable-default', - f'Unable to serialize value {default!r} with the plain serializer; excluding default from JSON schema', - ) - return json_schema - - # Sort set/frozenset defaults to ensure deterministic JSON schema generation - # We only sort if len > 1 because sets of size 0 or 1 are already deterministic - if isinstance(default, collections.abc.Set) and len(default) > 1: - try: - default = sorted(default) - except TypeError: # pragma: no cover - # If items aren't comparable (e.g. mixed types), we can't sort them. - pass - - try: - encoded_default = self.encode_default(default) - except pydantic_core.PydanticSerializationError: - self.emit_warning( - 'non-serializable-default', - f'Default value {default} is not JSON serializable; excluding default from JSON schema', - ) - # Return the inner schema, as though there was no default - return json_schema - - json_schema['default'] = encoded_default - return json_schema - - def get_default_value(self, schema: core_schema.WithDefaultSchema) -> Any: - """Get the default value to be used when generating a JSON Schema for a core schema with a default. - - The default implementation is to use the statically defined default value. This method can be overridden - if you want to make use of the default factory. - - Args: - schema: The `'with-default'` core schema. - - Returns: - The default value to use, or [`NoDefault`][pydantic.json_schema.NoDefault] if no default - value is available. - """ - return schema.get('default', NoDefault) - - def nullable_schema(self, schema: core_schema.NullableSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that allows null values. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - null_schema = {'type': 'null'} - inner_json_schema = self.generate_inner(schema['schema']) - - if inner_json_schema == null_schema: - return null_schema - else: - return self.get_union_of_schemas([inner_json_schema, null_schema]) - - def union_schema(self, schema: core_schema.UnionSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that allows values matching any of the given schemas. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - generated: list[JsonSchemaValue] = [] - - for choice in core_schema.iter_union_choices(schema): - try: - generated.append(self.generate_inner(choice)) - except PydanticOmit: # noqa: PERF203 - continue - except PydanticInvalidForJsonSchema as exc: - self.emit_warning('skipped-choice', exc.message) - if len(generated) == 1: - return generated[0] - return self.get_union_of_schemas(generated) - - def get_union_of_schemas(self, schemas: list[JsonSchemaValue]) -> JsonSchemaValue: - """Returns the JSON Schema representation for the union of the provided JSON Schemas. - - The result depends on the configured `'union_format'`. - - Args: - schemas: The list of JSON Schemas to be included in the union. - - Returns: - The JSON Schema representing the union of schemas. - """ - if self.union_format == 'primitive_type_array': - types: list[str] = [] - for schema in schemas: - schema_types: list[str] | str | None = schema.get('type') - if schema_types is None: - # No type, meaning it can be a ref or an empty schema. - break - if not isinstance(schema_types, list): - schema_types = [schema_types] - if not all(t in _PRIMITIVE_JSON_SCHEMA_TYPES for t in schema_types): - break - if len(schema) != 1: - # We only want to include types that don't have any constraints. For instance, - # if `schemas = [{'type': 'string', 'maxLength': 3}, {'type': 'string', 'minLength': 5}]`, - # we don't want to produce `{'type': 'string', 'maxLength': 3, 'minLength': 5}`. - # Same if we have some metadata (e.g. `title`) on a specific union member, we want to preserve it. - break - - types.extend(schema_types) - else: - # If we got there, all the schemas where valid to be used with the `'primitive_type_array` format - return {'type': list(dict.fromkeys(types))} - - return self.get_flattened_anyof(schemas) - - def tagged_union_schema(self, schema: core_schema.TaggedUnionSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that allows values matching any of the given schemas, where - the schemas are tagged with a discriminator field that indicates which schema should be used to validate - the value. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - generated: dict[str, JsonSchemaValue] = {} - for k, v in schema['choices'].items(): - if isinstance(k, Enum): - k = k.value - try: - # Use str(k) since keys must be strings for json; while not technically correct, - # it's the closest that can be represented in valid JSON - generated[str(k)] = self.generate_inner(v).copy() - except PydanticOmit: - continue - except PydanticInvalidForJsonSchema as exc: - self.emit_warning('skipped-choice', exc.message) - - one_of_choices = _deduplicate_schemas(generated.values()) - json_schema: JsonSchemaValue = {'oneOf': one_of_choices} - - # This reflects the v1 behavior; TODO: we should make it possible to exclude OpenAPI stuff from the JSON schema - openapi_discriminator = self._extract_discriminator(schema, one_of_choices) - if openapi_discriminator is not None: - json_schema['discriminator'] = { - 'propertyName': openapi_discriminator, - 'mapping': {k: v.get('$ref', v) for k, v in generated.items()}, - } - - return json_schema - - def _extract_discriminator( - self, schema: core_schema.TaggedUnionSchema, one_of_choices: list[JsonDict] - ) -> str | None: - """Extract a compatible OpenAPI discriminator from the schema and one_of choices that end up in the final - schema.""" - openapi_discriminator: str | None = None - - if isinstance(schema['discriminator'], str): - return schema['discriminator'] - - if isinstance(schema['discriminator'], list): - # If the discriminator is a single item list containing a string, that is equivalent to the string case - if len(schema['discriminator']) == 1 and isinstance(schema['discriminator'][0], str): - return schema['discriminator'][0] - # When an alias is used that is different from the field name, the discriminator will be a list of single - # str lists, one for the attribute and one for the actual alias. The logic here will work even if there is - # more than one possible attribute, and looks for whether a single alias choice is present as a documented - # property on all choices. If so, that property will be used as the OpenAPI discriminator. - for alias_path in schema['discriminator']: - if not isinstance(alias_path, list): - break # this means that the discriminator is not a list of alias paths - if len(alias_path) != 1: - continue # this means that the "alias" does not represent a single field - alias = alias_path[0] - if not isinstance(alias, str): - continue # this means that the "alias" does not represent a field - alias_is_present_on_all_choices = True - for choice in one_of_choices: - try: - choice = self.resolve_ref_schema(choice) - except RuntimeError as exc: - # TODO: fixme - this is a workaround for the fact that we can't always resolve refs - # for tagged union choices at this point in the schema gen process, we might need to do - # another pass at the end like we do for core schemas - self.emit_warning('skipped-discriminator', str(exc)) - choice = {} - properties = choice.get('properties', {}) - if not isinstance(properties, dict) or alias not in properties: - alias_is_present_on_all_choices = False - break - if alias_is_present_on_all_choices: - openapi_discriminator = alias - break - return openapi_discriminator - - def chain_schema(self, schema: core_schema.ChainSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a core_schema.ChainSchema. - - When generating a schema for validation, we return the validation JSON schema for the first step in the chain. - For serialization, we return the serialization JSON schema for the last step in the chain. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - step_index = 0 if self.mode == 'validation' else -1 # use first step for validation, last for serialization - return self.generate_inner(schema['steps'][step_index]) - - def lax_or_strict_schema(self, schema: core_schema.LaxOrStrictSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that allows values matching either the lax schema or the - strict schema. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - # TODO: Need to read the default value off of model config or whatever - use_strict = schema.get('strict', False) # TODO: replace this default False - # If your JSON schema fails to generate it is probably - # because one of the following two branches failed. - if use_strict: - return self.generate_inner(schema['strict_schema']) - else: - return self.generate_inner(schema['lax_schema']) - - def json_or_python_schema(self, schema: core_schema.JsonOrPythonSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that allows values matching either the JSON schema or the - Python schema. - - The JSON schema is used instead of the Python schema. If you want to use the Python schema, you should override - this method. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['json_schema']) - - def typed_dict_schema(self, schema: core_schema.TypedDictSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a typed dict. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - total = schema.get('total', True) - named_required_fields: list[tuple[str, bool, CoreSchemaField]] = [ - (name, self.field_is_required(field, total), field) - for name, field in schema['fields'].items() - if self.field_is_present(field) - ] - if self.mode == 'serialization': - named_required_fields.extend(self._name_required_computed_fields(schema.get('computed_fields', []))) - cls = schema.get('cls') - config = _get_typed_dict_config(cls) - with self._config_wrapper_stack.push(config): - json_schema = self._named_required_fields_schema(named_required_fields) - - # There's some duplication between `extra_behavior` and - # the config's `extra`/core config's `extra_fields_behavior`. - # However, it is common to manually create TypedDictSchemas, - # where you don't necessarily have a class. - # At runtime, `extra_behavior` takes priority over the config - # for validation, so follow the same for the JSON Schema: - if 'extras_schema' in schema and schema['extras_schema'] != core_schema.any_schema(): - allow_additional_props = self.generate_inner(schema['extras_schema']) - else: - allow_additional_props = True - - if schema.get('extra_behavior') == 'forbid': - json_schema['additionalProperties'] = False - elif schema.get('extra_behavior') == 'allow': - json_schema['additionalProperties'] = allow_additional_props - - if cls is not None: - # `_update_class_schema()` will not override - # `additionalProperties` if already present: - self._update_class_schema(json_schema, cls, config) - elif 'additionalProperties' not in json_schema: - extra = schema.get('config', {}).get('extra_fields_behavior') - if extra == 'forbid': - json_schema['additionalProperties'] = False - elif extra == 'allow': - json_schema['additionalProperties'] = allow_additional_props - - return json_schema - - @staticmethod - def _name_required_computed_fields( - computed_fields: list[ComputedField], - ) -> list[tuple[str, bool, core_schema.ComputedField]]: - return [(field['property_name'], True, field) for field in computed_fields] - - def _named_required_fields_schema( - self, named_required_fields: Sequence[tuple[str, bool, CoreSchemaField]] - ) -> JsonSchemaValue: - properties: dict[str, JsonSchemaValue] = {} - required_fields: list[str] = [] - for name, required, field in named_required_fields: - if self.by_alias: - name = self._get_alias_name(field, name) - try: - field_json_schema = self.generate_inner(field).copy() - except PydanticOmit: - continue - if 'title' not in field_json_schema and self.field_title_should_be_set(field): - title = self.get_title_from_name(name) - field_json_schema['title'] = title - field_json_schema = self.handle_ref_overrides(field_json_schema) - properties[name] = field_json_schema - if required: - required_fields.append(name) - - json_schema = {'type': 'object', 'properties': properties} - if required_fields: - json_schema['required'] = required_fields - return json_schema - - def _get_alias_name(self, field: CoreSchemaField, name: str) -> str: - if field['type'] == 'computed-field': - alias: Any = field.get('alias', name) - elif self.mode == 'validation': - alias = field.get('validation_alias', name) - else: - alias = field.get('serialization_alias', name) - if isinstance(alias, str): - name = alias - elif isinstance(alias, list): - alias = cast('list[str] | str', alias) - for path in alias: - if isinstance(path, list) and len(path) == 1 and isinstance(path[0], str): - # Use the first valid single-item string path; the code that constructs the alias array - # should ensure the first such item is what belongs in the JSON schema - name = path[0] - break - else: - assert_never(alias) - return name - - def typed_dict_field_schema(self, schema: core_schema.TypedDictField) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a typed dict field. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['schema']) - - def dataclass_field_schema(self, schema: core_schema.DataclassField) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a dataclass field. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['schema']) - - def model_field_schema(self, schema: core_schema.ModelField) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a model field. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['schema']) - - def computed_field_schema(self, schema: core_schema.ComputedField) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a computed field. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['return_schema']) - - def model_schema(self, schema: core_schema.ModelSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a model. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - # We do not use schema['model'].model_json_schema() here - # because it could lead to inconsistent refs handling, etc. - cls = cast('type[BaseModel]', schema['cls']) - config = cls.model_config - - with self._config_wrapper_stack.push(config): - json_schema = self.generate_inner(schema['schema']) - - self._update_class_schema(json_schema, cls, config) - - return json_schema - - def _update_class_schema(self, json_schema: JsonSchemaValue, cls: type[Any], config: ConfigDict) -> None: - """Update json_schema with the following, extracted from `config` and `cls`: - - * title - * description - * additional properties - * json_schema_extra - * deprecated - - Done in place, hence there's no return value as the original json_schema is mutated. - No ref resolving is involved here, as that's not appropriate for simple updates. - """ - from ._internal._dataclasses import is_stdlib_dataclass - from .main import BaseModel - - if (config_title := config.get('title')) is not None: - json_schema.setdefault('title', config_title) - elif model_title_generator := config.get('model_title_generator'): - title = model_title_generator(cls) - if not isinstance(title, str): - raise TypeError(f'model_title_generator {model_title_generator} must return str, not {title.__class__}') - json_schema.setdefault('title', title) - if 'title' not in json_schema: - json_schema['title'] = cls.__name__ - - # BaseModel and dataclasses; don't use cls.__doc__ as it will contain the verbose class signature by default - if cls is BaseModel: - docstring = None - elif is_stdlib_dataclass(cls): # For Pydantic dataclasses, we already handle this at class creation - # The `dataclass` module generates a `__doc__` based on the `inspect.signature()` - # result, which we don't want to use as a description. Such `__doc__` startswith - # `cls.__name__(`, which could lead to mistakenly discarding it if for some reason - # an explicitly set class docstring follows the same pattern, but this is unlikely - # to happen. - doc = cls.__doc__ - docstring = None if doc is None or doc.startswith(f'{cls.__name__}(') else doc - else: - docstring = cls.__doc__ - - if docstring: - json_schema.setdefault('description', inspect.cleandoc(docstring)) - - extra = config.get('extra') - if 'additionalProperties' not in json_schema: # This check is particularly important for `typed_dict_schema()` - if extra == 'allow': - json_schema['additionalProperties'] = True - elif extra == 'forbid': - json_schema['additionalProperties'] = False - - json_schema_extra = config.get('json_schema_extra') - if issubclass(cls, BaseModel) and cls.__pydantic_root_model__: - root_json_schema_extra = cls.model_fields['root'].json_schema_extra - if json_schema_extra and root_json_schema_extra: - raise ValueError( - '"model_config[\'json_schema_extra\']" and "Field.json_schema_extra" on "RootModel.root"' - ' field must not be set simultaneously' - ) - if root_json_schema_extra: - json_schema_extra = root_json_schema_extra - - if isinstance(json_schema_extra, (staticmethod, classmethod)): - # In older versions of python, this is necessary to ensure staticmethod/classmethods are callable - json_schema_extra = json_schema_extra.__get__(cls) - - if isinstance(json_schema_extra, dict): - json_schema.update(json_schema_extra) - elif callable(json_schema_extra): - if len(_typing_extra.signature_no_eval(json_schema_extra).parameters) > 1: - json_schema_extra = cast(Callable[[JsonDict, type[Any]], None], json_schema_extra) - json_schema_extra(json_schema, cls) - else: - json_schema_extra = cast(Callable[[JsonDict], None], json_schema_extra) - json_schema_extra(json_schema) - elif json_schema_extra is not None: - raise ValueError( - f"model_config['json_schema_extra']={json_schema_extra} should be a dict, callable, or None" - ) - - if hasattr(cls, '__deprecated__'): - json_schema['deprecated'] = True - - def resolve_ref_schema(self, json_schema: JsonSchemaValue) -> JsonSchemaValue: - """Resolve a JsonSchemaValue to the non-ref schema if it is a $ref schema. - - Args: - json_schema: The schema to resolve. - - Returns: - The resolved schema. - - Raises: - RuntimeError: If the schema reference can't be found in definitions. - """ - while '$ref' in json_schema: - ref = json_schema['$ref'] - schema_to_update = self.get_schema_from_definitions(JsonRef(ref)) - if schema_to_update is None: - raise RuntimeError(f'Cannot update undefined schema for $ref={ref}') - json_schema = schema_to_update - return json_schema - - def model_fields_schema(self, schema: core_schema.ModelFieldsSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a model's fields. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - named_required_fields: list[tuple[str, bool, CoreSchemaField]] = [ - (name, self.field_is_required(field, total=True), field) - for name, field in schema['fields'].items() - if self.field_is_present(field) - ] - if self.mode == 'serialization': - named_required_fields.extend(self._name_required_computed_fields(schema.get('computed_fields', []))) - json_schema = self._named_required_fields_schema(named_required_fields) - extras_schema = schema.get('extras_schema', None) - if extras_schema is not None: - schema_to_update = self.resolve_ref_schema(json_schema) - schema_to_update['additionalProperties'] = self.generate_inner(extras_schema) - return json_schema - - def field_is_present(self, field: CoreSchemaField) -> bool: - """Whether the field should be included in the generated JSON schema. - - Args: - field: The schema for the field itself. - - Returns: - `True` if the field should be included in the generated JSON schema, `False` otherwise. - """ - if self.mode == 'serialization': - # If you still want to include the field in the generated JSON schema, - # override this method and return True - return not field.get('serialization_exclude') - elif self.mode == 'validation': - return True - else: - assert_never(self.mode) - - def field_is_required( - self, - field: core_schema.ModelField | core_schema.DataclassField | core_schema.TypedDictField, - total: bool, - ) -> bool: - """Whether the field should be marked as required in the generated JSON schema. - (Note that this is irrelevant if the field is not present in the JSON schema.). - - Args: - field: The schema for the field itself. - total: Only applies to `TypedDictField`s. - Indicates if the `TypedDict` this field belongs to is total, in which case any fields that don't - explicitly specify `required=False` are required. - - Returns: - `True` if the field should be marked as required in the generated JSON schema, `False` otherwise. - """ - if field['type'] == 'typed-dict-field': - required = field.get('required', total) - else: - required = field['schema']['type'] != 'default' - - if self.mode == 'serialization': - has_exclude_if = field.get('serialization_exclude_if') is not None - if self._config.json_schema_serialization_defaults_required: - return not has_exclude_if - else: - return required and not has_exclude_if - else: - return required - - def dataclass_args_schema(self, schema: core_schema.DataclassArgsSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a dataclass's constructor arguments. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - named_required_fields: list[tuple[str, bool, CoreSchemaField]] = [ - (field['name'], self.field_is_required(field, total=True), field) - for field in schema['fields'] - if self.field_is_present(field) - ] - if self.mode == 'serialization': - named_required_fields.extend(self._name_required_computed_fields(schema.get('computed_fields', []))) - return self._named_required_fields_schema(named_required_fields) - - def dataclass_schema(self, schema: core_schema.DataclassSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a dataclass. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - - cls = schema['cls'] - config = cast('ConfigDict', getattr(cls, '__pydantic_config__', {})) - - with self._config_wrapper_stack.push(config): - json_schema = self.generate_inner(schema['schema']).copy() - - self._update_class_schema(json_schema, cls, config) - - return json_schema - - def arguments_schema(self, schema: core_schema.ArgumentsSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a function's arguments. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - prefer_positional = schema.get('metadata', {}).get('pydantic_js_prefer_positional_arguments') - - arguments = schema['arguments_schema'] - kw_only_arguments = [a for a in arguments if a.get('mode') == 'keyword_only'] - kw_or_p_arguments = [a for a in arguments if a.get('mode') in {'positional_or_keyword', None}] - p_only_arguments = [a for a in arguments if a.get('mode') == 'positional_only'] - var_args_schema = schema.get('var_args_schema') - var_kwargs_schema = schema.get('var_kwargs_schema') - - if prefer_positional: - positional_possible = not kw_only_arguments and not var_kwargs_schema - if positional_possible: - return self.p_arguments_schema(p_only_arguments + kw_or_p_arguments, var_args_schema) - - keyword_possible = not p_only_arguments and not var_args_schema - if keyword_possible: - return self.kw_arguments_schema(kw_or_p_arguments + kw_only_arguments, var_kwargs_schema) - - if not prefer_positional: - positional_possible = not kw_only_arguments and not var_kwargs_schema - if positional_possible: - return self.p_arguments_schema(p_only_arguments + kw_or_p_arguments, var_args_schema) - - raise PydanticInvalidForJsonSchema( - 'Unable to generate JSON schema for arguments validator with positional-only and keyword-only arguments' - ) - - def kw_arguments_schema( - self, arguments: list[core_schema.ArgumentsParameter], var_kwargs_schema: CoreSchema | None - ) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a function's keyword arguments. - - Args: - arguments: The core schema. - - Returns: - The generated JSON schema. - """ - properties: dict[str, JsonSchemaValue] = {} - required: list[str] = [] - for argument in arguments: - name = self.get_argument_name(argument) - argument_schema = self.generate_inner(argument['schema']).copy() - if 'title' not in argument_schema and self.field_title_should_be_set(argument['schema']): - argument_schema['title'] = self.get_title_from_name(name) - properties[name] = argument_schema - - if argument['schema']['type'] != 'default': - # This assumes that if the argument has a default value, - # the inner schema must be of type WithDefaultSchema. - # I believe this is true, but I am not 100% sure - required.append(name) - - json_schema: JsonSchemaValue = {'type': 'object', 'properties': properties} - if required: - json_schema['required'] = required - - if var_kwargs_schema: - additional_properties_schema = self.generate_inner(var_kwargs_schema) - if additional_properties_schema: - json_schema['additionalProperties'] = additional_properties_schema - else: - json_schema['additionalProperties'] = False - return json_schema - - def p_arguments_schema( - self, arguments: list[core_schema.ArgumentsParameter], var_args_schema: CoreSchema | None - ) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a function's positional arguments. - - Args: - arguments: The core schema. - - Returns: - The generated JSON schema. - """ - prefix_items: list[JsonSchemaValue] = [] - min_items = 0 - - for argument in arguments: - name = self.get_argument_name(argument) - - argument_schema = self.generate_inner(argument['schema']).copy() - if 'title' not in argument_schema and self.field_title_should_be_set(argument['schema']): - argument_schema['title'] = self.get_title_from_name(name) - prefix_items.append(argument_schema) - - if argument['schema']['type'] != 'default': - # This assumes that if the argument has a default value, - # the inner schema must be of type WithDefaultSchema. - # I believe this is true, but I am not 100% sure - min_items += 1 - - json_schema: JsonSchemaValue = {'type': 'array'} - if prefix_items: - json_schema['prefixItems'] = prefix_items - if min_items: - json_schema['minItems'] = min_items - - if var_args_schema: - items_schema = self.generate_inner(var_args_schema) - if items_schema: - json_schema['items'] = items_schema - else: - json_schema['maxItems'] = len(prefix_items) - - return json_schema - - def get_argument_name(self, argument: core_schema.ArgumentsParameter | core_schema.ArgumentsV3Parameter) -> str: - """Retrieves the name of an argument. - - Args: - argument: The core schema. - - Returns: - The name of the argument. - """ - name = argument['name'] - if self.by_alias: - alias = argument.get('alias') - if isinstance(alias, str): - name = alias - else: - pass # might want to do something else? - return name - - def arguments_v3_schema(self, schema: core_schema.ArgumentsV3Schema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a function's arguments. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - arguments = schema['arguments_schema'] - properties: dict[str, JsonSchemaValue] = {} - required: list[str] = [] - for argument in arguments: - mode = argument.get('mode', 'positional_or_keyword') - name = self.get_argument_name(argument) - argument_schema = self.generate_inner(argument['schema']).copy() - if mode == 'var_args': - argument_schema = {'type': 'array', 'items': argument_schema} - elif mode == 'var_kwargs_uniform': - argument_schema = {'type': 'object', 'additionalProperties': argument_schema} - - argument_schema.setdefault('title', self.get_title_from_name(name)) - properties[name] = argument_schema - - if ( - (mode == 'var_kwargs_unpacked_typed_dict' and 'required' in argument_schema) - or mode not in {'var_args', 'var_kwargs_uniform', 'var_kwargs_unpacked_typed_dict'} - and argument['schema']['type'] != 'default' - ): - # This assumes that if the argument has a default value, - # the inner schema must be of type WithDefaultSchema. - # I believe this is true, but I am not 100% sure - required.append(name) - - json_schema: JsonSchemaValue = {'type': 'object', 'properties': properties} - if required: - json_schema['required'] = required - return json_schema - - def call_schema(self, schema: core_schema.CallSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a function call. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['arguments_schema']) - - def custom_error_schema(self, schema: core_schema.CustomErrorSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a custom error. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return self.generate_inner(schema['schema']) - - def json_schema(self, schema: core_schema.JsonSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a JSON object. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - content_core_schema = schema.get('schema') or core_schema.any_schema() - content_json_schema = self.generate_inner(content_core_schema) - if self.mode == 'validation': - return {'type': 'string', 'contentMediaType': 'application/json', 'contentSchema': content_json_schema} - else: - # self.mode == 'serialization' - return content_json_schema - - def url_schema(self, schema: core_schema.UrlSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a URL. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - json_schema = {'type': 'string', 'format': 'uri', 'minLength': 1} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.string) - return json_schema - - def multi_host_url_schema(self, schema: core_schema.MultiHostUrlSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a URL that can be used with multiple hosts. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - # Note: 'multi-host-uri' is a custom/pydantic-specific format, not part of the JSON Schema spec - json_schema = {'type': 'string', 'format': 'multi-host-uri', 'minLength': 1} - self.update_with_validations(json_schema, schema, self.ValidationsMapping.string) - return json_schema - - def uuid_schema(self, schema: core_schema.UuidSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a UUID. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'string', 'format': 'uuid'} - - def definitions_schema(self, schema: core_schema.DefinitionsSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that defines a JSON object with definitions. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - for definition in schema['definitions']: - try: - self.generate_inner(definition) - except PydanticInvalidForJsonSchema as e: # noqa: PERF203 - core_ref: CoreRef = CoreRef(definition['ref']) # type: ignore - self._core_defs_invalid_for_json_schema[self.get_defs_ref((core_ref, self.mode))] = e - continue - return self.generate_inner(schema['schema']) - - def definition_ref_schema(self, schema: core_schema.DefinitionReferenceSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a schema that references a definition. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - core_ref = CoreRef(schema['schema_ref']) - _, ref_json_schema = self.get_cache_defs_ref_schema(core_ref) - return ref_json_schema - - def ser_schema( - self, schema: core_schema.SerSchema | core_schema.IncExSeqSerSchema | core_schema.IncExDictSerSchema - ) -> JsonSchemaValue | None: - """Generates a JSON schema that matches a schema that defines a serialized object. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - schema_type = schema['type'] - if schema_type == 'function-plain' or schema_type == 'function-wrap': - # PlainSerializerFunctionSerSchema or WrapSerializerFunctionSerSchema - return_schema = schema.get('return_schema') - if return_schema is not None: - return self.generate_inner(return_schema) - elif schema_type == 'format' or schema_type == 'to-string': - # FormatSerSchema or ToStringSerSchema - return self.str_schema(core_schema.str_schema()) - elif schema['type'] == 'model': - # ModelSerSchema - return self.generate_inner(schema['schema']) - return None - - def complex_schema(self, schema: core_schema.ComplexSchema) -> JsonSchemaValue: - """Generates a JSON schema that matches a complex number. - - JSON has no standard way to represent complex numbers. Complex number is not a numeric - type. Here we represent complex number as strings following the rule defined by Python. - For instance, '1+2j' is an accepted complex string. Details can be found in - [Python's `complex` documentation][complex]. - - Args: - schema: The core schema. - - Returns: - The generated JSON schema. - """ - return {'type': 'string'} - - # ### Utility methods - - def get_title_from_name(self, name: str) -> str: - """Retrieves a title from a name. - - Args: - name: The name to retrieve a title from. - - Returns: - The title. - """ - return name.title().replace('_', ' ').strip() - - def field_title_should_be_set(self, schema: CoreSchemaOrField) -> bool: - """Returns true if a field with the given schema should have a title set based on the field name. - - Intuitively, we want this to return true for schemas that wouldn't otherwise provide their own title - (e.g., int, float, str), and false for those that would (e.g., BaseModel subclasses). - - Args: - schema: The schema to check. - - Returns: - `True` if the field should have a title set, `False` otherwise. - """ - if _core_utils.is_core_schema_field(schema): - if schema['type'] == 'computed-field': - field_schema = schema['return_schema'] - else: - field_schema = schema['schema'] - return self.field_title_should_be_set(field_schema) - - elif _core_utils.is_core_schema(schema): - if schema.get('ref'): # things with refs, such as models and enums, should not have titles set - return False - if schema['type'] in {'default', 'nullable', 'definitions'}: - return self.field_title_should_be_set(schema['schema']) # type: ignore[typeddict-item] - if _core_utils.is_function_with_inner_schema(schema): - return self.field_title_should_be_set(schema['schema']) - if schema['type'] == 'definition-ref': - # Referenced schemas should not have titles set for the same reason - # schemas with refs should not - return False - return True # anything else should have title set - - else: - raise PydanticInvalidForJsonSchema(f'Unexpected schema type: schema={schema}') # pragma: no cover - - def normalize_name(self, name: str) -> str: - """Normalizes a name to be used as a key in a dictionary. - - Args: - name: The name to normalize. - - Returns: - The normalized name. - """ - return re.sub(r'[^a-zA-Z0-9.\-_]', '_', name).replace('.', '__') - - def get_defs_ref(self, core_mode_ref: CoreModeRef) -> DefsRef: - """Override this method to change the way that definitions keys are generated from a core reference. - - Args: - core_mode_ref: The core reference. - - Returns: - The definitions key. - """ - # Split the core ref into "components"; generic origins and arguments are each separate components - core_ref, mode = core_mode_ref - components = re.split(r'([\][,])', core_ref) - # Remove IDs from each component - components = [x.rsplit(':', 1)[0] for x in components] - core_ref_no_id = ''.join(components) - # Remove everything before the last period from each "component" - components = [re.sub(r'(?:[^.[\]]+\.)+((?:[^.[\]]+))', r'\1', x) for x in components] - short_ref = ''.join(components) - - mode_title = _MODE_TITLE_MAPPING[mode] - - # It is important that the generated defs_ref values be such that at least one choice will not - # be generated for any other core_ref. Currently, this should be the case because we include - # the id of the source type in the core_ref - name = DefsRef(self.normalize_name(short_ref)) - name_mode = DefsRef(self.normalize_name(short_ref) + f'-{mode_title}') - module_qualname = DefsRef(self.normalize_name(core_ref_no_id)) - module_qualname_mode = DefsRef(f'{module_qualname}-{mode_title}') - module_qualname_id = DefsRef(self.normalize_name(core_ref)) - occurrence_index = self._collision_index.get(module_qualname_id) - if occurrence_index is None: - self._collision_counter[module_qualname] += 1 - occurrence_index = self._collision_index[module_qualname_id] = self._collision_counter[module_qualname] - - module_qualname_occurrence = DefsRef(f'{module_qualname}__{occurrence_index}') - module_qualname_occurrence_mode = DefsRef(f'{module_qualname_mode}__{occurrence_index}') - - self._prioritized_defsref_choices[module_qualname_occurrence_mode] = [ - name, - name_mode, - module_qualname, - module_qualname_mode, - module_qualname_occurrence, - module_qualname_occurrence_mode, - ] - - return module_qualname_occurrence_mode - - def get_cache_defs_ref_schema(self, core_ref: CoreRef) -> tuple[DefsRef, JsonSchemaValue]: - """This method wraps the get_defs_ref method with some cache-lookup/population logic, - and returns both the produced defs_ref and the JSON schema that will refer to the right definition. - - Args: - core_ref: The core reference to get the definitions reference for. - - Returns: - A tuple of the definitions reference and the JSON schema that will refer to it. - """ - core_mode_ref = (core_ref, self.mode) - maybe_defs_ref = self.core_to_defs_refs.get(core_mode_ref) - if maybe_defs_ref is not None: - json_ref = self.core_to_json_refs[core_mode_ref] - return maybe_defs_ref, {'$ref': json_ref} - - defs_ref = self.get_defs_ref(core_mode_ref) - - # populate the ref translation mappings - self.core_to_defs_refs[core_mode_ref] = defs_ref - self.defs_to_core_refs[defs_ref] = core_mode_ref - - json_ref = JsonRef(self.ref_template.format(model=defs_ref)) - self.core_to_json_refs[core_mode_ref] = json_ref - self.json_to_defs_refs[json_ref] = defs_ref - ref_json_schema = {'$ref': json_ref} - return defs_ref, ref_json_schema - - def handle_ref_overrides(self, json_schema: JsonSchemaValue) -> JsonSchemaValue: - """Remove any sibling keys that are redundant with the referenced schema. - - Args: - json_schema: The schema to remove redundant sibling keys from. - - Returns: - The schema with redundant sibling keys removed. - """ - if '$ref' in json_schema: - # prevent modifications to the input; this copy may be safe to drop if there is significant overhead - json_schema = json_schema.copy() - - referenced_json_schema = self.get_schema_from_definitions(JsonRef(json_schema['$ref'])) - if referenced_json_schema is None: - # This can happen when building schemas for models with not-yet-defined references. - # It may be a good idea to do a recursive pass at the end of the generation to remove - # any redundant override keys. - return json_schema - for k, v in list(json_schema.items()): - if k == '$ref': - continue - if k in referenced_json_schema and referenced_json_schema[k] == v: - del json_schema[k] # redundant key - - return json_schema - - def get_schema_from_definitions(self, json_ref: JsonRef) -> JsonSchemaValue | None: - try: - def_ref = self.json_to_defs_refs[json_ref] - if def_ref in self._core_defs_invalid_for_json_schema: - raise self._core_defs_invalid_for_json_schema[def_ref] - return self.definitions.get(def_ref, None) - except KeyError: - if json_ref.startswith(('http://', 'https://')): - return None - raise - - def encode_default(self, dft: Any) -> Any: - """Encode a default value to a JSON-serializable value. - - This is used to encode default values for fields in the generated JSON schema. - - Args: - dft: The default value to encode. - - Returns: - The encoded default value. - """ - from .type_adapter import TypeAdapter, _type_has_config - - config = self._config - try: - default = ( - dft - if _type_has_config(type(dft)) - else TypeAdapter(type(dft), config=config.config_dict).dump_python( - dft, by_alias=self.by_alias, mode='json' - ) - ) - except PydanticSchemaGenerationError: - raise pydantic_core.PydanticSerializationError(f'Unable to encode default value {dft}') - - return pydantic_core.to_jsonable_python( - default, timedelta_mode=config.ser_json_timedelta, bytes_mode=config.ser_json_bytes, by_alias=self.by_alias - ) - - def update_with_validations( - self, json_schema: JsonSchemaValue, core_schema: CoreSchema, mapping: dict[str, str] - ) -> None: - """Update the json_schema with the corresponding validations specified in the core_schema, - using the provided mapping to translate keys in core_schema to the appropriate keys for a JSON schema. - - Args: - json_schema: The JSON schema to update. - core_schema: The core schema to get the validations from. - mapping: A mapping from core_schema attribute names to the corresponding JSON schema attribute names. - """ - for core_key, json_schema_key in mapping.items(): - if core_key in core_schema: - json_schema[json_schema_key] = core_schema[core_key] - - class ValidationsMapping: - """This class just contains mappings from core_schema attribute names to the corresponding - JSON schema attribute names. While I suspect it is unlikely to be necessary, you can in - principle override this class in a subclass of GenerateJsonSchema (by inheriting from - GenerateJsonSchema.ValidationsMapping) to change these mappings. - """ - - numeric = { - 'multiple_of': 'multipleOf', - 'le': 'maximum', - 'ge': 'minimum', - 'lt': 'exclusiveMaximum', - 'gt': 'exclusiveMinimum', - } - bytes = { - 'min_length': 'minLength', - 'max_length': 'maxLength', - } - string = { - 'min_length': 'minLength', - 'max_length': 'maxLength', - 'pattern': 'pattern', - } - array = { - 'min_length': 'minItems', - 'max_length': 'maxItems', - } - object = { - 'min_length': 'minProperties', - 'max_length': 'maxProperties', - } - - def get_flattened_anyof(self, schemas: list[JsonSchemaValue]) -> JsonSchemaValue: - members = [] - for schema in schemas: - if len(schema) == 1 and 'anyOf' in schema: - members.extend(schema['anyOf']) - else: - members.append(schema) - members = _deduplicate_schemas(members) - if len(members) == 1: - return members[0] - return {'anyOf': members} - - def get_json_ref_counts(self, json_schema: JsonSchemaValue) -> dict[JsonRef, int]: - """Get all values corresponding to the key '$ref' anywhere in the json_schema.""" - json_refs: dict[JsonRef, int] = Counter() - - def _add_json_refs(schema: Any) -> None: - if isinstance(schema, dict): - if '$ref' in schema: - json_ref = JsonRef(schema['$ref']) - if not isinstance(json_ref, str): - return # in this case, '$ref' might have been the name of a property - already_visited = json_ref in json_refs - json_refs[json_ref] += 1 - if already_visited: - return # prevent recursion on a definition that was already visited - try: - defs_ref = self.json_to_defs_refs[json_ref] - if defs_ref in self._core_defs_invalid_for_json_schema: - raise self._core_defs_invalid_for_json_schema[defs_ref] - _add_json_refs(self.definitions[defs_ref]) - except KeyError: - if not json_ref.startswith(('http://', 'https://')): - raise - - for k, v in schema.items(): - if k == 'examples' and isinstance(v, list): - # Skip examples that may contain arbitrary values and references - # (see the comment in `_get_all_json_refs` for more details). - continue - _add_json_refs(v) - elif isinstance(schema, list): - for v in schema: - _add_json_refs(v) - - _add_json_refs(json_schema) - return json_refs - - def handle_invalid_for_json_schema(self, schema: CoreSchemaOrField, error_info: str) -> JsonSchemaValue: - raise PydanticInvalidForJsonSchema(f'Cannot generate a JsonSchema for {error_info}') - - def emit_warning(self, kind: JsonSchemaWarningKind, detail: str) -> None: - """This method simply emits PydanticJsonSchemaWarnings based on handling in the `warning_message` method.""" - message = self.render_warning_message(kind, detail) - if message is not None: - warnings.warn(message, PydanticJsonSchemaWarning) - - def render_warning_message(self, kind: JsonSchemaWarningKind, detail: str) -> str | None: - """This method is responsible for ignoring warnings as desired, and for formatting the warning messages. - - You can override the value of `ignored_warning_kinds` in a subclass of GenerateJsonSchema - to modify what warnings are generated. If you want more control, you can override this method; - just return None in situations where you don't want warnings to be emitted. - - Args: - kind: The kind of warning to render. It can be one of the following: - - - 'skipped-choice': A choice field was skipped because it had no valid choices. - - 'non-serializable-default': A default value was skipped because it was not JSON-serializable. - detail: A string with additional details about the warning. - - Returns: - The formatted warning message, or `None` if no warning should be emitted. - """ - if kind in self.ignored_warning_kinds: - return None - return f'{detail} [{kind}]' - - def _build_definitions_remapping(self) -> _DefinitionsRemapping: - defs_to_json: dict[DefsRef, JsonRef] = {} - for defs_refs in self._prioritized_defsref_choices.values(): - for defs_ref in defs_refs: - json_ref = JsonRef(self.ref_template.format(model=defs_ref)) - defs_to_json[defs_ref] = json_ref - - return _DefinitionsRemapping.from_prioritized_choices( - self._prioritized_defsref_choices, defs_to_json, self.definitions - ) - - def _garbage_collect_definitions(self, schema: JsonSchemaValue) -> None: - visited_defs_refs: set[DefsRef] = set() - unvisited_json_refs = _get_all_json_refs(schema) - while unvisited_json_refs: - next_json_ref = unvisited_json_refs.pop() - try: - next_defs_ref = self.json_to_defs_refs[next_json_ref] - if next_defs_ref in visited_defs_refs: - continue - visited_defs_refs.add(next_defs_ref) - unvisited_json_refs.update(_get_all_json_refs(self.definitions[next_defs_ref])) - except KeyError: - if not next_json_ref.startswith(('http://', 'https://')): - raise - - self.definitions = {k: v for k, v in self.definitions.items() if k in visited_defs_refs} - - -# ##### Start JSON Schema Generation Functions ##### - - -def model_json_schema( - cls: type[BaseModel] | type[PydanticDataclass], - by_alias: bool = True, - ref_template: str = DEFAULT_REF_TEMPLATE, - union_format: Literal['any_of', 'primitive_type_array'] = 'any_of', - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, - mode: JsonSchemaMode = 'validation', -) -> dict[str, Any]: - """Utility function to generate a JSON Schema for a model. - - Args: - cls: The model class to generate a JSON Schema for. - by_alias: If `True` (the default), fields will be serialized according to their alias. - If `False`, fields will be serialized according to their attribute name. - ref_template: The template to use for generating JSON Schema references. - union_format: The format to use when combining schemas from unions together. Can be one of: - - - `'any_of'`: Use the [`anyOf`](https://json-schema.org/understanding-json-schema/reference/combining#anyOf) - keyword to combine schemas (the default). - - `'primitive_type_array'`: Use the [`type`](https://json-schema.org/understanding-json-schema/reference/type) - keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive - type (`string`, `boolean`, `null`, `integer` or `number`) or contains constraints/metadata, falls back to - `any_of`. - schema_generator: The class to use for generating the JSON Schema. - mode: The mode to use for generating the JSON Schema. It can be one of the following: - - - 'validation': Generate a JSON Schema for validating data. - - 'serialization': Generate a JSON Schema for serializing data. - - Returns: - The generated JSON Schema. - """ - from .main import BaseModel - - schema_generator_instance = schema_generator( - by_alias=by_alias, ref_template=ref_template, union_format=union_format - ) - - if isinstance(cls.__pydantic_core_schema__, _mock_val_ser.MockCoreSchema): - cls.__pydantic_core_schema__.rebuild() - - if cls is BaseModel: - raise AttributeError('model_json_schema() must be called on a subclass of BaseModel, not BaseModel itself.') - - assert not isinstance(cls.__pydantic_core_schema__, _mock_val_ser.MockCoreSchema), 'this is a bug! please report it' - return schema_generator_instance.generate(cls.__pydantic_core_schema__, mode=mode) - - -def models_json_schema( - models: Sequence[tuple[type[BaseModel] | type[PydanticDataclass], JsonSchemaMode]], - *, - by_alias: bool = True, - title: str | None = None, - description: str | None = None, - ref_template: str = DEFAULT_REF_TEMPLATE, - union_format: Literal['any_of', 'primitive_type_array'] = 'any_of', - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, -) -> tuple[dict[tuple[type[BaseModel] | type[PydanticDataclass], JsonSchemaMode], JsonSchemaValue], JsonSchemaValue]: - """Utility function to generate a JSON Schema for multiple models. - - Args: - models: A sequence of tuples of the form (model, mode). - by_alias: Whether field aliases should be used as keys in the generated JSON Schema. - title: The title of the generated JSON Schema. - description: The description of the generated JSON Schema. - ref_template: The reference template to use for generating JSON Schema references. - union_format: The format to use when combining schemas from unions together. Can be one of: - - - `'any_of'`: Use the [`anyOf`](https://json-schema.org/understanding-json-schema/reference/combining#anyOf) - keyword to combine schemas (the default). - - `'primitive_type_array'`: Use the [`type`](https://json-schema.org/understanding-json-schema/reference/type) - keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive - type (`string`, `boolean`, `null`, `integer` or `number`) or contains constraints/metadata, falls back to - `any_of`. - schema_generator: The schema generator to use for generating the JSON Schema. - - Returns: - A tuple where: - - The first element is a dictionary whose keys are tuples of JSON schema key type and JSON mode, and - whose values are the JSON schema corresponding to that pair of inputs. (These schemas may have - JsonRef references to definitions that are defined in the second returned element.) - - The second element is a JSON schema containing all definitions referenced in the first returned - element, along with the optional title and description keys. - """ - for cls, _ in models: - if isinstance(cls.__pydantic_core_schema__, _mock_val_ser.MockCoreSchema): - cls.__pydantic_core_schema__.rebuild() - - instance = schema_generator(by_alias=by_alias, ref_template=ref_template, union_format=union_format) - inputs: list[tuple[type[BaseModel] | type[PydanticDataclass], JsonSchemaMode, CoreSchema]] = [ - (m, mode, m.__pydantic_core_schema__) for m, mode in models - ] - json_schemas_map, definitions = instance.generate_definitions(inputs) - - json_schema: dict[str, Any] = {} - if definitions: - json_schema['$defs'] = definitions - if title: - json_schema['title'] = title - if description: - json_schema['description'] = description - - return json_schemas_map, json_schema - - -# ##### End JSON Schema Generation Functions ##### - - -_HashableJsonValue: TypeAlias = Union[ - int, float, str, bool, None, tuple['_HashableJsonValue', ...], tuple[tuple[str, '_HashableJsonValue'], ...] -] - - -def _deduplicate_schemas(schemas: Iterable[JsonDict]) -> list[JsonDict]: - return list({_make_json_hashable(schema): schema for schema in schemas}.values()) - - -def _make_json_hashable(value: JsonValue) -> _HashableJsonValue: - if isinstance(value, dict): - return tuple(sorted((k, _make_json_hashable(v)) for k, v in value.items())) - elif isinstance(value, list): - return tuple(_make_json_hashable(v) for v in value) - else: - return value - - -@dataclasses.dataclass(**_internal_dataclass.slots_true) -class WithJsonSchema: - """!!! abstract "Usage Documentation" - [`WithJsonSchema` Annotation](../concepts/json_schema.md#withjsonschema-annotation) - - An annotation used to override the JSON Schema for a type. - - This is useful when you want to set a JSON Schema for a type that don't produce any JSON Schemas by default - (e.g. [`Callable`][collections.abc.Callable]). - - If `mode` is set this will only apply to that schema generation mode, allowing you to set different JSON Schemas for validation and serialization. - - !!! note - If the `WithJsonSchema` annotation is coupled with the [`Field()`][pydantic.Field] function, the behavior overriding will vary depending on the location: - - * If the [`Annotated`][typing.Annotated] metadata is specified at the "top-level" field, `Field()` metadata arguments - (excluding [constraints](../concepts/fields.md#field-constraints)) such as `title` and `description` will be applied on - top of the `WithJsonSchema`, no matter the order: - - ```python - from typing import Annotated - - from pydantic import BaseModel, Field, WithJsonSchema - - class Model(BaseModel): - field: Annotated[ - int, - Field(title='My Field'), - WithJsonSchema({'type': 'integer', 'extra': 'data'}), - ] - - Model.model_json_schema()['properties']['field'] - #> {'type': 'integer', 'extra': 'data', 'title': 'My Field'} - ``` - - * If the [`Annotated`][typing.Annotated] metadata is specified on a specific inner type, `WithJsonSchema` will unconditionally - override the JSON Schema: - - ```python - from typing import Annotated - - from pydantic import BaseModel, Field, WithJsonSchema - - class Model(BaseModel): - field: list[ - Annotated[ - int, - Field(title='My Field'), - WithJsonSchema({'type': 'integer', 'extra': 'data'}), - ] - ] - - Model.model_json_schema()['properties']['field'] - #> {'items': {'extra': 'data', 'type': 'integer'}, 'title': 'Field', 'type': 'array'} - ``` - - See also the documentation about [the annotated pattern](../concepts/fields.md#the-annotated-pattern). - """ - - json_schema: JsonSchemaValue | None - mode: Literal['validation', 'serialization'] | None = None - - def __get_pydantic_json_schema__( - self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - if self.mode is not None and self.mode != handler.mode: - return handler(core_schema) - if self.json_schema is None: - # This exception is handled in pydantic.json_schema.GenerateJsonSchema._named_required_fields_schema - raise PydanticOmit - else: - return self.json_schema.copy() - - def __hash__(self) -> int: - return hash(type(self.mode)) - - -class Examples: - """Add examples to a JSON schema. - - If the JSON Schema already contains examples, the provided examples - will be appended. - - If `mode` is set this will only apply to that schema generation mode, - allowing you to add different examples for validation and serialization. - """ - - @overload - @deprecated('Using a dict for `examples` is deprecated since v2.9 and will be removed in v3.0. Use a list instead.') - def __init__( - self, examples: dict[str, Any], mode: Literal['validation', 'serialization'] | None = None - ) -> None: ... - - @overload - def __init__(self, examples: list[Any], mode: Literal['validation', 'serialization'] | None = None) -> None: ... - - def __init__( - self, examples: dict[str, Any] | list[Any], mode: Literal['validation', 'serialization'] | None = None - ) -> None: - if isinstance(examples, dict): - warnings.warn( - 'Using a dict for `examples` is deprecated, use a list instead.', - PydanticDeprecatedSince29, - stacklevel=2, - ) - self.examples = examples - self.mode = mode - - def __get_pydantic_json_schema__( - self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - mode = self.mode or handler.mode - json_schema = handler(core_schema) - if mode != handler.mode: - return json_schema - examples = json_schema.get('examples') - if examples is None: - json_schema['examples'] = to_jsonable_python(self.examples) - if isinstance(examples, dict): - if isinstance(self.examples, list): - warnings.warn( - 'Updating existing JSON Schema examples of type dict with examples of type list. ' - 'Only the existing examples values will be retained. Note that dict support for ' - 'examples is deprecated and will be removed in v3.0.', - UserWarning, - ) - json_schema['examples'] = to_jsonable_python( - [ex for value in examples.values() for ex in value] + self.examples - ) - else: - json_schema['examples'] = to_jsonable_python({**examples, **self.examples}) - if isinstance(examples, list): - if isinstance(self.examples, list): - json_schema['examples'] = to_jsonable_python(examples + self.examples) - elif isinstance(self.examples, dict): - warnings.warn( - 'Updating existing JSON Schema examples of type list with examples of type dict. ' - 'Only the examples values will be retained. Note that dict support for ' - 'examples is deprecated and will be removed in v3.0.', - UserWarning, - ) - json_schema['examples'] = to_jsonable_python( - examples + [ex for value in self.examples.values() for ex in value] - ) - - return json_schema - - def __hash__(self) -> int: - return hash(type(self.mode)) - - -def _get_all_json_refs(item: Any) -> set[JsonRef]: - """Get all the definitions references from a JSON schema.""" - refs: set[JsonRef] = set() - stack = [item] - - while stack: - current = stack.pop() - if isinstance(current, dict): - for key, value in current.items(): - if key == 'examples' and isinstance(value, list): - # Skip examples that may contain arbitrary values and references - # (e.g. `{"examples": [{"$ref": "..."}]}`). Note: checking for value - # of type list is necessary to avoid skipping valid portions of the schema, - # for instance when "examples" is used as a property key. A more robust solution - # could be found, but would require more advanced JSON Schema parsing logic. - continue - if key == '$ref' and isinstance(value, str): - refs.add(JsonRef(value)) - elif isinstance(value, dict): - stack.append(value) - elif isinstance(value, list): - stack.extend(value) - elif isinstance(current, list): - stack.extend(current) - - return refs - - -AnyType = TypeVar('AnyType') - -if TYPE_CHECKING: - SkipJsonSchema = Annotated[AnyType, ...] -else: - - @dataclasses.dataclass(**_internal_dataclass.slots_true) - class SkipJsonSchema: - """!!! abstract "Usage Documentation" - [`SkipJsonSchema` Annotation](../concepts/json_schema.md#skipjsonschema-annotation) - - Add this as an annotation on a field to skip generating a JSON schema for that field. - - Example: - ```python - from pprint import pprint - from typing import Union - - from pydantic import BaseModel - from pydantic.json_schema import SkipJsonSchema - - class Model(BaseModel): - a: Union[int, None] = None # (1)! - b: Union[int, SkipJsonSchema[None]] = None # (2)! - c: SkipJsonSchema[Union[int, None]] = None # (3)! - - pprint(Model.model_json_schema()) - ''' - { - 'properties': { - 'a': { - 'anyOf': [ - {'type': 'integer'}, - {'type': 'null'} - ], - 'default': None, - 'title': 'A' - }, - 'b': { - 'default': None, - 'title': 'B', - 'type': 'integer' - } - }, - 'title': 'Model', - 'type': 'object' - } - ''' - ``` - - 1. The integer and null types are both included in the schema for `a`. - 2. The integer type is the only type included in the schema for `b`. - 3. The entirety of the `c` field is omitted from the schema. - """ - - def __class_getitem__(cls, item: AnyType) -> AnyType: - return Annotated[item, cls()] - - def __get_pydantic_json_schema__( - self, core_schema: CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - raise PydanticOmit - - def __hash__(self) -> int: - return hash(type(self)) - - -def _get_typed_dict_config(cls: type[Any] | None) -> ConfigDict: - if cls is not None: - try: - return _decorators.get_attribute_from_bases(cls, '__pydantic_config__') - except AttributeError: - pass - return {} - - -def _get_ser_schema_for_default_value(schema: CoreSchema) -> core_schema.PlainSerializerFunctionSerSchema | None: - """Get a `'function-plain'` serialization schema that can be used to serialize a default value. - - This takes into account having the serialization schema nested under validation schema(s). - """ - if ( - (ser_schema := schema.get('serialization')) - and ser_schema['type'] == 'function-plain' - and not ser_schema.get('info_arg') - ): - return ser_schema - if _core_utils.is_function_with_inner_schema(schema): - return _get_ser_schema_for_default_value(schema['schema']) diff --git a/functions/python/data-processor/pydantic/main.py b/functions/python/data-processor/pydantic/main.py deleted file mode 100644 index 2ed62e5f3..000000000 --- a/functions/python/data-processor/pydantic/main.py +++ /dev/null @@ -1,1838 +0,0 @@ -"""Logic for creating models.""" - -# Because `dict` is in the local namespace of the `BaseModel` class, we use `Dict` for annotations. -# TODO v3 fallback to `dict` when the deprecated `dict` method gets removed. -# ruff: noqa: UP035 - -from __future__ import annotations as _annotations - -import operator -import sys -import types -import warnings -from collections.abc import Generator, Mapping -from copy import copy, deepcopy -from functools import cached_property -from typing import ( - TYPE_CHECKING, - Any, - Callable, - ClassVar, - Dict, - Generic, - Literal, - TypeVar, - Union, - cast, - overload, -) - -import pydantic_core -import typing_extensions -from pydantic_core import PydanticUndefined, ValidationError -from typing_extensions import Self, TypeAlias, Unpack - -from . import PydanticDeprecatedSince20, PydanticDeprecatedSince211 -from ._internal import ( - _config, - _decorators, - _fields, - _forward_ref, - _generics, - _mock_val_ser, - _model_construction, - _namespace_utils, - _repr, - _typing_extra, - _utils, -) -from ._migration import getattr_migration -from .aliases import AliasChoices, AliasPath -from .annotated_handlers import GetCoreSchemaHandler, GetJsonSchemaHandler -from .config import ConfigDict, ExtraValues -from .errors import PydanticUndefinedAnnotation, PydanticUserError -from .json_schema import DEFAULT_REF_TEMPLATE, GenerateJsonSchema, JsonSchemaMode, JsonSchemaValue, model_json_schema -from .plugin._schema_validator import PluggableSchemaValidator - -if TYPE_CHECKING: - from inspect import Signature - from pathlib import Path - - from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator - - from ._internal._fields import PydanticExtraInfo - from ._internal._namespace_utils import MappingNamespace - from ._internal._utils import AbstractSetIntStr, MappingIntStrAny - from .deprecated.parse import Protocol as DeprecatedParseProtocol - from .fields import ComputedFieldInfo, FieldInfo, ModelPrivateAttr - - -__all__ = 'BaseModel', 'create_model' - -# Keep these type aliases available at runtime: -TupleGenerator: TypeAlias = Generator[tuple[str, Any], None, None] -# NOTE: In reality, `bool` should be replaced by `Literal[True]` but mypy fails to correctly apply bidirectional -# type inference (e.g. when using `{'a': {'b': True}}`): -# NOTE: Keep this type alias in sync with the stub definition in `pydantic-core`: -IncEx: TypeAlias = Union[set[int], set[str], Mapping[int, Union['IncEx', bool]], Mapping[str, Union['IncEx', bool]]] - -_object_setattr = _model_construction.object_setattr - - -def _check_frozen(model_cls: type[BaseModel], name: str, value: Any) -> None: - if model_cls.model_config.get('frozen'): - error_type = 'frozen_instance' - elif getattr(model_cls.__pydantic_fields__.get(name), 'frozen', False): - error_type = 'frozen_field' - else: - return - - raise ValidationError.from_exception_data( - model_cls.__name__, [{'type': error_type, 'loc': (name,), 'input': value}] - ) - - -def _model_field_setattr_handler(model: BaseModel, name: str, val: Any) -> None: - model.__dict__[name] = val - model.__pydantic_fields_set__.add(name) - - -def _private_setattr_handler(model: BaseModel, name: str, val: Any) -> None: - if getattr(model, '__pydantic_private__', None) is None: - # While the attribute should be present at this point, this may not be the case if - # users do unusual stuff with `model_post_init()` (which is where the `__pydantic_private__` - # is initialized, by wrapping the user-defined `model_post_init()`), e.g. if they mock - # the `model_post_init()` call. Ideally we should find a better way to init private attrs. - object.__setattr__(model, '__pydantic_private__', {}) - model.__pydantic_private__[name] = val # pyright: ignore[reportOptionalSubscript] - - -_SIMPLE_SETATTR_HANDLERS: Mapping[str, Callable[[BaseModel, str, Any], None]] = { - 'model_field': _model_field_setattr_handler, - 'validate_assignment': lambda model, name, val: model.__pydantic_validator__.validate_assignment(model, name, val), # pyright: ignore[reportAssignmentType] - 'private': _private_setattr_handler, - 'cached_property': lambda model, name, val: model.__dict__.__setitem__(name, val), - 'extra_known': lambda model, name, val: _object_setattr(model, name, val), -} - - -class BaseModel(metaclass=_model_construction.ModelMetaclass): - """!!! abstract "Usage Documentation" - [Models](../concepts/models.md) - - A base class for creating Pydantic models. - - Attributes: - __class_vars__: The names of the class variables defined on the model. - __private_attributes__: Metadata about the private attributes of the model. - __signature__: The synthesized `__init__` [`Signature`][inspect.Signature] of the model. - - __pydantic_complete__: Whether model building is completed, or if there are still undefined fields. - __pydantic_core_schema__: The core schema of the model. - __pydantic_custom_init__: Whether the model has a custom `__init__` function. - __pydantic_decorators__: Metadata containing the decorators defined on the model. - This replaces `Model.__validators__` and `Model.__root_validators__` from Pydantic V1. - __pydantic_generic_metadata__: A dictionary containing metadata about generic Pydantic models. - The `origin` and `args` items map to the [`__origin__`][genericalias.__origin__] - and [`__args__`][genericalias.__args__] attributes of [generic aliases][types-genericalias], - and the `parameter` item maps to the `__parameter__` attribute of generic classes. - __pydantic_parent_namespace__: Parent namespace of the model, used for automatic rebuilding of models. - __pydantic_post_init__: The name of the post-init method for the model, if defined. - __pydantic_root_model__: Whether the model is a [`RootModel`][pydantic.root_model.RootModel]. - __pydantic_serializer__: The `pydantic-core` `SchemaSerializer` used to dump instances of the model. - __pydantic_validator__: The `pydantic-core` `SchemaValidator` used to validate instances of the model. - - __pydantic_fields__: A dictionary of field names and their corresponding [`FieldInfo`][pydantic.fields.FieldInfo] objects. - __pydantic_computed_fields__: A dictionary of computed field names and their corresponding [`ComputedFieldInfo`][pydantic.fields.ComputedFieldInfo] objects. - - __pydantic_extra__: A dictionary containing extra values, if [`extra`][pydantic.config.ConfigDict.extra] - is set to `'allow'`. - __pydantic_fields_set__: The names of fields explicitly set during instantiation. - __pydantic_private__: Values of private attributes set on the model instance. - """ - - # Note: Many of the below class vars are defined in the metaclass, but we define them here for type checking purposes. - - model_config: ClassVar[ConfigDict] = ConfigDict() - """ - Configuration for the model, should be a dictionary conforming to [`ConfigDict`][pydantic.config.ConfigDict]. - """ - - __class_vars__: ClassVar[set[str]] - """The names of the class variables defined on the model.""" - - __private_attributes__: ClassVar[Dict[str, ModelPrivateAttr]] # noqa: UP006 - """Metadata about the private attributes of the model.""" - - __signature__: ClassVar[Signature] - """The synthesized `__init__` [`Signature`][inspect.Signature] of the model.""" - - __pydantic_complete__: ClassVar[bool] = False - """Whether model building is completed, or if there are still undefined fields.""" - - __pydantic_core_schema__: ClassVar[CoreSchema] - """The core schema of the model.""" - - __pydantic_custom_init__: ClassVar[bool] - """Whether the model has a custom `__init__` method.""" - - # Must be set for `GenerateSchema.model_schema` to work for a plain `BaseModel` annotation. - __pydantic_decorators__: ClassVar[_decorators.DecoratorInfos] = _decorators.DecoratorInfos() - """Metadata containing the decorators defined on the model. - This replaces `Model.__validators__` and `Model.__root_validators__` from Pydantic V1.""" - - __pydantic_generic_metadata__: ClassVar[_generics.PydanticGenericMetadata] - """A dictionary containing metadata about generic Pydantic models. - - The `origin` and `args` items map to the [`__origin__`][genericalias.__origin__] - and [`__args__`][genericalias.__args__] attributes of [generic aliases][types-genericalias], - and the `parameter` item maps to the `__parameter__` attribute of generic classes. - """ - - __pydantic_parent_namespace__: ClassVar[Dict[str, Any] | None] = None # noqa: UP006 - """Parent namespace of the model, used for automatic rebuilding of models.""" - - __pydantic_post_init__: ClassVar[None | Literal['model_post_init']] - """The name of the post-init method for the model, if defined.""" - - __pydantic_root_model__: ClassVar[bool] = False - """Whether the model is a [`RootModel`][pydantic.root_model.RootModel].""" - - __pydantic_serializer__: ClassVar[SchemaSerializer] - """The `pydantic-core` `SchemaSerializer` used to dump instances of the model.""" - - __pydantic_validator__: ClassVar[SchemaValidator | PluggableSchemaValidator] - """The `pydantic-core` `SchemaValidator` used to validate instances of the model.""" - - __pydantic_fields__: ClassVar[Dict[str, FieldInfo]] # noqa: UP006 - """A dictionary of field names and their corresponding [`FieldInfo`][pydantic.fields.FieldInfo] objects. - This replaces `Model.__fields__` from Pydantic V1. - """ - - __pydantic_setattr_handlers__: ClassVar[Dict[str, Callable[[BaseModel, str, Any], None]]] # noqa: UP006 - """`__setattr__` handlers. Memoizing the handlers leads to a dramatic performance improvement in `__setattr__`""" - - __pydantic_computed_fields__: ClassVar[Dict[str, ComputedFieldInfo]] # noqa: UP006 - """A dictionary of computed field names and their corresponding [`ComputedFieldInfo`][pydantic.fields.ComputedFieldInfo] objects.""" - - __pydantic_extra_info__: ClassVar[PydanticExtraInfo | None] - """A wrapper around the `__pydantic_extra__` annotation, if explicitly annotated on a model. - - This is a private attribute, not meant to be used outside Pydantic. - """ - - __pydantic_extra__: Dict[str, Any] | None = _model_construction.NoInitField(init=False) # noqa: UP006 - """A dictionary containing extra values, if [`extra`][pydantic.config.ConfigDict.extra] is set to `'allow'`.""" - - __pydantic_fields_set__: set[str] = _model_construction.NoInitField(init=False) - """The names of fields explicitly set during instantiation.""" - - __pydantic_private__: Dict[str, Any] | None = _model_construction.NoInitField(init=False) # noqa: UP006 - """Values of private attributes set on the model instance.""" - - if not TYPE_CHECKING: - # Prevent `BaseModel` from being instantiated directly - # (defined in an `if not TYPE_CHECKING` block for clarity and to avoid type checking errors): - __pydantic_core_schema__ = _mock_val_ser.MockCoreSchema( - 'Pydantic models should inherit from BaseModel, BaseModel cannot be instantiated directly', - code='base-model-instantiated', - ) - __pydantic_validator__ = _mock_val_ser.MockValSer( - 'Pydantic models should inherit from BaseModel, BaseModel cannot be instantiated directly', - val_or_ser='validator', - code='base-model-instantiated', - ) - __pydantic_serializer__ = _mock_val_ser.MockValSer( - 'Pydantic models should inherit from BaseModel, BaseModel cannot be instantiated directly', - val_or_ser='serializer', - code='base-model-instantiated', - ) - - __slots__ = '__dict__', '__pydantic_fields_set__', '__pydantic_extra__', '__pydantic_private__' - - def __init__(self, /, **data: Any) -> None: - """Create a new model by parsing and validating input data from keyword arguments. - - Raises [`ValidationError`][pydantic_core.ValidationError] if the input data cannot be - validated to form a valid model. - - `self` is explicitly positional-only to allow `self` as a field name. - """ - # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks - __tracebackhide__ = True - validated_self = self.__pydantic_validator__.validate_python(data, self_instance=self) - if self is not validated_self: - warnings.warn( - 'A custom validator is returning a value other than `self`.\n' - "Returning anything other than `self` from a top level model validator isn't supported when validating via `__init__`.\n" - 'See the `model_validator` docs (https://docs.pydantic.dev/latest/concepts/validators/#model-validators) for more details.', - stacklevel=2, - ) - - # The following line sets a flag that we use to determine when `__init__` gets overridden by the user - __init__.__pydantic_base_init__ = True # pyright: ignore[reportFunctionMemberAccess] - - @_utils.deprecated_instance_property - @classmethod - def model_fields(cls) -> dict[str, FieldInfo]: - """A mapping of field names to their respective [`FieldInfo`][pydantic.fields.FieldInfo] instances. - - !!! warning - Accessing this attribute from a model instance is deprecated, and will not work in Pydantic V3. - Instead, you should access this attribute from the model class. - """ - return getattr(cls, '__pydantic_fields__', {}) - - @_utils.deprecated_instance_property - @classmethod - def model_computed_fields(cls) -> dict[str, ComputedFieldInfo]: - """A mapping of computed field names to their respective [`ComputedFieldInfo`][pydantic.fields.ComputedFieldInfo] instances. - - !!! warning - Accessing this attribute from a model instance is deprecated, and will not work in Pydantic V3. - Instead, you should access this attribute from the model class. - """ - return getattr(cls, '__pydantic_computed_fields__', {}) - - @property - def model_extra(self) -> dict[str, Any] | None: - """Get extra fields set during validation. - - Returns: - A dictionary of extra fields, or `None` if `config.extra` is not set to `"allow"`. - """ - return self.__pydantic_extra__ - - @property - def model_fields_set(self) -> set[str]: - """Returns the set of fields that have been explicitly set on this model instance. - - Returns: - A set of strings representing the fields that have been set, - i.e. that were not filled from defaults. - """ - return self.__pydantic_fields_set__ - - @classmethod - def model_construct(cls, _fields_set: set[str] | None = None, **values: Any) -> Self: # noqa: C901 - """Creates a new instance of the `Model` class with validated data. - - Creates a new model setting `__dict__` and `__pydantic_fields_set__` from trusted or pre-validated data. - Default values are respected, but no other validation is performed. - - !!! note - `model_construct()` generally respects the `model_config.extra` setting on the provided model. - That is, if `model_config.extra == 'allow'`, then all extra passed values are added to the model instance's `__dict__` - and `__pydantic_extra__` fields. If `model_config.extra == 'ignore'` (the default), then all extra passed values are ignored. - Because no validation is performed with a call to `model_construct()`, having `model_config.extra == 'forbid'` does not result in - an error if extra values are passed, but they will be ignored. - - Args: - _fields_set: A set of field names that were originally explicitly set during instantiation. If provided, - this is directly used for the [`model_fields_set`][pydantic.BaseModel.model_fields_set] attribute. - Otherwise, the field names from the `values` argument will be used. - values: Trusted or pre-validated data dictionary. - - Returns: - A new instance of the `Model` class with validated data. - """ - m = cls.__new__(cls) - fields_values: dict[str, Any] = {} - fields_set = set() - - for name, field in cls.__pydantic_fields__.items(): - if field.alias is not None and field.alias in values: - fields_values[name] = values.pop(field.alias) - fields_set.add(name) - - if (name not in fields_set) and (field.validation_alias is not None): - validation_aliases: list[str | AliasPath] = ( - field.validation_alias.choices - if isinstance(field.validation_alias, AliasChoices) - else [field.validation_alias] - ) - - for alias in validation_aliases: - if isinstance(alias, str) and alias in values: - fields_values[name] = values.pop(alias) - fields_set.add(name) - break - elif isinstance(alias, AliasPath): - value = alias.search_dict_for_path(values) - if value is not PydanticUndefined: - fields_values[name] = value - fields_set.add(name) - break - - if name not in fields_set: - if name in values: - fields_values[name] = values.pop(name) - fields_set.add(name) - elif not field.is_required(): - fields_values[name] = field.get_default(call_default_factory=True, validated_data=fields_values) - if _fields_set is None: - _fields_set = fields_set - - _extra: dict[str, Any] | None = values if cls.model_config.get('extra') == 'allow' else None - _object_setattr(m, '__dict__', fields_values) - _object_setattr(m, '__pydantic_fields_set__', _fields_set) - if not cls.__pydantic_root_model__: - _object_setattr(m, '__pydantic_extra__', _extra) - _object_setattr(m, '__pydantic_private__', None) - - if cls.__pydantic_post_init__: - m.model_post_init(None) - # update private attributes with values set - if hasattr(m, '__pydantic_private__') and m.__pydantic_private__ is not None: - for k, v in values.items(): - if k in m.__private_attributes__: - m.__pydantic_private__[k] = v - - return m - - def model_copy(self, *, update: Mapping[str, Any] | None = None, deep: bool = False) -> Self: - """!!! abstract "Usage Documentation" - [`model_copy`](../concepts/models.md#model-copy) - - Returns a copy of the model. - - !!! note - The underlying instance's [`__dict__`][object.__dict__] attribute is copied. This - might have unexpected side effects if you store anything in it, on top of the model - fields (e.g. the value of [cached properties][functools.cached_property]). - - Args: - update: Values to change/add in the new model. Note: the data is not validated - before creating the new model. You should trust this data. - deep: Set to `True` to make a deep copy of the model. - - Returns: - New model instance. - """ - copied = self.__deepcopy__() if deep else self.__copy__() - if update: - if self.model_config.get('extra') == 'allow': - for k, v in update.items(): - if k in self.__pydantic_fields__: - copied.__dict__[k] = v - else: - if copied.__pydantic_extra__ is None: - copied.__pydantic_extra__ = {} - copied.__pydantic_extra__[k] = v - else: - copied.__dict__.update(update) - copied.__pydantic_fields_set__.update(update.keys()) - return copied - - def model_dump( - self, - *, - mode: Literal['json', 'python'] | str = 'python', - include: IncEx | None = None, - exclude: IncEx | None = None, - context: Any | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - ) -> dict[str, Any]: - """!!! abstract "Usage Documentation" - [`model_dump`](../concepts/serialization.md#python-mode) - - Generate a dictionary representation of the model, optionally specifying which fields to include or exclude. - - Args: - mode: The mode in which `to_python` should run. - If mode is 'json', the output will only contain JSON serializable types. - If mode is 'python', the output may contain non-JSON-serializable Python objects. - include: A set of fields to include in the output. - exclude: A set of fields to exclude from the output. - context: Additional context to pass to the serializer. - by_alias: Whether to use the field's alias in the dictionary key if defined. - exclude_unset: Whether to exclude fields that have not been explicitly set. - exclude_defaults: Whether to exclude fields that are set to their default value. - exclude_none: Whether to exclude fields that have a value of `None`. - exclude_computed_fields: Whether to exclude computed fields. - While this can be useful for round-tripping, it is usually recommended to use the dedicated - `round_trip` parameter instead. - round_trip: If True, dumped values should be valid as input for non-idempotent types such as Json[T]. - warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered. If not provided, - a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - - Returns: - A dictionary representation of the model. - """ - return self.__pydantic_serializer__.to_python( - self, - mode=mode, - by_alias=by_alias, - include=include, - exclude=exclude, - context=context, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - exclude_computed_fields=exclude_computed_fields, - round_trip=round_trip, - warnings=warnings, - fallback=fallback, - serialize_as_any=serialize_as_any, - polymorphic_serialization=polymorphic_serialization, - ) - - def model_dump_json( - self, - *, - indent: int | None = None, - ensure_ascii: bool = False, - include: IncEx | None = None, - exclude: IncEx | None = None, - context: Any | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - ) -> str: - """!!! abstract "Usage Documentation" - [`model_dump_json`](../concepts/serialization.md#json-mode) - - Generates a JSON representation of the model using Pydantic's `to_json` method. - - Args: - indent: Indentation to use in the JSON output. If None is passed, the output will be compact. - ensure_ascii: If `True`, the output is guaranteed to have all incoming non-ASCII characters escaped. - If `False` (the default), these characters will be output as-is. - include: Field(s) to include in the JSON output. - exclude: Field(s) to exclude from the JSON output. - context: Additional context to pass to the serializer. - by_alias: Whether to serialize using field aliases. - exclude_unset: Whether to exclude fields that have not been explicitly set. - exclude_defaults: Whether to exclude fields that are set to their default value. - exclude_none: Whether to exclude fields that have a value of `None`. - exclude_computed_fields: Whether to exclude computed fields. - While this can be useful for round-tripping, it is usually recommended to use the dedicated - `round_trip` parameter instead. - round_trip: If True, dumped values should be valid as input for non-idempotent types such as Json[T]. - warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered. If not provided, - a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - - Returns: - A JSON string representation of the model. - """ - return self.__pydantic_serializer__.to_json( - self, - indent=indent, - ensure_ascii=ensure_ascii, - include=include, - exclude=exclude, - context=context, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - exclude_computed_fields=exclude_computed_fields, - round_trip=round_trip, - warnings=warnings, - fallback=fallback, - serialize_as_any=serialize_as_any, - polymorphic_serialization=polymorphic_serialization, - ).decode() - - @classmethod - def model_json_schema( - cls, - by_alias: bool = True, - ref_template: str = DEFAULT_REF_TEMPLATE, - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, - mode: JsonSchemaMode = 'validation', - *, - union_format: Literal['any_of', 'primitive_type_array'] = 'any_of', - ) -> dict[str, Any]: - """Generates a JSON schema for a model class. - - Args: - by_alias: Whether to use attribute aliases or not. - ref_template: The reference template. - union_format: The format to use when combining schemas from unions together. Can be one of: - - - `'any_of'`: Use the [`anyOf`](https://json-schema.org/understanding-json-schema/reference/combining#anyOf) - keyword to combine schemas (the default). - - `'primitive_type_array'`: Use the [`type`](https://json-schema.org/understanding-json-schema/reference/type) - keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive - type (`string`, `boolean`, `null`, `integer` or `number`) or contains constraints/metadata, falls back to - `any_of`. - schema_generator: To override the logic used to generate the JSON schema, as a subclass of - `GenerateJsonSchema` with your desired modifications - mode: The mode in which to generate the schema. - - Returns: - The JSON schema for the given model class. - """ - return model_json_schema( - cls, - by_alias=by_alias, - ref_template=ref_template, - union_format=union_format, - schema_generator=schema_generator, - mode=mode, - ) - - @classmethod - def model_parametrized_name(cls, params: tuple[type[Any], ...]) -> str: - """Compute the class name for parametrizations of generic classes. - - This method can be overridden to achieve a custom naming scheme for generic BaseModels. - - Args: - params: Tuple of types of the class. Given a generic class - `Model` with 2 type variables and a concrete model `Model[str, int]`, - the value `(str, int)` would be passed to `params`. - - Returns: - String representing the new class where `params` are passed to `cls` as type variables. - - Raises: - TypeError: Raised when trying to generate concrete names for non-generic models. - """ - if not issubclass(cls, Generic): - raise TypeError('Concrete names should only be generated for generic models.') - - # Any strings received should represent forward references, so we handle them specially below. - # If we eventually move toward wrapping them in a ForwardRef in __class_getitem__ in the future, - # we may be able to remove this special case. - param_names = [param if isinstance(param, str) else _repr.display_as_type(param) for param in params] - params_component = ', '.join(param_names) - return f'{cls.__name__}[{params_component}]' - - def model_post_init(self, context: Any, /) -> None: - """Override this method to perform additional initialization after `__init__` and `model_construct`. - This is useful if you want to do some validation that requires the entire model to be initialized. - """ - - @classmethod - def model_rebuild( - cls, - *, - force: bool = False, - raise_errors: bool = True, - _parent_namespace_depth: int = 2, - _types_namespace: MappingNamespace | None = None, - ) -> bool | None: - """Try to rebuild the pydantic-core schema for the model. - - This may be necessary when one of the annotations is a ForwardRef which could not be resolved during - the initial attempt to build the schema, and automatic rebuilding fails. - - Args: - force: Whether to force the rebuilding of the model schema, defaults to `False`. - raise_errors: Whether to raise errors, defaults to `True`. - _parent_namespace_depth: The depth level of the parent namespace, defaults to 2. - _types_namespace: The types namespace, defaults to `None`. - - Returns: - Returns `None` if the schema is already "complete" and rebuilding was not required. - If rebuilding _was_ required, returns `True` if rebuilding was successful, otherwise `False`. - """ - already_complete = cls.__pydantic_complete__ - if already_complete and not force: - return None - - cls.__pydantic_complete__ = False - - for attr in ('__pydantic_core_schema__', '__pydantic_validator__', '__pydantic_serializer__'): - if attr in cls.__dict__ and not isinstance(getattr(cls, attr), _mock_val_ser.MockValSer): - # Deleting the validator/serializer is necessary as otherwise they can get reused in - # pydantic-core. We do so only if they aren't mock instances, otherwise — as `model_rebuild()` - # isn't thread-safe — concurrent model instantiations can lead to the parent validator being used. - # Same applies for the core schema that can be reused in schema generation. - delattr(cls, attr) - - if _types_namespace is not None: - rebuild_ns = _types_namespace - elif _parent_namespace_depth > 0: - rebuild_ns = _typing_extra.parent_frame_namespace(parent_depth=_parent_namespace_depth, force=True) or {} - else: - rebuild_ns = {} - - parent_ns = _model_construction.unpack_lenient_weakvaluedict(cls.__pydantic_parent_namespace__) or {} - - ns_resolver = _namespace_utils.NsResolver( - parent_namespace={**rebuild_ns, **parent_ns}, - ) - - return _model_construction.complete_model_class( - cls, - _config.ConfigWrapper(cls.model_config, check=False), - ns_resolver, - raise_errors=raise_errors, - # If the model was already complete, we don't need to call the hook again. - call_on_complete_hook=not already_complete, - is_force_rebuild=force, - ) - - @classmethod - def model_validate( - cls, - obj: Any, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - from_attributes: bool | None = None, - context: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> Self: - """Validate a pydantic model instance. - - Args: - obj: The object to validate. - strict: Whether to enforce types strictly. - extra: Whether to ignore, allow, or forbid extra data during model validation. - See the [`extra` configuration value][pydantic.ConfigDict.extra] for details. - from_attributes: Whether to extract data from object attributes. - context: Additional context to pass to the validator. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Raises: - ValidationError: If the object could not be validated. - - Returns: - The validated model instance. - """ - # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks - __tracebackhide__ = True - - if by_alias is False and by_name is not True: - raise PydanticUserError( - 'At least one of `by_alias` or `by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return cls.__pydantic_validator__.validate_python( - obj, - strict=strict, - extra=extra, - from_attributes=from_attributes, - context=context, - by_alias=by_alias, - by_name=by_name, - ) - - @classmethod - def model_validate_json( - cls, - json_data: str | bytes | bytearray, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - context: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> Self: - """!!! abstract "Usage Documentation" - [JSON Parsing](../concepts/json.md#json-parsing) - - Validate the given JSON data against the Pydantic model. - - Args: - json_data: The JSON data to validate. - strict: Whether to enforce types strictly. - extra: Whether to ignore, allow, or forbid extra data during model validation. - See the [`extra` configuration value][pydantic.ConfigDict.extra] for details. - context: Extra variables to pass to the validator. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Returns: - The validated Pydantic model. - - Raises: - ValidationError: If `json_data` is not a JSON string or the object could not be validated. - """ - # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks - __tracebackhide__ = True - - if by_alias is False and by_name is not True: - raise PydanticUserError( - 'At least one of `by_alias` or `by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return cls.__pydantic_validator__.validate_json( - json_data, strict=strict, extra=extra, context=context, by_alias=by_alias, by_name=by_name - ) - - @classmethod - def model_validate_strings( - cls, - obj: Any, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - context: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> Self: - """Validate the given object with string data against the Pydantic model. - - Args: - obj: The object containing string data to validate. - strict: Whether to enforce types strictly. - extra: Whether to ignore, allow, or forbid extra data during model validation. - See the [`extra` configuration value][pydantic.ConfigDict.extra] for details. - context: Extra variables to pass to the validator. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Returns: - The validated Pydantic model. - """ - # `__tracebackhide__` tells pytest and some other tools to omit this function from tracebacks - __tracebackhide__ = True - - if by_alias is False and by_name is not True: - raise PydanticUserError( - 'At least one of `by_alias` or `by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return cls.__pydantic_validator__.validate_strings( - obj, strict=strict, extra=extra, context=context, by_alias=by_alias, by_name=by_name - ) - - @classmethod - def __get_pydantic_core_schema__(cls, source: type[BaseModel], handler: GetCoreSchemaHandler, /) -> CoreSchema: - # This warning is only emitted when calling `super().__get_pydantic_core_schema__` from a model subclass. - # In the generate schema logic, this method (`BaseModel.__get_pydantic_core_schema__`) is special cased to - # *not* be called if not overridden. - warnings.warn( - 'The `__get_pydantic_core_schema__` method of the `BaseModel` class is deprecated. If you are calling ' - '`super().__get_pydantic_core_schema__` when overriding the method on a Pydantic model, consider using ' - '`handler(source)` instead. However, note that overriding this method on models can lead to unexpected ' - 'side effects.', - PydanticDeprecatedSince211, - stacklevel=2, - ) - # Logic copied over from `GenerateSchema._model_schema`: - schema = cls.__dict__.get('__pydantic_core_schema__') - if schema is not None and not isinstance(schema, _mock_val_ser.MockCoreSchema): - return cls.__pydantic_core_schema__ - - return handler(source) - - @classmethod - def __get_pydantic_json_schema__( - cls, - core_schema: CoreSchema, - handler: GetJsonSchemaHandler, - /, - ) -> JsonSchemaValue: - """Hook into generating the model's JSON schema. - - Args: - core_schema: A `pydantic-core` CoreSchema. - You can ignore this argument and call the handler with a new CoreSchema, - wrap this CoreSchema (`{'type': 'nullable', 'schema': current_schema}`), - or just call the handler with the original schema. - handler: Call into Pydantic's internal JSON schema generation. - This will raise a `pydantic.errors.PydanticInvalidForJsonSchema` if JSON schema - generation fails. - Since this gets called by `BaseModel.model_json_schema` you can override the - `schema_generator` argument to that function to change JSON schema generation globally - for a type. - - Returns: - A JSON schema, as a Python object. - """ - return handler(core_schema) - - @classmethod - def __pydantic_init_subclass__(cls, **kwargs: Any) -> None: - """This is intended to behave just like `__init_subclass__`, but is called by `ModelMetaclass` - only after basic class initialization is complete. In particular, attributes like `model_fields` will - be present when this is called, but forward annotations are not guaranteed to be resolved yet, - meaning that creating an instance of the class may fail. - - This is necessary because `__init_subclass__` will always be called by `type.__new__`, - and it would require a prohibitively large refactor to the `ModelMetaclass` to ensure that - `type.__new__` was called in such a manner that the class would already be sufficiently initialized. - - This will receive the same `kwargs` that would be passed to the standard `__init_subclass__`, namely, - any kwargs passed to the class definition that aren't used internally by Pydantic. - - Args: - **kwargs: Any keyword arguments passed to the class definition that aren't used internally - by Pydantic. - - Note: - You may want to override [`__pydantic_on_complete__()`][pydantic.main.BaseModel.__pydantic_on_complete__] - instead, which is called once the class and its fields are fully initialized and ready for validation. - """ - - @classmethod - def __pydantic_on_complete__(cls) -> None: - """This is called once the class and its fields are fully initialized and ready to be used. - - This typically happens when the class is created (just before - [`__pydantic_init_subclass__()`][pydantic.main.BaseModel.__pydantic_init_subclass__] is called on the superclass), - except when forward annotations are used that could not immediately be resolved. - In that case, it will be called later, when the model is rebuilt automatically or explicitly using - [`model_rebuild()`][pydantic.main.BaseModel.model_rebuild]. - """ - - def __class_getitem__( - cls, typevar_values: type[Any] | tuple[type[Any], ...] - ) -> type[BaseModel] | _forward_ref.PydanticRecursiveRef: - cached = _generics.get_cached_generic_type_early(cls, typevar_values) - if cached is not None: - return cached - - if cls is BaseModel: - raise TypeError('Type parameters should be placed on typing.Generic, not BaseModel') - if not hasattr(cls, '__parameters__'): - raise TypeError(f'{cls} cannot be parametrized because it does not inherit from typing.Generic') - if not cls.__pydantic_generic_metadata__['parameters'] and Generic not in cls.__bases__: - raise TypeError(f'{cls} is not a generic class') - - if not isinstance(typevar_values, tuple): - typevar_values = (typevar_values,) - - # For a model `class Model[T, U, V = int](BaseModel): ...` parametrized with `(str, bool)`, - # this gives us `{T: str, U: bool, V: int}`: - typevars_map = _generics.map_generic_model_arguments(cls, typevar_values) - # We also update the provided args to use defaults values (`(str, bool)` becomes `(str, bool, int)`): - typevar_values = tuple(v for v in typevars_map.values()) - - if _utils.all_identical(typevars_map.keys(), typevars_map.values()) and typevars_map: - submodel = cls # if arguments are equal to parameters it's the same object - _generics.set_cached_generic_type(cls, typevar_values, submodel) - else: - parent_args = cls.__pydantic_generic_metadata__['args'] - if not parent_args: - args = typevar_values - else: - args = tuple(_generics.replace_types(arg, typevars_map) for arg in parent_args) - - origin = cls.__pydantic_generic_metadata__['origin'] or cls - model_name = origin.model_parametrized_name(args) - params = tuple( - dict.fromkeys(_generics.iter_contained_typevars(typevars_map.values())) - ) # use dict as ordered set - - with _generics.generic_recursion_self_type(origin, args) as maybe_self_type: - cached = _generics.get_cached_generic_type_late(cls, typevar_values, origin, args) - if cached is not None: - return cached - - if maybe_self_type is not None: - return maybe_self_type - - # Attempt to rebuild the origin in case new types have been defined - try: - # depth 2 gets you above this __class_getitem__ call. - # Note that we explicitly provide the parent ns, otherwise - # `model_rebuild` will use the parent ns no matter if it is the ns of a module. - # We don't want this here, as this has unexpected effects when a model - # is being parametrized during a forward annotation evaluation. - parent_ns = _typing_extra.parent_frame_namespace(parent_depth=2) or {} - origin.model_rebuild(_types_namespace=parent_ns) - except PydanticUndefinedAnnotation: - # It's okay if it fails, it just means there are still undefined types - # that could be evaluated later. - pass - - submodel = _generics.create_generic_submodel(model_name, origin, args, params) - - _generics.set_cached_generic_type(cls, typevar_values, submodel, origin, args) - - return submodel - - def __copy__(self) -> Self: - """Returns a shallow copy of the model.""" - cls = type(self) - m = cls.__new__(cls) - _object_setattr(m, '__dict__', copy(self.__dict__)) - _object_setattr(m, '__pydantic_extra__', copy(self.__pydantic_extra__)) - _object_setattr(m, '__pydantic_fields_set__', copy(self.__pydantic_fields_set__)) - - if not hasattr(self, '__pydantic_private__') or self.__pydantic_private__ is None: - _object_setattr(m, '__pydantic_private__', None) - else: - _object_setattr( - m, - '__pydantic_private__', - {k: v for k, v in self.__pydantic_private__.items() if v is not PydanticUndefined}, - ) - - return m - - def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: - """Returns a deep copy of the model.""" - cls = type(self) - m = cls.__new__(cls) - _object_setattr(m, '__dict__', deepcopy(self.__dict__, memo=memo)) - _object_setattr(m, '__pydantic_extra__', deepcopy(self.__pydantic_extra__, memo=memo)) - # This next line doesn't need a deepcopy because __pydantic_fields_set__ is a set[str], - # and attempting a deepcopy would be marginally slower. - _object_setattr(m, '__pydantic_fields_set__', copy(self.__pydantic_fields_set__)) - - if not hasattr(self, '__pydantic_private__') or self.__pydantic_private__ is None: - _object_setattr(m, '__pydantic_private__', None) - else: - _object_setattr( - m, - '__pydantic_private__', - deepcopy({k: v for k, v in self.__pydantic_private__.items() if v is not PydanticUndefined}, memo=memo), - ) - - return m - - if not TYPE_CHECKING: - # We put `__getattr__` in a non-TYPE_CHECKING block because otherwise, mypy allows arbitrary attribute access - # The same goes for __setattr__ and __delattr__, see: https://github.com/pydantic/pydantic/issues/8643 - - def __getattr__(self, item: str) -> Any: - private_attributes = object.__getattribute__(self, '__private_attributes__') - if item in private_attributes: - attribute = private_attributes[item] - if hasattr(attribute, '__get__'): - return attribute.__get__(self, type(self)) # type: ignore - - try: - # Note: self.__pydantic_private__ cannot be None if self.__private_attributes__ has items - return self.__pydantic_private__[item] # type: ignore - except KeyError as exc: - raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}') from exc - else: - # `__pydantic_extra__` can fail to be set if the model is not yet fully initialized. - # See `BaseModel.__repr_args__` for more details - try: - pydantic_extra = object.__getattribute__(self, '__pydantic_extra__') - except AttributeError: - pydantic_extra = None - - if pydantic_extra and item in pydantic_extra: - return pydantic_extra[item] - else: - if hasattr(self.__class__, item): - return super().__getattribute__(item) # Raises AttributeError if appropriate - else: - # this is the current error - raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}') - - def __setattr__(self, name: str, value: Any) -> None: - if (setattr_handler := self.__pydantic_setattr_handlers__.get(name)) is not None: - setattr_handler(self, name, value) - # if None is returned from _setattr_handler, the attribute was set directly - elif (setattr_handler := self._setattr_handler(name, value)) is not None: - setattr_handler(self, name, value) # call here to not memo on possibly unknown fields - self.__pydantic_setattr_handlers__[name] = setattr_handler # memoize the handler for faster access - - def _setattr_handler(self, name: str, value: Any) -> Callable[[BaseModel, str, Any], None] | None: - """Get a handler for setting an attribute on the model instance. - - Returns: - A handler for setting an attribute on the model instance. Used for memoization of the handler. - Memoizing the handlers leads to a dramatic performance improvement in `__setattr__` - Returns `None` when memoization is not safe, then the attribute is set directly. - """ - cls = self.__class__ - if name in cls.__class_vars__: - raise AttributeError( - f'{name!r} is a ClassVar of `{cls.__name__}` and cannot be set on an instance. ' - f'If you want to set a value on the class, use `{cls.__name__}.{name} = value`.' - ) - elif not _fields.is_valid_field_name(name): - if (attribute := cls.__private_attributes__.get(name)) is not None: - if hasattr(attribute, '__set__'): - return lambda model, _name, val: attribute.__set__(model, val) - else: - return _SIMPLE_SETATTR_HANDLERS['private'] - else: - _object_setattr(self, name, value) - return None # Can not return memoized handler with possibly freeform attr names - - attr = getattr(cls, name, None) - # NOTE: We currently special case properties and `cached_property`, but we might need - # to generalize this to all data/non-data descriptors at some point. For non-data descriptors - # (such as `cached_property`), it isn't obvious though. `cached_property` caches the value - # to the instance's `__dict__`, but other non-data descriptors might do things differently. - if isinstance(attr, cached_property): - return _SIMPLE_SETATTR_HANDLERS['cached_property'] - - _check_frozen(cls, name, value) - - # We allow properties to be set only on non frozen models for now (to match dataclasses). - # This can be changed if it ever gets requested. - if isinstance(attr, property): - return lambda model, _name, val: attr.__set__(model, val) - elif cls.model_config.get('validate_assignment'): - return _SIMPLE_SETATTR_HANDLERS['validate_assignment'] - elif name not in cls.__pydantic_fields__: - if cls.model_config.get('extra') != 'allow': - # TODO - matching error - raise ValueError(f'"{cls.__name__}" object has no field "{name}"') - elif attr is None: - # attribute does not exist, so put it in extra - self.__pydantic_extra__[name] = value - self.__pydantic_fields_set__.add(name) - return None # Can not return memoized handler with possibly freeform attr names - else: - # attribute _does_ exist, and was not in extra, so update it - return _SIMPLE_SETATTR_HANDLERS['extra_known'] - else: - return _SIMPLE_SETATTR_HANDLERS['model_field'] - - def __delattr__(self, item: str) -> Any: - cls = self.__class__ - - if item in self.__private_attributes__: - attribute = self.__private_attributes__[item] - if hasattr(attribute, '__delete__'): - attribute.__delete__(self) # type: ignore - return - - try: - # Note: self.__pydantic_private__ cannot be None if self.__private_attributes__ has items - del self.__pydantic_private__[item] # type: ignore - return - except KeyError as exc: - raise AttributeError(f'{cls.__name__!r} object has no attribute {item!r}') from exc - - # Allow cached properties to be deleted (even if the class is frozen): - attr = getattr(cls, item, None) - if isinstance(attr, cached_property): - return object.__delattr__(self, item) - - _check_frozen(cls, name=item, value=None) - - if item in self.__pydantic_fields__: - object.__delattr__(self, item) - elif self.__pydantic_extra__ is not None and item in self.__pydantic_extra__: - del self.__pydantic_extra__[item] - else: - try: - object.__delattr__(self, item) - except AttributeError: - raise AttributeError(f'{type(self).__name__!r} object has no attribute {item!r}') - - # Because we make use of `@dataclass_transform()`, `__replace__` is already synthesized by - # type checkers, so we define the implementation in this `if not TYPE_CHECKING:` block: - def __replace__(self, **changes: Any) -> Self: - return self.model_copy(update=changes) - - def __getstate__(self) -> dict[Any, Any]: - private = self.__pydantic_private__ - if private: - private = {k: v for k, v in private.items() if v is not PydanticUndefined} - return { - '__dict__': self.__dict__, - '__pydantic_extra__': self.__pydantic_extra__, - '__pydantic_fields_set__': self.__pydantic_fields_set__, - '__pydantic_private__': private, - } - - def __setstate__(self, state: dict[Any, Any]) -> None: - _object_setattr(self, '__pydantic_fields_set__', state.get('__pydantic_fields_set__', {})) - _object_setattr(self, '__pydantic_extra__', state.get('__pydantic_extra__', {})) - _object_setattr(self, '__pydantic_private__', state.get('__pydantic_private__', {})) - _object_setattr(self, '__dict__', state.get('__dict__', {})) - - if not TYPE_CHECKING: - - def __eq__(self, other: Any) -> bool: - if isinstance(other, BaseModel): - # When comparing instances of generic types for equality, as long as all field values are equal, - # only require their generic origin types to be equal, rather than exact type equality. - # This prevents headaches like MyGeneric(x=1) != MyGeneric[Any](x=1). - self_type = self.__pydantic_generic_metadata__['origin'] or self.__class__ - other_type = other.__pydantic_generic_metadata__['origin'] or other.__class__ - - # Perform common checks first - if not ( - self_type is other_type - and getattr(self, '__pydantic_private__', None) == getattr(other, '__pydantic_private__', None) - # We need to assume `None` and `{}` are equivalent, because extra behavior - # can be controlled at validation time: - and (self.__pydantic_extra__ or {}) == (other.__pydantic_extra__ or {}) - ): - return False - - # We only want to compare pydantic fields but ignoring fields is costly. - # We'll perform a fast check first, and fallback only when needed - # See GH-7444 and GH-7825 for rationale and a performance benchmark - - # First, do the fast (and sometimes faulty) __dict__ comparison - if self.__dict__ == other.__dict__: - # If the check above passes, then pydantic fields are equal, we can return early - return True - - # We don't want to trigger unnecessary costly filtering of __dict__ on all unequal objects, so we return - # early if there are no keys to ignore (we would just return False later on anyway) - model_fields = type(self).__pydantic_fields__.keys() - if self.__dict__.keys() <= model_fields and other.__dict__.keys() <= model_fields: - return False - - # If we reach here, there are non-pydantic-fields keys, mapped to unequal values, that we need to ignore - # Resort to costly filtering of the __dict__ objects - # We use operator.itemgetter because it is much faster than dict comprehensions - # NOTE: Contrary to standard python class and instances, when the Model class has a default value for an - # attribute and the model instance doesn't have a corresponding attribute, accessing the missing attribute - # raises an error in BaseModel.__getattr__ instead of returning the class attribute - # So we can use operator.itemgetter() instead of operator.attrgetter() - getter = operator.itemgetter(*model_fields) if model_fields else lambda _: _utils._SENTINEL - try: - return getter(self.__dict__) == getter(other.__dict__) - except KeyError: - # In rare cases (such as when using the deprecated BaseModel.copy() method), - # the __dict__ may not contain all model fields, which is how we can get here. - # getter(self.__dict__) is much faster than any 'safe' method that accounts - # for missing keys, and wrapping it in a `try` doesn't slow things down much - # in the common case. - self_fields_proxy = _utils.SafeGetItemProxy(self.__dict__) - other_fields_proxy = _utils.SafeGetItemProxy(other.__dict__) - return getter(self_fields_proxy) == getter(other_fields_proxy) - - # other instance is not a BaseModel - else: - return NotImplemented # delegate to the other item in the comparison - - if TYPE_CHECKING: - # We put `__init_subclass__` in a TYPE_CHECKING block because, even though we want the type-checking benefits - # described in the signature of `__init_subclass__` below, we don't want to modify the default behavior of - # subclass initialization. - - def __init_subclass__(cls, **kwargs: Unpack[ConfigDict]): - """This signature is included purely to help type-checkers check arguments to class declaration, which - provides a way to conveniently set model_config key/value pairs. - - ```python - from pydantic import BaseModel - - class MyModel(BaseModel, extra='allow'): ... - ``` - - However, this may be deceiving, since the _actual_ calls to `__init_subclass__` will not receive any - of the config arguments, and will only receive any keyword arguments passed during class initialization - that are _not_ expected keys in ConfigDict. (This is due to the way `ModelMetaclass.__new__` works.) - - Args: - **kwargs: Keyword arguments passed to the class definition, which set model_config - - Note: - You may want to override `__pydantic_init_subclass__` instead, which behaves similarly but is called - *after* the class is fully initialized. - """ - - def __iter__(self) -> TupleGenerator: - """So `dict(model)` works.""" - yield from [(k, v) for (k, v) in self.__dict__.items() if not k.startswith('_')] - extra = self.__pydantic_extra__ - if extra: - yield from extra.items() - - def __repr__(self) -> str: - return f'{self.__repr_name__()}({self.__repr_str__(", ")})' - - def __repr_args__(self) -> _repr.ReprArgs: - # Eagerly create the repr of computed fields, as this may trigger access of cached properties and as such - # modify the instance's `__dict__`. If we don't do it now, it could happen when iterating over the `__dict__` - # below if the instance happens to be referenced in a field, and would modify the `__dict__` size *during* iteration. - computed_fields_repr_args = [ - (k, getattr(self, k)) for k, v in self.__pydantic_computed_fields__.items() if v.repr - ] - - for k, v in self.__dict__.items(): - field = self.__pydantic_fields__.get(k) - if field and field.repr: - if v is not self: - yield k, v - else: - yield k, self.__repr_recursion__(v) - # `__pydantic_extra__` can fail to be set if the model is not yet fully initialized. - # This can happen if a `ValidationError` is raised during initialization and the instance's - # repr is generated as part of the exception handling. Therefore, we use `getattr` here - # with a fallback, even though the type hints indicate the attribute will always be present. - try: - pydantic_extra = object.__getattribute__(self, '__pydantic_extra__') - except AttributeError: - pydantic_extra = None - - if pydantic_extra is not None: - yield from ((k, v) for k, v in pydantic_extra.items()) - yield from computed_fields_repr_args - - # take logic from `_repr.Representation` without the side effects of inheritance, see #5740 - __repr_name__ = _repr.Representation.__repr_name__ - __repr_recursion__ = _repr.Representation.__repr_recursion__ - __repr_str__ = _repr.Representation.__repr_str__ - __pretty__ = _repr.Representation.__pretty__ - __rich_repr__ = _repr.Representation.__rich_repr__ - - def __str__(self) -> str: - return self.__repr_str__(' ') - - # ##### Deprecated methods from v1 ##### - @property - @typing_extensions.deprecated( - 'The `__fields__` attribute is deprecated, use the `model_fields` class property instead.', category=None - ) - def __fields__(self) -> dict[str, FieldInfo]: - warnings.warn( - 'The `__fields__` attribute is deprecated, use the `model_fields` class property instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return getattr(type(self), '__pydantic_fields__', {}) - - @property - @typing_extensions.deprecated( - 'The `__fields_set__` attribute is deprecated, use `model_fields_set` instead.', - category=None, - ) - def __fields_set__(self) -> set[str]: - warnings.warn( - 'The `__fields_set__` attribute is deprecated, use `model_fields_set` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return self.__pydantic_fields_set__ - - @typing_extensions.deprecated('The `dict` method is deprecated; use `model_dump` instead.', category=None) - def dict( # noqa: D102 - self, - *, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = False, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> Dict[str, Any]: # noqa UP006 - warnings.warn( - 'The `dict` method is deprecated; use `model_dump` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return self.model_dump( - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - @typing_extensions.deprecated('The `json` method is deprecated; use `model_dump_json` instead.', category=None) - def json( # noqa: D102 - self, - *, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool = False, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - encoder: Callable[[Any], Any] | None = PydanticUndefined, # type: ignore[assignment] - models_as_dict: bool = PydanticUndefined, # type: ignore[assignment] - **dumps_kwargs: Any, - ) -> str: - warnings.warn( - 'The `json` method is deprecated; use `model_dump_json` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - if encoder is not PydanticUndefined: - raise TypeError('The `encoder` argument is no longer supported; use field serializers instead.') - if models_as_dict is not PydanticUndefined: - raise TypeError('The `models_as_dict` argument is no longer supported; use a model serializer instead.') - if dumps_kwargs: - raise TypeError('`dumps_kwargs` keyword arguments are no longer supported.') - return self.model_dump_json( - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - - @classmethod - @typing_extensions.deprecated('The `parse_obj` method is deprecated; use `model_validate` instead.', category=None) - def parse_obj(cls, obj: Any) -> Self: # noqa: D102 - warnings.warn( - 'The `parse_obj` method is deprecated; use `model_validate` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return cls.model_validate(obj) - - @classmethod - @typing_extensions.deprecated( - 'The `parse_raw` method is deprecated; if your data is JSON use `model_validate_json`, ' - 'otherwise load the data then use `model_validate` instead.', - category=None, - ) - def parse_raw( # noqa: D102 - cls, - b: str | bytes, - *, - content_type: str | None = None, - encoding: str = 'utf8', - proto: DeprecatedParseProtocol | None = None, - allow_pickle: bool = False, - ) -> Self: # pragma: no cover - warnings.warn( - 'The `parse_raw` method is deprecated; if your data is JSON use `model_validate_json`, ' - 'otherwise load the data then use `model_validate` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import parse - - try: - obj = parse.load_str_bytes( - b, - proto=proto, - content_type=content_type, - encoding=encoding, - allow_pickle=allow_pickle, - ) - except (ValueError, TypeError) as exc: - import json - - # try to match V1 - if isinstance(exc, UnicodeDecodeError): - type_str = 'value_error.unicodedecode' - elif isinstance(exc, json.JSONDecodeError): - type_str = 'value_error.jsondecode' - elif isinstance(exc, ValueError): - type_str = 'value_error' - else: - type_str = 'type_error' - - # ctx is missing here, but since we've added `input` to the error, we're not pretending it's the same - error: pydantic_core.InitErrorDetails = { - # The type: ignore on the next line is to ignore the requirement of LiteralString - 'type': pydantic_core.PydanticCustomError(type_str, str(exc)), # type: ignore - 'loc': ('__root__',), - 'input': b, - } - raise pydantic_core.ValidationError.from_exception_data(cls.__name__, [error]) - return cls.model_validate(obj) - - @classmethod - @typing_extensions.deprecated( - 'The `parse_file` method is deprecated; load the data from file, then if your data is JSON ' - 'use `model_validate_json`, otherwise `model_validate` instead.', - category=None, - ) - def parse_file( # noqa: D102 - cls, - path: str | Path, - *, - content_type: str | None = None, - encoding: str = 'utf8', - proto: DeprecatedParseProtocol | None = None, - allow_pickle: bool = False, - ) -> Self: - warnings.warn( - 'The `parse_file` method is deprecated; load the data from file, then if your data is JSON ' - 'use `model_validate_json`, otherwise `model_validate` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import parse - - obj = parse.load_file( - path, - proto=proto, - content_type=content_type, - encoding=encoding, - allow_pickle=allow_pickle, - ) - return cls.parse_obj(obj) - - @classmethod - @typing_extensions.deprecated( - 'The `from_orm` method is deprecated; set ' - "`model_config['from_attributes']=True` and use `model_validate` instead.", - category=None, - ) - def from_orm(cls, obj: Any) -> Self: # noqa: D102 - warnings.warn( - 'The `from_orm` method is deprecated; set ' - "`model_config['from_attributes']=True` and use `model_validate` instead.", - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - if not cls.model_config.get('from_attributes', None): - raise PydanticUserError( - 'You must set the config attribute `from_attributes=True` to use from_orm', code=None - ) - return cls.model_validate(obj) - - @classmethod - @typing_extensions.deprecated('The `construct` method is deprecated; use `model_construct` instead.', category=None) - def construct(cls, _fields_set: set[str] | None = None, **values: Any) -> Self: # noqa: D102 - warnings.warn( - 'The `construct` method is deprecated; use `model_construct` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return cls.model_construct(_fields_set=_fields_set, **values) - - @typing_extensions.deprecated( - 'The `copy` method is deprecated; use `model_copy` instead. ' - 'See the docstring of `BaseModel.copy` for details about how to handle `include` and `exclude`.', - category=None, - ) - def copy( - self, - *, - include: AbstractSetIntStr | MappingIntStrAny | None = None, - exclude: AbstractSetIntStr | MappingIntStrAny | None = None, - update: Dict[str, Any] | None = None, # noqa UP006 - deep: bool = False, - ) -> Self: # pragma: no cover - """Returns a copy of the model. - - !!! warning "Deprecated" - This method is now deprecated; use `model_copy` instead. - - If you need `include` or `exclude`, use: - - ```python {test="skip" lint="skip"} - data = self.model_dump(include=include, exclude=exclude, round_trip=True) - data = {**data, **(update or {})} - copied = self.model_validate(data) - ``` - - Args: - include: Optional set or mapping specifying which fields to include in the copied model. - exclude: Optional set or mapping specifying which fields to exclude in the copied model. - update: Optional dictionary of field-value pairs to override field values in the copied model. - deep: If True, the values of fields that are Pydantic models will be deep-copied. - - Returns: - A copy of the model with included, excluded and updated fields as specified. - """ - warnings.warn( - 'The `copy` method is deprecated; use `model_copy` instead. ' - 'See the docstring of `BaseModel.copy` for details about how to handle `include` and `exclude`.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import copy_internals - - values = dict( - copy_internals._iter( - self, to_dict=False, by_alias=False, include=include, exclude=exclude, exclude_unset=False - ), - **(update or {}), - ) - if self.__pydantic_private__ is None: - private = None - else: - private = {k: v for k, v in self.__pydantic_private__.items() if v is not PydanticUndefined} - - if self.__pydantic_extra__ is None: - extra: dict[str, Any] | None = None - else: - extra = self.__pydantic_extra__.copy() - for k in list(self.__pydantic_extra__): - if k not in values: # k was in the exclude - extra.pop(k) - for k in list(values): - if k in self.__pydantic_extra__: # k must have come from extra - extra[k] = values.pop(k) - - # new `__pydantic_fields_set__` can have unset optional fields with a set value in `update` kwarg - if update: - fields_set = self.__pydantic_fields_set__ | update.keys() - else: - fields_set = set(self.__pydantic_fields_set__) - - # removing excluded fields from `__pydantic_fields_set__` - if exclude: - fields_set -= set(exclude) - - return copy_internals._copy_and_set_values(self, values, fields_set, extra, private, deep=deep) - - @classmethod - @typing_extensions.deprecated('The `schema` method is deprecated; use `model_json_schema` instead.', category=None) - def schema( # noqa: D102 - cls, by_alias: bool = True, ref_template: str = DEFAULT_REF_TEMPLATE - ) -> Dict[str, Any]: # noqa UP006 - warnings.warn( - 'The `schema` method is deprecated; use `model_json_schema` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return cls.model_json_schema(by_alias=by_alias, ref_template=ref_template) - - @classmethod - @typing_extensions.deprecated( - 'The `schema_json` method is deprecated; use `model_json_schema` and json.dumps instead.', - category=None, - ) - def schema_json( # noqa: D102 - cls, *, by_alias: bool = True, ref_template: str = DEFAULT_REF_TEMPLATE, **dumps_kwargs: Any - ) -> str: # pragma: no cover - warnings.warn( - 'The `schema_json` method is deprecated; use `model_json_schema` and json.dumps instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - import json - - from .deprecated.json import pydantic_encoder - - return json.dumps( - cls.model_json_schema(by_alias=by_alias, ref_template=ref_template), - default=pydantic_encoder, - **dumps_kwargs, - ) - - @classmethod - @typing_extensions.deprecated('The `validate` method is deprecated; use `model_validate` instead.', category=None) - def validate(cls, value: Any) -> Self: # noqa: D102 - warnings.warn( - 'The `validate` method is deprecated; use `model_validate` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - return cls.model_validate(value) - - @classmethod - @typing_extensions.deprecated( - 'The `update_forward_refs` method is deprecated; use `model_rebuild` instead.', - category=None, - ) - def update_forward_refs(cls, **localns: Any) -> None: # noqa: D102 - warnings.warn( - 'The `update_forward_refs` method is deprecated; use `model_rebuild` instead.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - if localns: # pragma: no cover - raise TypeError('`localns` arguments are not longer accepted.') - cls.model_rebuild(force=True) - - @typing_extensions.deprecated( - 'The private method `_iter` will be removed and should no longer be used.', category=None - ) - def _iter(self, *args: Any, **kwargs: Any) -> Any: - warnings.warn( - 'The private method `_iter` will be removed and should no longer be used.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import copy_internals - - return copy_internals._iter(self, *args, **kwargs) - - @typing_extensions.deprecated( - 'The private method `_copy_and_set_values` will be removed and should no longer be used.', - category=None, - ) - def _copy_and_set_values(self, *args: Any, **kwargs: Any) -> Any: - warnings.warn( - 'The private method `_copy_and_set_values` will be removed and should no longer be used.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import copy_internals - - return copy_internals._copy_and_set_values(self, *args, **kwargs) - - @classmethod - @typing_extensions.deprecated( - 'The private method `_get_value` will be removed and should no longer be used.', - category=None, - ) - def _get_value(cls, *args: Any, **kwargs: Any) -> Any: - warnings.warn( - 'The private method `_get_value` will be removed and should no longer be used.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import copy_internals - - return copy_internals._get_value(cls, *args, **kwargs) - - @typing_extensions.deprecated( - 'The private method `_calculate_keys` will be removed and should no longer be used.', - category=None, - ) - def _calculate_keys(self, *args: Any, **kwargs: Any) -> Any: - warnings.warn( - 'The private method `_calculate_keys` will be removed and should no longer be used.', - category=PydanticDeprecatedSince20, - stacklevel=2, - ) - from .deprecated import copy_internals - - return copy_internals._calculate_keys(self, *args, **kwargs) - - -ModelT = TypeVar('ModelT', bound=BaseModel) - - -@overload -def create_model( - model_name: str, - /, - *, - __config__: ConfigDict | None = None, - __doc__: str | None = None, - __base__: None = None, - __module__: str = __name__, - __validators__: dict[str, Callable[..., Any]] | None = None, - __cls_kwargs__: dict[str, Any] | None = None, - __qualname__: str | None = None, - **field_definitions: Any | tuple[Any, Any], -) -> type[BaseModel]: ... - - -@overload -def create_model( - model_name: str, - /, - *, - __config__: ConfigDict | None = None, - __doc__: str | None = None, - __base__: type[ModelT] | tuple[type[ModelT], ...], - __module__: str = __name__, - __validators__: dict[str, Callable[..., Any]] | None = None, - __cls_kwargs__: dict[str, Any] | None = None, - __qualname__: str | None = None, - **field_definitions: Any | tuple[Any, Any], -) -> type[ModelT]: ... - - -def create_model( # noqa: C901 - model_name: str, - /, - *, - __config__: ConfigDict | None = None, - __doc__: str | None = None, - __base__: type[ModelT] | tuple[type[ModelT], ...] | None = None, - __module__: str | None = None, - __validators__: dict[str, Callable[..., Any]] | None = None, - __cls_kwargs__: dict[str, Any] | None = None, - __qualname__: str | None = None, - # TODO PEP 747: replace `Any` by the TypeForm: - **field_definitions: Any | tuple[Any, Any], -) -> type[ModelT]: - """!!! abstract "Usage Documentation" - [Dynamic Model Creation](../concepts/models.md#dynamic-model-creation) - - Dynamically creates and returns a new Pydantic model, in other words, `create_model` dynamically creates a - subclass of [`BaseModel`][pydantic.BaseModel]. - - !!! warning - This function may execute arbitrary code contained in field annotations, if string references need to be evaluated. - - See [Security implications of introspecting annotations](https://docs.python.org/3/library/annotationlib.html#annotationlib-security) for more information. - - Args: - model_name: The name of the newly created model. - __config__: The configuration of the new model. - __doc__: The docstring of the new model. - __base__: The base class or classes for the new model. - __module__: The name of the module that the model belongs to; - if `None`, the value is taken from `sys._getframe(1)` - __validators__: A dictionary of methods that validate fields. The keys are the names of the validation methods to - be added to the model, and the values are the validation methods themselves. You can read more about functional - validators [here](https://docs.pydantic.dev/2.9/concepts/validators/#field-validators). - __cls_kwargs__: A dictionary of keyword arguments for class creation, such as `metaclass`. - __qualname__: The qualified name of the newly created model. - **field_definitions: Field definitions of the new model. Either: - - - a single element, representing the type annotation of the field. - - a two-tuple, the first element being the type and the second element the assigned value - (either a default or the [`Field()`][pydantic.Field] function). - - Returns: - The new [model][pydantic.BaseModel]. - - Raises: - PydanticUserError: If `__base__` and `__config__` are both passed. - """ - if __base__ is None: - __base__ = (cast('type[ModelT]', BaseModel),) - elif not isinstance(__base__, tuple): - __base__ = (__base__,) - - __cls_kwargs__ = __cls_kwargs__ or {} - - fields: dict[str, Any] = {} - annotations: dict[str, Any] = {} - - for f_name, f_def in field_definitions.items(): - if isinstance(f_def, tuple): - if len(f_def) != 2: - raise PydanticUserError( - f'Field definition for {f_name!r} should a single element representing the type or a two-tuple, the first element ' - 'being the type and the second element the assigned value (either a default or the `Field()` function).', - code='create-model-field-definitions', - ) - - annotations[f_name] = f_def[0] - fields[f_name] = f_def[1] - else: - annotations[f_name] = f_def - - if __module__ is None: - f = sys._getframe(1) - __module__ = f.f_globals['__name__'] - - namespace: dict[str, Any] = {'__annotations__': annotations, '__module__': __module__} - if __doc__: - namespace['__doc__'] = __doc__ - if __qualname__ is not None: - namespace['__qualname__'] = __qualname__ - if __validators__: - namespace.update(__validators__) - namespace.update(fields) - if __config__: - namespace['model_config'] = __config__ - resolved_bases = types.resolve_bases(__base__) - meta, ns, kwds = types.prepare_class(model_name, resolved_bases, kwds=__cls_kwargs__) - if resolved_bases is not __base__: - ns['__orig_bases__'] = __base__ - namespace.update(ns) - - return meta( - model_name, - resolved_bases, - namespace, - __pydantic_reset_parent_namespace__=False, - _create_model_module=__module__, - **kwds, - ) - - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/mypy.py b/functions/python/data-processor/pydantic/mypy.py deleted file mode 100644 index ed4c30869..000000000 --- a/functions/python/data-processor/pydantic/mypy.py +++ /dev/null @@ -1,1412 +0,0 @@ -"""This module includes classes and functions designed specifically for use with the mypy plugin.""" - -from __future__ import annotations - -import sys -from collections.abc import Iterator -from configparser import ConfigParser -from typing import Any, Callable - -from mypy.errorcodes import ErrorCode -from mypy.expandtype import expand_type, expand_type_by_instance -from mypy.nodes import ( - ARG_NAMED, - ARG_NAMED_OPT, - ARG_OPT, - ARG_POS, - ARG_STAR2, - INVARIANT, - MDEF, - Argument, - AssignmentStmt, - Block, - CallExpr, - ClassDef, - Context, - Decorator, - DictExpr, - EllipsisExpr, - Expression, - FuncDef, - IfStmt, - JsonDict, - MemberExpr, - NameExpr, - PassStmt, - PlaceholderNode, - RefExpr, - Statement, - StrExpr, - SymbolTableNode, - TempNode, - TypeAlias, - TypeInfo, - Var, -) -from mypy.options import Options -from mypy.plugin import ( - CheckerPluginInterface, - ClassDefContext, - DynamicClassDefContext, - MethodContext, - Plugin, - ReportConfigContext, - SemanticAnalyzerPluginInterface, -) -from mypy.plugins.common import ( - deserialize_and_fixup_type, -) -from mypy.semanal import set_callable_name -from mypy.server.trigger import make_wildcard_trigger -from mypy.state import state -from mypy.type_visitor import TypeTranslator -from mypy.typeops import map_type_from_supertype -from mypy.types import ( - AnyType, - CallableType, - Instance, - NoneType, - Type, - TypeOfAny, - TypeType, - TypeVarType, - UnionType, - get_proper_type, -) -from mypy.typevars import fill_typevars -from mypy.util import get_unique_redefinition_name -from mypy.version import __version__ as mypy_version - -from pydantic._internal import _fields -from pydantic.version import parse_mypy_version - -CONFIGFILE_KEY = 'pydantic-mypy' -METADATA_KEY = 'pydantic-mypy-metadata' -BASEMODEL_FULLNAME = 'pydantic.main.BaseModel' -CREATE_MODEL_FULLNAME = 'pydantic.main.create_model' -BASESETTINGS_FULLNAME = 'pydantic_settings.main.BaseSettings' -ROOT_MODEL_FULLNAME = 'pydantic.root_model.RootModel' -MODEL_METACLASS_FULLNAME = 'pydantic._internal._model_construction.ModelMetaclass' -FIELD_FULLNAME = 'pydantic.fields.Field' -DATACLASS_FULLNAME = 'pydantic.dataclasses.dataclass' -MODEL_VALIDATOR_FULLNAME = 'pydantic.functional_validators.model_validator' -DECORATOR_FULLNAMES = { - 'pydantic.functional_validators.field_validator', - 'pydantic.functional_validators.model_validator', - 'pydantic.functional_serializers.serializer', - 'pydantic.functional_serializers.model_serializer', - 'pydantic.deprecated.class_validators.validator', - 'pydantic.deprecated.class_validators.root_validator', -} -IMPLICIT_CLASSMETHOD_DECORATOR_FULLNAMES = DECORATOR_FULLNAMES - {'pydantic.functional_serializers.model_serializer'} - - -MYPY_VERSION_TUPLE = parse_mypy_version(mypy_version) -BUILTINS_NAME = 'builtins' - -# Increment version if plugin changes and mypy caches should be invalidated -__version__ = 2 - - -def plugin(version: str) -> type[Plugin]: - """`version` is the mypy version string. - - We might want to use this to print a warning if the mypy version being used is - newer, or especially older, than we expect (or need). - - Args: - version: The mypy version string. - - Return: - The Pydantic mypy plugin type. - """ - return PydanticPlugin - - -class PydanticPlugin(Plugin): - """The Pydantic mypy plugin.""" - - def __init__(self, options: Options) -> None: - self.plugin_config = PydanticPluginConfig(options) - self._plugin_data = self.plugin_config.to_data() - super().__init__(options) - - def get_base_class_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: - """Update Pydantic model class.""" - sym = self.lookup_fully_qualified(fullname) - if sym and isinstance(sym.node, TypeInfo): # pragma: no branch - # No branching may occur if the mypy cache has not been cleared - if sym.node.has_base(BASEMODEL_FULLNAME): - return self._pydantic_model_class_maker_callback - return None - - def get_metaclass_hook(self, fullname: str) -> Callable[[ClassDefContext], None] | None: - """Update Pydantic `ModelMetaclass` definition.""" - if fullname == MODEL_METACLASS_FULLNAME: - return self._pydantic_model_metaclass_marker_callback - return None - - def get_method_hook(self, fullname: str) -> Callable[[MethodContext], Type] | None: - """Adjust return type of `from_orm` method call.""" - if fullname.endswith('.from_orm'): - return from_attributes_callback - return None - - def get_dynamic_class_hook(self, fullname: str) -> Callable[[DynamicClassDefContext], None] | None: - """Recognize `create_model()` calls as dynamic BaseModel subclasses.""" - if fullname == CREATE_MODEL_FULLNAME: - return self._pydantic_create_model_callback - return None - - def report_config_data(self, ctx: ReportConfigContext) -> dict[str, Any]: - """Return all plugin config data. - - Used by mypy to determine if cache needs to be discarded. - """ - return self._plugin_data - - def _pydantic_model_class_maker_callback(self, ctx: ClassDefContext) -> None: - transformer = PydanticModelTransformer(ctx.cls, ctx.reason, ctx.api, self.plugin_config) - transformer.transform() - - def _pydantic_model_metaclass_marker_callback(self, ctx: ClassDefContext) -> None: - """Reset dataclass_transform_spec attribute of ModelMetaclass. - - Let the plugin handle it. This behavior can be disabled - if 'debug_dataclass_transform' is set to True', for testing purposes. - """ - if self.plugin_config.debug_dataclass_transform: - return - info_metaclass = ctx.cls.info.declared_metaclass - assert info_metaclass, "callback not passed from 'get_metaclass_hook'" - if getattr(info_metaclass.type, 'dataclass_transform_spec', None): - info_metaclass.type.dataclass_transform_spec = None - - def _pydantic_create_model_callback(self, ctx: DynamicClassDefContext) -> None: - """Make variables assigned from `create_model()` usable as types by mypy.""" - # Determine the base class from __base__ argument if provided - base_fullname = BASEMODEL_FULLNAME - for arg_name, arg_expr in zip(ctx.call.arg_names, ctx.call.args): - if arg_name == '__base__' and isinstance(arg_expr, RefExpr) and arg_expr.node is not None: - if isinstance(arg_expr.node, TypeInfo): - base_fullname = arg_expr.node.fullname - elif isinstance(arg_expr.node, Var) and isinstance(arg_expr.node.type, Instance): - base_fullname = arg_expr.node.type.type.fullname - - base_sym = ctx.api.lookup_fully_qualified_or_none(base_fullname) - if base_sym is None or not isinstance(base_sym.node, TypeInfo): - # Fall back to BaseModel - base_sym = ctx.api.lookup_fully_qualified_or_none(BASEMODEL_FULLNAME) - if base_sym is None or not isinstance(base_sym.node, TypeInfo): - return - - base_info = base_sym.node - base_instance = fill_typevars(base_info) - assert isinstance(base_instance, Instance) - - info = ctx.api.basic_new_typeinfo(ctx.name, base_instance, ctx.call.line) - info.metaclass_type = base_info.metaclass_type - - ctx.api.add_symbol_table_node(ctx.name, SymbolTableNode(MDEF, info)) - - -class PydanticPluginConfig: - """A Pydantic mypy plugin config holder. - - Attributes: - init_forbid_extra: Whether to add a `**kwargs` at the end of the generated `__init__` signature. - init_typed: Whether to annotate fields in the generated `__init__`. - warn_required_dynamic_aliases: Whether to raise required dynamic aliases error. - debug_dataclass_transform: Whether to not reset `dataclass_transform_spec` attribute - of `ModelMetaclass` for testing purposes. - """ - - __slots__ = ( - 'init_forbid_extra', - 'init_typed', - 'warn_required_dynamic_aliases', - 'debug_dataclass_transform', - ) - init_forbid_extra: bool - init_typed: bool - warn_required_dynamic_aliases: bool - debug_dataclass_transform: bool # undocumented - - def __init__(self, options: Options) -> None: - if options.config_file is None: # pragma: no cover - return - - toml_config = parse_toml(options.config_file) - if toml_config is not None: - config = toml_config.get('tool', {}).get('pydantic-mypy', {}) - for key in self.__slots__: - setting = config.get(key, False) - if not isinstance(setting, bool): - raise ValueError(f'Configuration value must be a boolean for key: {key}') - setattr(self, key, setting) - else: - plugin_config = ConfigParser() - plugin_config.read(options.config_file) - for key in self.__slots__: - setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False) - setattr(self, key, setting) - - def to_data(self) -> dict[str, Any]: - """Returns a dict of config names to their values.""" - return {key: getattr(self, key) for key in self.__slots__} - - -def from_attributes_callback(ctx: MethodContext) -> Type: - """Raise an error if from_attributes is not enabled.""" - model_type: Instance - ctx_type = ctx.type - if isinstance(ctx_type, TypeType): - ctx_type = ctx_type.item - if isinstance(ctx_type, CallableType) and isinstance(ctx_type.ret_type, Instance): - model_type = ctx_type.ret_type # called on the class - elif isinstance(ctx_type, Instance): - model_type = ctx_type # called on an instance (unusual, but still valid) - else: # pragma: no cover - detail = f'ctx.type: {ctx_type} (of type {ctx_type.__class__.__name__})' - error_unexpected_behavior(detail, ctx.api, ctx.context) - return ctx.default_return_type - pydantic_metadata = model_type.type.metadata.get(METADATA_KEY) - if pydantic_metadata is None: - return ctx.default_return_type - if not model_type.type.has_base(BASEMODEL_FULLNAME): - # not a Pydantic v2 model - return ctx.default_return_type - from_attributes = pydantic_metadata.get('config', {}).get('from_attributes') - if from_attributes is not True: - error_from_attributes(model_type.type.name, ctx.api, ctx.context) - return ctx.default_return_type - - -class PydanticModelField: - """Based on mypy.plugins.dataclasses.DataclassAttribute.""" - - def __init__( - self, - name: str, - alias: str | None, - is_frozen: bool, - has_dynamic_alias: bool, - has_default: bool, - strict: bool | None, - line: int, - column: int, - type: Type | None, - info: TypeInfo, - ): - self.name = name - self.alias = alias - self.is_frozen = is_frozen - self.has_dynamic_alias = has_dynamic_alias - self.has_default = has_default - self.strict = strict - self.line = line - self.column = column - self.type = type - self.info = info - - def to_argument( - self, - current_info: TypeInfo, - typed: bool, - model_strict: bool, - force_optional: bool, - use_alias: bool, - api: SemanticAnalyzerPluginInterface, - force_typevars_invariant: bool, - is_root_model_root: bool, - ) -> Argument: - """Based on mypy.plugins.dataclasses.DataclassAttribute.to_argument.""" - variable = self.to_var(current_info, api, use_alias, force_typevars_invariant) - - strict = model_strict if self.strict is None else self.strict - if typed or strict: - type_annotation = self.expand_type(current_info, api, include_root_type=True) - else: - type_annotation = AnyType(TypeOfAny.explicit) - - return Argument( - variable=variable, - type_annotation=type_annotation, - initializer=None, - kind=ARG_OPT - if is_root_model_root - else (ARG_NAMED_OPT if force_optional or self.has_default else ARG_NAMED), - ) - - def expand_type( - self, - current_info: TypeInfo, - api: SemanticAnalyzerPluginInterface, - force_typevars_invariant: bool = False, - include_root_type: bool = False, - ) -> Type | None: - """Based on mypy.plugins.dataclasses.DataclassAttribute.expand_type.""" - if force_typevars_invariant: - # In some cases, mypy will emit an error "Cannot use a covariant type variable as a parameter" - # To prevent that, we add an option to replace typevars with invariant ones while building certain - # method signatures (in particular, `__init__`). There may be a better way to do this, if this causes - # us problems in the future, we should look into why the dataclasses plugin doesn't have this issue. - if isinstance(self.type, TypeVarType): - modified_type = self.type.copy_modified() - modified_type.variance = INVARIANT - self.type = modified_type - - if self.type is not None and self.info.self_type is not None: - # In general, it is not safe to call `expand_type()` during semantic analysis, - # however this plugin is called very late, so all types should be fully ready. - # Also, it is tricky to avoid eager expansion of Self types here (e.g. because - # we serialize attributes). - with state.strict_optional_set(api.options.strict_optional): - filled_with_typevars = fill_typevars(current_info) - # Cannot be TupleType as current_info represents a Pydantic model: - assert isinstance(filled_with_typevars, Instance) - if force_typevars_invariant: - for arg in filled_with_typevars.args: - if isinstance(arg, TypeVarType): - arg.variance = INVARIANT - - expanded_type = expand_type(self.type, {self.info.self_type.id: filled_with_typevars}) - if include_root_type and isinstance(expanded_type, Instance) and is_root_model(expanded_type.type): - # When a root model is used as a field, Pydantic allows both an instance of the root model - # as well as instances of the `root` field type: - root_type = expanded_type.type['root'].type - if root_type is None: - # Happens if the hint for 'root' has unsolved forward references - return expanded_type - expanded_root_type = expand_type_by_instance(root_type, expanded_type) - expanded_type = UnionType([expanded_type, expanded_root_type]) - return expanded_type - return self.type - - def to_var( - self, - current_info: TypeInfo, - api: SemanticAnalyzerPluginInterface, - use_alias: bool, - force_typevars_invariant: bool = False, - ) -> Var: - """Based on mypy.plugins.dataclasses.DataclassAttribute.to_var.""" - if use_alias and self.alias is not None: - name = self.alias - else: - name = self.name - - return Var(name, self.expand_type(current_info, api, force_typevars_invariant)) - - def serialize(self) -> JsonDict: - """Based on mypy.plugins.dataclasses.DataclassAttribute.serialize.""" - assert self.type - return { - 'name': self.name, - 'alias': self.alias, - 'is_frozen': self.is_frozen, - 'has_dynamic_alias': self.has_dynamic_alias, - 'has_default': self.has_default, - 'strict': self.strict, - 'line': self.line, - 'column': self.column, - 'type': self.type.serialize(), - } - - @classmethod - def deserialize(cls, info: TypeInfo, data: JsonDict, api: SemanticAnalyzerPluginInterface) -> PydanticModelField: - """Based on mypy.plugins.dataclasses.DataclassAttribute.deserialize.""" - data = data.copy() - typ = deserialize_and_fixup_type(data.pop('type'), api) - return cls(type=typ, info=info, **data) - - def expand_typevar_from_subtype(self, sub_type: TypeInfo, api: SemanticAnalyzerPluginInterface) -> None: - """Expands type vars in the context of a subtype when an attribute is inherited - from a generic super type. - """ - if self.type is not None: - with state.strict_optional_set(api.options.strict_optional): - self.type = map_type_from_supertype(self.type, sub_type, self.info) - - -class PydanticModelClassVar: - """Based on mypy.plugins.dataclasses.DataclassAttribute. - - ClassVars are ignored by subclasses. - - Attributes: - name: the ClassVar name - """ - - def __init__(self, name): - self.name = name - - @classmethod - def deserialize(cls, data: JsonDict) -> PydanticModelClassVar: - """Based on mypy.plugins.dataclasses.DataclassAttribute.deserialize.""" - data = data.copy() - return cls(**data) - - def serialize(self) -> JsonDict: - """Based on mypy.plugins.dataclasses.DataclassAttribute.serialize.""" - return { - 'name': self.name, - } - - -class PydanticModelTransformer: - """Transform the BaseModel subclass according to the plugin settings. - - Attributes: - tracked_config_fields: A set of field configs that the plugin has to track their value. - """ - - tracked_config_fields: set[str] = { - 'extra', - 'frozen', - 'from_attributes', - 'populate_by_name', - 'validate_by_alias', - 'validate_by_name', - 'alias_generator', - 'strict', - } - - def __init__( - self, - cls: ClassDef, - reason: Expression | Statement, - api: SemanticAnalyzerPluginInterface, - plugin_config: PydanticPluginConfig, - ) -> None: - self._cls = cls - self._reason = reason - self._api = api - - self.plugin_config = plugin_config - - def transform(self) -> bool: - """Configures the BaseModel subclass according to the plugin settings. - - In particular: - - * determines the model config and fields, - * adds a fields-aware signature for the initializer and construct methods - * freezes the class if frozen = True - * stores the fields, config, and if the class is settings in the mypy metadata for access by subclasses - """ - info = self._cls.info - is_a_root_model = is_root_model(info) - config = self.collect_config() - fields, class_vars = self.collect_fields_and_class_vars(config, is_a_root_model) - if fields is None or class_vars is None: - # Some definitions are not ready. We need another pass. - return False - for field in fields: - if field.type is None: - return False - - is_settings = info.has_base(BASESETTINGS_FULLNAME) - self.add_initializer(fields, config, is_settings, is_a_root_model) - self.add_model_construct_method(fields, config, is_settings, is_a_root_model) - self.set_frozen(fields, self._api, frozen=config.frozen is True) - - self.adjust_decorator_signatures() - - info.metadata[METADATA_KEY] = { - 'fields': {field.name: field.serialize() for field in fields}, - 'class_vars': {class_var.name: class_var.serialize() for class_var in class_vars}, - 'config': config.get_values_dict(), - } - - return True - - def adjust_decorator_signatures(self) -> None: - """When we decorate a function `f` with `pydantic.validator(...)`, `pydantic.field_validator` - or `pydantic.serializer(...)`, mypy sees `f` as a regular method taking a `self` instance, - even though pydantic internally wraps `f` with `classmethod` if necessary. - - Teach mypy this by marking any function whose outermost decorator is a `validator()`, - `field_validator()` or `serializer()` call as a `classmethod`. - """ - for sym in self._cls.info.names.values(): - if isinstance(sym.node, Decorator): - first_dec = sym.node.original_decorators[0] - if ( - isinstance(first_dec, CallExpr) - and isinstance(first_dec.callee, NameExpr) - and first_dec.callee.fullname in IMPLICIT_CLASSMETHOD_DECORATOR_FULLNAMES - # @model_validator(mode="after") is an exception, it expects a regular method - and not ( - first_dec.callee.fullname == MODEL_VALIDATOR_FULLNAME - and any( - first_dec.arg_names[i] == 'mode' and isinstance(arg, StrExpr) and arg.value == 'after' - for i, arg in enumerate(first_dec.args) - ) - ) - ): - # TODO: Only do this if the first argument of the decorated function is `cls` - sym.node.func.is_class = True - - def collect_config(self) -> ModelConfigData: # noqa: C901 (ignore complexity) - """Collects the values of the config attributes that are used by the plugin, accounting for parent classes.""" - cls = self._cls - config = ModelConfigData() - - has_config_kwargs = False - has_config_from_namespace = False - - # Handle `class MyModel(BaseModel, =, ...):` - for name, expr in cls.keywords.items(): - config_data = self.get_config_update(name, expr) - if config_data: - has_config_kwargs = True - config.update(config_data) - - # Handle `model_config` - stmt: Statement | None = None - for stmt in cls.defs.body: - if not isinstance(stmt, (AssignmentStmt, ClassDef)): - continue - - if isinstance(stmt, AssignmentStmt): - lhs = stmt.lvalues[0] - if not isinstance(lhs, NameExpr) or lhs.name != 'model_config': - continue - - if isinstance(stmt.rvalue, CallExpr): # calls to `dict` or `ConfigDict` - for arg_name, arg in zip(stmt.rvalue.arg_names, stmt.rvalue.args): - if arg_name is None: - continue - config.update(self.get_config_update(arg_name, arg, lax_extra=True)) - elif isinstance(stmt.rvalue, DictExpr): # dict literals - for key_expr, value_expr in stmt.rvalue.items: - if not isinstance(key_expr, StrExpr): - continue - config.update(self.get_config_update(key_expr.value, value_expr)) - - elif isinstance(stmt, ClassDef): - if stmt.name != 'Config': # 'deprecated' Config-class - continue - for substmt in stmt.defs.body: - if not isinstance(substmt, AssignmentStmt): - continue - lhs = substmt.lvalues[0] - if not isinstance(lhs, NameExpr): - continue - config.update(self.get_config_update(lhs.name, substmt.rvalue)) - - if has_config_kwargs: - self._api.fail( - 'Specifying config in two places is ambiguous, use either Config attribute or class kwargs', - cls, - ) - break - - has_config_from_namespace = True - - if has_config_kwargs or has_config_from_namespace: - if ( - stmt - and config.has_alias_generator - and not (config.validate_by_name or config.populate_by_name) - and self.plugin_config.warn_required_dynamic_aliases - ): - error_required_dynamic_aliases(self._api, stmt) - - for info in cls.info.mro[1:]: # 0 is the current class - if METADATA_KEY not in info.metadata: - continue - - # Each class depends on the set of fields in its ancestors - self._api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) - for name, value in info.metadata[METADATA_KEY]['config'].items(): - config.setdefault(name, value) - return config - - def collect_fields_and_class_vars( - self, model_config: ModelConfigData, is_root_model: bool - ) -> tuple[list[PydanticModelField] | None, list[PydanticModelClassVar] | None]: - """Collects the fields for the model, accounting for parent classes.""" - cls = self._cls - - # First, collect fields and ClassVars belonging to any class in the MRO, ignoring duplicates. - # - # We iterate through the MRO in reverse because attrs defined in the parent must appear - # earlier in the attributes list than attrs defined in the child. See: - # https://docs.python.org/3/library/dataclasses.html#inheritance - # - # However, we also want fields defined in the subtype to override ones defined - # in the parent. We can implement this via a dict without disrupting the attr order - # because dicts preserve insertion order in Python 3.7+. - found_fields: dict[str, PydanticModelField] = {} - found_class_vars: dict[str, PydanticModelClassVar] = {} - for info in reversed(cls.info.mro[1:-1]): # 0 is the current class, -2 is BaseModel, -1 is object - # if BASEMODEL_METADATA_TAG_KEY in info.metadata and BASEMODEL_METADATA_KEY not in info.metadata: - # # We haven't processed the base class yet. Need another pass. - # return None, None - if METADATA_KEY not in info.metadata: - continue - - # Each class depends on the set of attributes in its dataclass ancestors. - self._api.add_plugin_dependency(make_wildcard_trigger(info.fullname)) - - for name, data in info.metadata[METADATA_KEY]['fields'].items(): - field = PydanticModelField.deserialize(info, data, self._api) - # (The following comment comes directly from the dataclasses plugin) - # TODO: We shouldn't be performing type operations during the main - # semantic analysis pass, since some TypeInfo attributes might - # still be in flux. This should be performed in a later phase. - field.expand_typevar_from_subtype(cls.info, self._api) - found_fields[name] = field - - sym_node = cls.info.names.get(name) - if sym_node and sym_node.node and not isinstance(sym_node.node, (Var, PlaceholderNode)): - self._api.fail( - 'BaseModel field may only be overridden by another field', - sym_node.node, - ) - # Collect ClassVars - for name, data in info.metadata[METADATA_KEY]['class_vars'].items(): - found_class_vars[name] = PydanticModelClassVar.deserialize(data) - - # Second, collect fields and ClassVars belonging to the current class. - current_field_names: set[str] = set() - current_class_vars_names: set[str] = set() - for stmt in self._get_assignment_statements_from_block(cls.defs): - maybe_field = self.collect_field_or_class_var_from_stmt(stmt, model_config, found_class_vars) - if maybe_field is None: - continue - - lhs = stmt.lvalues[0] - assert isinstance(lhs, NameExpr) # collect_field_or_class_var_from_stmt guarantees this - if isinstance(maybe_field, PydanticModelField): - if is_root_model and lhs.name != 'root': - error_extra_fields_on_root_model(self._api, stmt) - else: - current_field_names.add(lhs.name) - found_fields[lhs.name] = maybe_field - elif isinstance(maybe_field, PydanticModelClassVar): - current_class_vars_names.add(lhs.name) - found_class_vars[lhs.name] = maybe_field - - return list(found_fields.values()), list(found_class_vars.values()) - - def _get_assignment_statements_from_if_statement(self, stmt: IfStmt) -> Iterator[AssignmentStmt]: - for body in stmt.body: - if not body.is_unreachable: - yield from self._get_assignment_statements_from_block(body) - if stmt.else_body is not None and not stmt.else_body.is_unreachable: - yield from self._get_assignment_statements_from_block(stmt.else_body) - - def _get_assignment_statements_from_block(self, block: Block) -> Iterator[AssignmentStmt]: - for stmt in block.body: - if isinstance(stmt, AssignmentStmt): - yield stmt - elif isinstance(stmt, IfStmt): - yield from self._get_assignment_statements_from_if_statement(stmt) - - def collect_field_or_class_var_from_stmt( # noqa C901 - self, stmt: AssignmentStmt, model_config: ModelConfigData, class_vars: dict[str, PydanticModelClassVar] - ) -> PydanticModelField | PydanticModelClassVar | None: - """Get pydantic model field from statement. - - Args: - stmt: The statement. - model_config: Configuration settings for the model. - class_vars: ClassVars already known to be defined on the model. - - Returns: - A pydantic model field if it could find the field in statement. Otherwise, `None`. - """ - cls = self._cls - - lhs = stmt.lvalues[0] - if not isinstance(lhs, NameExpr) or not _fields.is_valid_field_name(lhs.name) or lhs.name == 'model_config': - return None - - if not stmt.new_syntax: - if ( - isinstance(stmt.rvalue, CallExpr) - and isinstance(stmt.rvalue.callee, CallExpr) - and isinstance(stmt.rvalue.callee.callee, NameExpr) - and stmt.rvalue.callee.callee.fullname in DECORATOR_FULLNAMES - ): - # This is a (possibly-reused) validator or serializer, not a field - # In particular, it looks something like: my_validator = validator('my_field')(f) - # Eventually, we may want to attempt to respect model_config['ignored_types'] - return None - - if lhs.name in class_vars: - # Class vars are not fields and are not required to be annotated - return None - - # The assignment does not have an annotation, and it's not anything else we recognize - error_untyped_fields(self._api, stmt) - return None - - lhs = stmt.lvalues[0] - if not isinstance(lhs, NameExpr): - return None - - if not _fields.is_valid_field_name(lhs.name) or lhs.name == 'model_config': - return None - - sym = cls.info.names.get(lhs.name) - if sym is None: # pragma: no cover - # This is likely due to a star import (see the dataclasses plugin for a more detailed explanation) - # This is the same logic used in the dataclasses plugin - return None - - node = sym.node - if isinstance(node, PlaceholderNode): # pragma: no cover - # See the PlaceholderNode docstring for more detail about how this can occur - # Basically, it is an edge case when dealing with complex import logic - - # The dataclasses plugin now asserts this cannot happen, but I'd rather not error if it does.. - return None - - if isinstance(node, TypeAlias): - self._api.fail( - 'Type aliases inside BaseModel definitions are not supported at runtime', - node, - ) - # Skip processing this node. This doesn't match the runtime behaviour, - # but the only alternative would be to modify the SymbolTable, - # and it's a little hairy to do that in a plugin. - return None - - if not isinstance(node, Var): # pragma: no cover - # Don't know if this edge case still happens with the `is_valid_field` check above - # but better safe than sorry - - # The dataclasses plugin now asserts this cannot happen, but I'd rather not error if it does.. - return None - - # x: ClassVar[int] is not a field - if node.is_classvar: - return PydanticModelClassVar(lhs.name) - - # x: InitVar[int] is not supported in BaseModel - node_type = get_proper_type(node.type) - if isinstance(node_type, Instance) and node_type.type.fullname == 'dataclasses.InitVar': - self._api.fail( - 'InitVar is not supported in BaseModel', - node, - ) - - has_default = self.get_has_default(stmt) - strict = self.get_strict(stmt) - - if sym.type is None and node.is_final and node.is_inferred: - # This follows the logic from the dataclasses plugin. The following comment is taken verbatim: - # - # This is a special case, assignment like x: Final = 42 is classified - # annotated above, but mypy strips the `Final` turning it into x = 42. - # We do not support inferred types in dataclasses, so we can try inferring - # type for simple literals, and otherwise require an explicit type - # argument for Final[...]. - typ = self._api.analyze_simple_literal_type(stmt.rvalue, is_final=True) - if typ: - node.type = typ - else: - self._api.fail( - 'Need type argument for Final[...] with non-literal default in BaseModel', - stmt, - ) - node.type = AnyType(TypeOfAny.from_error) - - if node.is_final and has_default: - # TODO this path should be removed (see https://github.com/pydantic/pydantic/issues/11119) - return PydanticModelClassVar(lhs.name) - - alias, has_dynamic_alias = self.get_alias_info(stmt) - if ( - has_dynamic_alias - and not (model_config.validate_by_name or model_config.populate_by_name) - and self.plugin_config.warn_required_dynamic_aliases - ): - error_required_dynamic_aliases(self._api, stmt) - is_frozen = self.is_field_frozen(stmt) - - init_type = self._infer_dataclass_attr_init_type(sym, lhs.name, stmt) - return PydanticModelField( - name=lhs.name, - has_dynamic_alias=has_dynamic_alias, - has_default=has_default, - strict=strict, - alias=alias, - is_frozen=is_frozen, - line=stmt.line, - column=stmt.column, - type=init_type, - info=cls.info, - ) - - def _infer_dataclass_attr_init_type(self, sym: SymbolTableNode, name: str, context: Context) -> Type | None: - """Infer __init__ argument type for an attribute. - - In particular, possibly use the signature of __set__. - """ - default = sym.type - if sym.implicit: - return default - t = get_proper_type(sym.type) - - # Perform a simple-minded inference from the signature of __set__, if present. - # We can't use mypy.checkmember here, since this plugin runs before type checking. - # We only support some basic scanerios here, which is hopefully sufficient for - # the vast majority of use cases. - if not isinstance(t, Instance): - return default - setter = t.type.get('__set__') - if setter: - if isinstance(setter.node, FuncDef): - super_info = t.type.get_containing_type_info('__set__') - assert super_info - if setter.type: - setter_type = get_proper_type(map_type_from_supertype(setter.type, t.type, super_info)) - else: - return AnyType(TypeOfAny.unannotated) - if isinstance(setter_type, CallableType) and setter_type.arg_kinds == [ - ARG_POS, - ARG_POS, - ARG_POS, - ]: - return expand_type_by_instance(setter_type.arg_types[2], t) - else: - self._api.fail(f'Unsupported signature for "__set__" in "{t.type.name}"', context) - else: - self._api.fail(f'Unsupported "__set__" in "{t.type.name}"', context) - - return default - - def add_initializer( - self, fields: list[PydanticModelField], config: ModelConfigData, is_settings: bool, is_root_model: bool - ) -> None: - """Adds a fields-aware `__init__` method to the class. - - The added `__init__` will be annotated with types vs. all `Any` depending on the plugin settings. - """ - if '__init__' in self._cls.info.names and not self._cls.info.names['__init__'].plugin_generated: - return # Don't generate an __init__ if one already exists - - typed = self.plugin_config.init_typed - model_strict = bool(config.strict) - use_alias = not (config.validate_by_name or config.populate_by_name) and config.validate_by_alias is not False - requires_dynamic_aliases = bool(config.has_alias_generator and not config.validate_by_name) - args = self.get_field_arguments( - fields, - typed=typed, - model_strict=model_strict, - requires_dynamic_aliases=requires_dynamic_aliases, - use_alias=use_alias, - is_settings=is_settings, - is_root_model=is_root_model, - force_typevars_invariant=True, - ) - - if is_settings: - base_settings_node = self._api.lookup_fully_qualified(BASESETTINGS_FULLNAME).node - assert isinstance(base_settings_node, TypeInfo) - if '__init__' in base_settings_node.names: - base_settings_init_node = base_settings_node.names['__init__'].node - assert isinstance(base_settings_init_node, FuncDef) - if base_settings_init_node is not None and base_settings_init_node.type is not None: - func_type = base_settings_init_node.type - assert isinstance(func_type, CallableType) - for arg_idx, arg_name in enumerate(func_type.arg_names): - if arg_name is None or arg_name.startswith('__') or not arg_name.startswith('_'): - continue - analyzed_variable_type = self._api.anal_type(func_type.arg_types[arg_idx]) - if analyzed_variable_type is not None and arg_name in ( - '_cli_settings_source', - '_build_sources', - ): - # These arg names are annotated with types explicitly parameterized with `Any`, and as such - # the Any causes issues with --disallow-any-explicit. As a workaround, change - # the Any type (as if the generic type was left unparameterized): - analyzed_variable_type = analyzed_variable_type.accept( - ChangeExplicitTypeOfAny(TypeOfAny.from_omitted_generics) - ) - variable = Var(arg_name, analyzed_variable_type) - args.append(Argument(variable, analyzed_variable_type, None, ARG_OPT)) - - if not self.should_init_forbid_extra(fields, config): - var = Var('kwargs') - args.append(Argument(var, AnyType(TypeOfAny.explicit), None, ARG_STAR2)) - - add_method(self._api, self._cls, '__init__', args=args, return_type=NoneType()) - - def add_model_construct_method( - self, - fields: list[PydanticModelField], - config: ModelConfigData, - is_settings: bool, - is_root_model: bool, - ) -> None: - """Adds a fully typed `model_construct` classmethod to the class. - - Similar to the fields-aware __init__ method, but always uses the field names (not aliases), - and does not treat settings fields as optional. - """ - set_str = self._api.named_type(f'{BUILTINS_NAME}.set', [self._api.named_type(f'{BUILTINS_NAME}.str')]) - optional_set_str = UnionType([set_str, NoneType()]) - fields_set_argument = Argument(Var('_fields_set', optional_set_str), optional_set_str, None, ARG_OPT) - with state.strict_optional_set(self._api.options.strict_optional): - args = self.get_field_arguments( - fields, - typed=True, - model_strict=bool(config.strict), - requires_dynamic_aliases=False, - use_alias=False, - is_settings=is_settings, - is_root_model=is_root_model, - ) - if not self.should_init_forbid_extra(fields, config): - var = Var('kwargs') - args.append(Argument(var, AnyType(TypeOfAny.explicit), None, ARG_STAR2)) - - args = args + [fields_set_argument] if is_root_model else [fields_set_argument] + args - - add_method( - self._api, - self._cls, - 'model_construct', - args=args, - return_type=fill_typevars(self._cls.info), - is_classmethod=True, - ) - - def set_frozen(self, fields: list[PydanticModelField], api: SemanticAnalyzerPluginInterface, frozen: bool) -> None: - """Marks all fields as properties so that attempts to set them trigger mypy errors. - - This is the same approach used by the attrs and dataclasses plugins. - """ - info = self._cls.info - for field in fields: - sym_node = info.names.get(field.name) - if sym_node is not None: - var = sym_node.node - if isinstance(var, Var): - var.is_property = frozen or field.is_frozen - elif isinstance(var, PlaceholderNode) and not self._api.final_iteration: - # See https://github.com/pydantic/pydantic/issues/5191 to hit this branch for test coverage - self._api.defer() - # `var` can also be a FuncDef or Decorator node (e.g. when overriding a field with a function or property). - # In that case, we don't want to do anything. Mypy will already raise an error that a field was not properly - # overridden. - else: - var = field.to_var(info, api, use_alias=False) - var.info = info - var.is_property = frozen - var._fullname = info.fullname + '.' + var.name - info.names[var.name] = SymbolTableNode(MDEF, var) - - def get_config_update(self, name: str, arg: Expression, lax_extra: bool = False) -> ModelConfigData | None: - """Determines the config update due to a single kwarg in the ConfigDict definition. - - Warns if a tracked config attribute is set to a value the plugin doesn't know how to interpret (e.g., an int) - """ - if name not in self.tracked_config_fields: - return None - if name == 'extra': - if isinstance(arg, StrExpr): - forbid_extra = arg.value == 'forbid' - elif isinstance(arg, MemberExpr): - forbid_extra = arg.name == 'forbid' - else: - if not lax_extra: - # Only emit an error for other types of `arg` (e.g., `NameExpr`, `ConditionalExpr`, etc.) when - # reading from a config class, etc. If a ConfigDict is used, then we don't want to emit an error - # because you'll get type checking from the ConfigDict itself. - # - # It would be nice if we could introspect the types better otherwise, but I don't know what the API - # is to evaluate an expr into its type and then check if that type is compatible with the expected - # type. Note that you can still get proper type checking via: `model_config = ConfigDict(...)`, just - # if you don't use an explicit string, the plugin won't be able to infer whether extra is forbidden. - error_invalid_config_value(name, self._api, arg) - return None - return ModelConfigData(forbid_extra=forbid_extra) - if name == 'alias_generator': - has_alias_generator = True - if isinstance(arg, NameExpr) and arg.fullname == 'builtins.None': - has_alias_generator = False - return ModelConfigData(has_alias_generator=has_alias_generator) - if isinstance(arg, NameExpr) and arg.fullname in ('builtins.True', 'builtins.False'): - return ModelConfigData(**{name: arg.fullname == 'builtins.True'}) - error_invalid_config_value(name, self._api, arg) - return None - - @staticmethod - def get_has_default(stmt: AssignmentStmt) -> bool: - """Returns a boolean indicating whether the field defined in `stmt` is a required field.""" - expr = stmt.rvalue - if isinstance(expr, TempNode): - # TempNode means annotation-only, so has no default - return False - if isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME: - # The "default value" is a call to `Field`; at this point, the field has a default if and only if: - # * there is a positional argument that is not `...` - # * there is a keyword argument named "default" that is not `...` - # * there is a "default_factory" that is not `None` - for arg, name in zip(expr.args, expr.arg_names): - # If name is None, then this arg is the default because it is the only positional argument. - if name is None or name == 'default': - return arg.__class__ is not EllipsisExpr - if name == 'default_factory': - return not (isinstance(arg, NameExpr) and arg.fullname == 'builtins.None') - return False - # Has no default if the "default value" is Ellipsis (i.e., `field_name: Annotation = ...`) - return not isinstance(expr, EllipsisExpr) - - @staticmethod - def get_strict(stmt: AssignmentStmt) -> bool | None: - """Returns a the `strict` value of a field if defined, otherwise `None`.""" - expr = stmt.rvalue - if isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME: - for arg, name in zip(expr.args, expr.arg_names): - if name != 'strict': - continue - if isinstance(arg, NameExpr): - if arg.fullname == 'builtins.True': - return True - elif arg.fullname == 'builtins.False': - return False - return None - return None - - @staticmethod - def get_alias_info(stmt: AssignmentStmt) -> tuple[str | None, bool]: - """Returns a pair (alias, has_dynamic_alias), extracted from the declaration of the field defined in `stmt`. - - `has_dynamic_alias` is True if and only if an alias is provided, but not as a string literal. - If `has_dynamic_alias` is True, `alias` will be None. - """ - expr = stmt.rvalue - if isinstance(expr, TempNode): - # TempNode means annotation-only - return None, False - - if not ( - isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME - ): - # Assigned value is not a call to pydantic.fields.Field - return None, False - - if 'validation_alias' in expr.arg_names: - arg = expr.args[expr.arg_names.index('validation_alias')] - elif 'alias' in expr.arg_names: - arg = expr.args[expr.arg_names.index('alias')] - else: - return None, False - - if isinstance(arg, StrExpr): - return arg.value, False - else: - return None, True - - @staticmethod - def is_field_frozen(stmt: AssignmentStmt) -> bool: - """Returns whether the field is frozen, extracted from the declaration of the field defined in `stmt`. - - Note that this is only whether the field was declared to be frozen in a ` = Field(frozen=True)` - sense; this does not determine whether the field is frozen because the entire model is frozen; that is - handled separately. - """ - expr = stmt.rvalue - if isinstance(expr, TempNode): - # TempNode means annotation-only - return False - - if not ( - isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME - ): - # Assigned value is not a call to pydantic.fields.Field - return False - - for i, arg_name in enumerate(expr.arg_names): - if arg_name == 'frozen': - arg = expr.args[i] - return isinstance(arg, NameExpr) and arg.fullname == 'builtins.True' - return False - - def get_field_arguments( - self, - fields: list[PydanticModelField], - typed: bool, - model_strict: bool, - use_alias: bool, - requires_dynamic_aliases: bool, - is_settings: bool, - is_root_model: bool, - force_typevars_invariant: bool = False, - ) -> list[Argument]: - """Helper function used during the construction of the `__init__` and `model_construct` method signatures. - - Returns a list of mypy Argument instances for use in the generated signatures. - """ - info = self._cls.info - arguments = [ - field.to_argument( - info, - typed=typed, - model_strict=model_strict, - force_optional=requires_dynamic_aliases or is_settings, - use_alias=use_alias, - api=self._api, - force_typevars_invariant=force_typevars_invariant, - is_root_model_root=is_root_model and field.name == 'root', - ) - for field in fields - if not (use_alias and field.has_dynamic_alias) - ] - return arguments - - def should_init_forbid_extra(self, fields: list[PydanticModelField], config: ModelConfigData) -> bool: - """Indicates whether the generated `__init__` should get a `**kwargs` at the end of its signature. - - We disallow arbitrary kwargs if the extra config setting is "forbid", or if the plugin config says to, - *unless* a required dynamic alias is present (since then we can't determine a valid signature). - """ - if not (config.validate_by_name or config.populate_by_name): - if self.is_dynamic_alias_present(fields, bool(config.has_alias_generator)): - return False - if config.forbid_extra: - return True - return self.plugin_config.init_forbid_extra - - @staticmethod - def is_dynamic_alias_present(fields: list[PydanticModelField], has_alias_generator: bool) -> bool: - """Returns whether any fields on the model have a "dynamic alias", i.e., an alias that cannot be - determined during static analysis. - """ - for field in fields: - if field.has_dynamic_alias: - return True - if has_alias_generator: - for field in fields: - if field.alias is None: - return True - return False - - -class ChangeExplicitTypeOfAny(TypeTranslator): - """A type translator used to change type of Any's, if explicit.""" - - def __init__(self, type_of_any: int) -> None: - self._type_of_any = type_of_any - super().__init__() - - def visit_any(self, t: AnyType) -> Type: # noqa: D102 - if t.type_of_any == TypeOfAny.explicit: - return t.copy_modified(type_of_any=self._type_of_any) - else: - return t - - -class ModelConfigData: - """Pydantic mypy plugin model config class.""" - - def __init__( - self, - forbid_extra: bool | None = None, - frozen: bool | None = None, - from_attributes: bool | None = None, - populate_by_name: bool | None = None, - validate_by_alias: bool | None = None, - validate_by_name: bool | None = None, - has_alias_generator: bool | None = None, - strict: bool | None = None, - ): - self.forbid_extra = forbid_extra - self.frozen = frozen - self.from_attributes = from_attributes - self.populate_by_name = populate_by_name - self.validate_by_alias = validate_by_alias - self.validate_by_name = validate_by_name - self.has_alias_generator = has_alias_generator - self.strict = strict - - def get_values_dict(self) -> dict[str, Any]: - """Returns a dict of Pydantic model config names to their values. - - It includes the config if config value is not `None`. - """ - return {k: v for k, v in self.__dict__.items() if v is not None} - - def update(self, config: ModelConfigData | None) -> None: - """Update Pydantic model config values.""" - if config is None: - return - for k, v in config.get_values_dict().items(): - setattr(self, k, v) - - def setdefault(self, key: str, value: Any) -> None: - """Set default value for Pydantic model config if config value is `None`.""" - if getattr(self, key) is None: - setattr(self, key, value) - - -def is_root_model(info: TypeInfo) -> bool: - """Return whether the type info is a root model subclass (or the `RootModel` class itself).""" - return info.has_base(ROOT_MODEL_FULLNAME) - - -ERROR_ORM = ErrorCode('pydantic-orm', 'Invalid from_attributes call', 'Pydantic') -ERROR_CONFIG = ErrorCode('pydantic-config', 'Invalid config value', 'Pydantic') -ERROR_ALIAS = ErrorCode('pydantic-alias', 'Dynamic alias disallowed', 'Pydantic') -ERROR_UNEXPECTED = ErrorCode('pydantic-unexpected', 'Unexpected behavior', 'Pydantic') -ERROR_UNTYPED = ErrorCode('pydantic-field', 'Untyped field disallowed', 'Pydantic') -ERROR_FIELD_DEFAULTS = ErrorCode('pydantic-field', 'Invalid Field defaults', 'Pydantic') -ERROR_EXTRA_FIELD_ROOT_MODEL = ErrorCode('pydantic-field', 'Extra field on RootModel subclass', 'Pydantic') - - -def error_from_attributes(model_name: str, api: CheckerPluginInterface, context: Context) -> None: - """Emits an error when the model does not have `from_attributes=True`.""" - api.fail(f'"{model_name}" does not have from_attributes=True', context, code=ERROR_ORM) - - -def error_invalid_config_value(name: str, api: SemanticAnalyzerPluginInterface, context: Context) -> None: - """Emits an error when the config value is invalid.""" - api.fail(f'Invalid value for "Config.{name}"', context, code=ERROR_CONFIG) - - -def error_required_dynamic_aliases(api: SemanticAnalyzerPluginInterface, context: Context) -> None: - """Emits required dynamic aliases error. - - This will be called when `warn_required_dynamic_aliases=True`. - """ - api.fail('Required dynamic aliases disallowed', context, code=ERROR_ALIAS) - - -def error_unexpected_behavior( - detail: str, api: CheckerPluginInterface | SemanticAnalyzerPluginInterface, context: Context -) -> None: # pragma: no cover - """Emits unexpected behavior error.""" - # Can't think of a good way to test this, but I confirmed it renders as desired by adding to a non-error path - link = 'https://github.com/pydantic/pydantic/issues/new/choose' - full_message = f'The pydantic mypy plugin ran into unexpected behavior: {detail}\n' - full_message += f'Please consider reporting this bug at {link} so we can try to fix it!' - api.fail(full_message, context, code=ERROR_UNEXPECTED) - - -def error_untyped_fields(api: SemanticAnalyzerPluginInterface, context: Context) -> None: - """Emits an error when there is an untyped field in the model.""" - api.fail('Untyped fields disallowed', context, code=ERROR_UNTYPED) - - -def error_extra_fields_on_root_model(api: CheckerPluginInterface, context: Context) -> None: - """Emits an error when there is more than just a root field defined for a subclass of RootModel.""" - api.fail('Only `root` is allowed as a field of a `RootModel`', context, code=ERROR_EXTRA_FIELD_ROOT_MODEL) - - -def add_method( - api: SemanticAnalyzerPluginInterface | CheckerPluginInterface, - cls: ClassDef, - name: str, - args: list[Argument], - return_type: Type, - self_type: Type | None = None, - tvar_def: TypeVarType | None = None, - is_classmethod: bool = False, -) -> None: - """Very closely related to `mypy.plugins.common.add_method_to_class`, with a few pydantic-specific changes.""" - info = cls.info - - # First remove any previously generated methods with the same name - # to avoid clashes and problems in the semantic analyzer. - if name in info.names: - sym = info.names[name] - if sym.plugin_generated and isinstance(sym.node, FuncDef): - cls.defs.body.remove(sym.node) # pragma: no cover - - if isinstance(api, SemanticAnalyzerPluginInterface): - function_type = api.named_type('builtins.function') - else: - function_type = api.named_generic_type('builtins.function', []) - - if is_classmethod: - self_type = self_type or TypeType(fill_typevars(info)) - first = [Argument(Var('_cls'), self_type, None, ARG_POS, True)] - else: - self_type = self_type or fill_typevars(info) - # `self` is positional *ONLY* here, but this can't be expressed - # fully in the mypy internal API. ARG_POS is the closest we can get. - # Using ARG_POS will, however, give mypy errors if a `self` field - # is present on a model: - # - # Name "self" already defined (possibly by an import) [no-redef] - # - # As a workaround, we give this argument a name that will - # never conflict. By its positional nature, this name will not - # be used or exposed to users. - first = [Argument(Var('__pydantic_self__'), self_type, None, ARG_POS)] - args = first + args - - arg_types, arg_names, arg_kinds = [], [], [] - for arg in args: - assert arg.type_annotation, 'All arguments must be fully typed.' - arg_types.append(arg.type_annotation) - arg_names.append(arg.variable.name) - arg_kinds.append(arg.kind) - - signature = CallableType( - arg_types, arg_kinds, arg_names, return_type, function_type, variables=[tvar_def] if tvar_def else None - ) - - func = FuncDef(name, args, Block([PassStmt()])) - func.info = info - func.type = set_callable_name(signature, func) - func.is_class = is_classmethod - func._fullname = info.fullname + '.' + name - func.line = info.line - - # NOTE: we would like the plugin generated node to dominate, but we still - # need to keep any existing definitions so they get semantically analyzed. - if name in info.names: - # Get a nice unique name instead. - r_name = get_unique_redefinition_name(name, info.names) - info.names[r_name] = info.names[name] - - # Add decorator for is_classmethod - # The dataclasses plugin claims this is unnecessary for classmethods, but not including it results in a - # signature incompatible with the superclass, which causes mypy errors to occur for every subclass of BaseModel. - if is_classmethod: - func.is_decorated = True - v = Var(name, func.type) - v.info = info - v._fullname = func._fullname - v.is_classmethod = True - dec = Decorator(func, [NameExpr('classmethod')], v) - dec.line = info.line - sym = SymbolTableNode(MDEF, dec) - else: - sym = SymbolTableNode(MDEF, func) - sym.plugin_generated = True - info.names[name] = sym - - info.defn.defs.body.append(func) - - -def parse_toml(config_file: str) -> dict[str, Any] | None: - """Returns a dict of config keys to values. - - It reads configs from toml file and returns `None` if the file is not a toml file. - """ - if not config_file.endswith('.toml'): - return None - - if sys.version_info >= (3, 11): - import tomllib as toml_ - else: - try: - import tomli as toml_ - except ImportError: # pragma: no cover - import warnings - - warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.', stacklevel=2) - return None - - with open(config_file, 'rb') as rf: - return toml_.load(rf) diff --git a/functions/python/data-processor/pydantic/networks.py b/functions/python/data-processor/pydantic/networks.py deleted file mode 100644 index 4015bad6b..000000000 --- a/functions/python/data-processor/pydantic/networks.py +++ /dev/null @@ -1,1332 +0,0 @@ -"""The networks module contains types for common network-related fields.""" - -from __future__ import annotations as _annotations - -import dataclasses as _dataclasses -import re -from dataclasses import fields -from functools import lru_cache -from importlib.metadata import version -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from typing import TYPE_CHECKING, Annotated, Any, ClassVar - -from pydantic_core import ( - MultiHostHost, - PydanticCustomError, - PydanticSerializationUnexpectedValue, - SchemaSerializer, - core_schema, -) -from pydantic_core import MultiHostUrl as _CoreMultiHostUrl -from pydantic_core import Url as _CoreUrl -from typing_extensions import Self, TypeAlias - -from pydantic.errors import PydanticUserError - -from ._internal import _repr, _schema_generation_shared -from ._migration import getattr_migration -from .annotated_handlers import GetCoreSchemaHandler -from .json_schema import JsonSchemaValue -from .type_adapter import TypeAdapter - -if TYPE_CHECKING: - import email_validator - - NetworkType: TypeAlias = 'str | bytes | int | tuple[str | bytes | int, str | int]' - -else: - email_validator = None - - -__all__ = [ - 'AnyUrl', - 'AnyHttpUrl', - 'FileUrl', - 'FtpUrl', - 'HttpUrl', - 'WebsocketUrl', - 'AnyWebsocketUrl', - 'UrlConstraints', - 'EmailStr', - 'NameEmail', - 'IPvAnyAddress', - 'IPvAnyInterface', - 'IPvAnyNetwork', - 'PostgresDsn', - 'CockroachDsn', - 'AmqpDsn', - 'RedisDsn', - 'MongoDsn', - 'KafkaDsn', - 'NatsDsn', - 'validate_email', - 'MySQLDsn', - 'MariaDBDsn', - 'ClickHouseDsn', - 'SnowflakeDsn', -] - - -@_dataclasses.dataclass -class UrlConstraints: - """Url constraints. - - Attributes: - max_length: The maximum length of the url. Defaults to `None`. - allowed_schemes: The allowed schemes. Defaults to `None`. - host_required: Whether the host is required. Defaults to `None`. - default_host: The default host. Defaults to `None`. - default_port: The default port. Defaults to `None`. - default_path: The default path. Defaults to `None`. - preserve_empty_path: Whether to preserve empty URL paths. Defaults to `None`. - """ - - max_length: int | None = None - allowed_schemes: list[str] | None = None - host_required: bool | None = None - default_host: str | None = None - default_port: int | None = None - default_path: str | None = None - preserve_empty_path: bool | None = None - - def __hash__(self) -> int: - return hash( - ( - self.max_length, - tuple(self.allowed_schemes) if self.allowed_schemes is not None else None, - self.host_required, - self.default_host, - self.default_port, - self.default_path, - self.preserve_empty_path, - ) - ) - - @property - def defined_constraints(self) -> dict[str, Any]: - """Fetch a key / value mapping of constraints to values that are not None. Used for core schema updates.""" - return {field.name: value for field in fields(self) if (value := getattr(self, field.name)) is not None} - - def __get_pydantic_core_schema__(self, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source) - - # for function-wrap schemas, url constraints is applied to the inner schema - # because when we generate schemas for urls, we wrap a core_schema.url_schema() with a function-wrap schema - # that helps with validation on initialization, see _BaseUrl and _BaseMultiHostUrl below. - schema_to_mutate = schema['schema'] if schema['type'] == 'function-wrap' else schema - if (annotated_type := schema_to_mutate['type']) not in ('url', 'multi-host-url'): - raise PydanticUserError( - f"'UrlConstraints' cannot annotate '{annotated_type}'.", code='invalid-annotated-type' - ) - for constraint_key, constraint_value in self.defined_constraints.items(): - schema_to_mutate[constraint_key] = constraint_value - return schema - - -class _BaseUrl: - _constraints: ClassVar[UrlConstraints] = UrlConstraints() - _url: _CoreUrl - - def __init__(self, url: str | _CoreUrl | _BaseUrl) -> None: - self._url = _build_type_adapter(self.__class__).validate_python(url)._url - - @property - def scheme(self) -> str: - """The scheme part of the URL. - - e.g. `https` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.scheme - - @property - def username(self) -> str | None: - """The username part of the URL, or `None`. - - e.g. `user` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.username - - @property - def password(self) -> str | None: - """The password part of the URL, or `None`. - - e.g. `pass` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.password - - @property - def host(self) -> str | None: - """The host part of the URL, or `None`. - - If the URL must be punycode encoded, this is the encoded host, e.g if the input URL is `https://£££.com`, - `host` will be `xn--9aaa.com` - """ - return self._url.host - - def unicode_host(self) -> str | None: - """The host part of the URL as a unicode string, or `None`. - - e.g. `host` in `https://user:pass@host:port/path?query#fragment` - - If the URL must be punycode encoded, this is the decoded host, e.g if the input URL is `https://£££.com`, - `unicode_host()` will be `£££.com` - """ - return self._url.unicode_host() - - @property - def port(self) -> int | None: - """The port part of the URL, or `None`. - - e.g. `port` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.port - - @property - def path(self) -> str | None: - """The path part of the URL, or `None`. - - e.g. `/path` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.path - - @property - def query(self) -> str | None: - """The query part of the URL, or `None`. - - e.g. `query` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.query - - def query_params(self) -> list[tuple[str, str]]: - """The query part of the URL as a list of key-value pairs. - - e.g. `[('foo', 'bar')]` in `https://user:pass@host:port/path?foo=bar#fragment` - """ - return self._url.query_params() - - @property - def fragment(self) -> str | None: - """The fragment part of the URL, or `None`. - - e.g. `fragment` in `https://user:pass@host:port/path?query#fragment` - """ - return self._url.fragment - - def unicode_string(self) -> str: - """The URL as a unicode string, unlike `__str__()` this will not punycode encode the host. - - If the URL must be punycode encoded, this is the decoded string, e.g if the input URL is `https://£££.com`, - `unicode_string()` will be `https://£££.com` - """ - return self._url.unicode_string() - - def encoded_string(self) -> str: - """The URL's encoded string representation via __str__(). - - This returns the punycode-encoded host version of the URL as a string. - """ - return str(self) - - def __str__(self) -> str: - """The URL as a string, this will punycode encode the host if required.""" - return str(self._url) - - def __repr__(self) -> str: - return f'{self.__class__.__name__}({str(self._url)!r})' - - def __deepcopy__(self, memo: dict) -> Self: - return self.__class__(self._url) - - def __eq__(self, other: Any) -> bool: - return self.__class__ is other.__class__ and self._url == other._url - - def __lt__(self, other: Any) -> bool: - return self.__class__ is other.__class__ and self._url < other._url - - def __gt__(self, other: Any) -> bool: - return self.__class__ is other.__class__ and self._url > other._url - - def __le__(self, other: Any) -> bool: - return self.__class__ is other.__class__ and self._url <= other._url - - def __ge__(self, other: Any) -> bool: - return self.__class__ is other.__class__ and self._url >= other._url - - def __hash__(self) -> int: - return hash(self._url) - - def __len__(self) -> int: - return len(str(self._url)) - - @classmethod - def build( - cls, - *, - scheme: str, - username: str | None = None, - password: str | None = None, - host: str, - port: int | None = None, - path: str | None = None, - query: str | None = None, - fragment: str | None = None, - ) -> Self: - """Build a new `Url` instance from its component parts. - - Args: - scheme: The scheme part of the URL. - username: The username part of the URL, or omit for no username. - password: The password part of the URL, or omit for no password. - host: The host part of the URL. - port: The port part of the URL, or omit for no port. - path: The path part of the URL, or omit for no path. - query: The query part of the URL, or omit for no query. - fragment: The fragment part of the URL, or omit for no fragment. - - Returns: - An instance of URL - """ - return cls( - _CoreUrl.build( - scheme=scheme, - username=username, - password=password, - host=host, - port=port, - path=path, - query=query, - fragment=fragment, - ) - ) - - @classmethod - def serialize_url(cls, url: Any, info: core_schema.SerializationInfo) -> str | Self: - if not isinstance(url, cls): - raise PydanticSerializationUnexpectedValue( - f"Expected `{cls}` but got `{type(url)}` with value `'{url}'` - serialized value may not be as expected." - ) - if info.mode == 'json': - return str(url) - return url - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[_BaseUrl], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - def wrap_val(v, h): - if isinstance(v, source): - return v - if isinstance(v, _BaseUrl): - v = str(v) - core_url = h(v) - instance = source.__new__(source) - instance._url = core_url - return instance - - return core_schema.no_info_wrap_validator_function( - wrap_val, - schema=core_schema.url_schema(**cls._constraints.defined_constraints), - serialization=core_schema.plain_serializer_function_ser_schema( - cls.serialize_url, info_arg=True, when_used='always' - ), - ) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - # we use the url schema for json schema generation, but we might have to extract it from - # the function-wrap schema we use as a tool for validation on initialization - inner_schema = core_schema['schema'] if core_schema['type'] == 'function-wrap' else core_schema - return handler(inner_schema) - - __pydantic_serializer__ = SchemaSerializer(core_schema.any_schema(serialization=core_schema.to_string_ser_schema())) - - -class _BaseMultiHostUrl: - _constraints: ClassVar[UrlConstraints] = UrlConstraints() - _url: _CoreMultiHostUrl - - def __init__(self, url: str | _CoreMultiHostUrl | _BaseMultiHostUrl) -> None: - self._url = _build_type_adapter(self.__class__).validate_python(url)._url - - @property - def scheme(self) -> str: - """The scheme part of the URL. - - e.g. `https` in `https://foo.com,bar.com/path?query#fragment` - """ - return self._url.scheme - - @property - def path(self) -> str | None: - """The path part of the URL, or `None`. - - e.g. `/path` in `https://foo.com,bar.com/path?query#fragment` - """ - return self._url.path - - @property - def query(self) -> str | None: - """The query part of the URL, or `None`. - - e.g. `query` in `https://foo.com,bar.com/path?query#fragment` - """ - return self._url.query - - def query_params(self) -> list[tuple[str, str]]: - """The query part of the URL as a list of key-value pairs. - - e.g. `[('foo', 'bar')]` in `https://foo.com,bar.com/path?foo=bar#fragment` - """ - return self._url.query_params() - - @property - def fragment(self) -> str | None: - """The fragment part of the URL, or `None`. - - e.g. `fragment` in `https://foo.com,bar.com/path?query#fragment` - """ - return self._url.fragment - - def hosts(self) -> list[MultiHostHost]: - '''The hosts of the `MultiHostUrl` as [`MultiHostHost`][pydantic_core.MultiHostHost] typed dicts. - - ```python - from pydantic_core import MultiHostUrl - - mhu = MultiHostUrl('https://foo.com:123,foo:bar@bar.com/path') - print(mhu.hosts()) - """ - [ - {'username': None, 'password': None, 'host': 'foo.com', 'port': 123}, - {'username': 'foo', 'password': 'bar', 'host': 'bar.com', 'port': 443} - ] - ``` - Returns: - A list of dicts, each representing a host. - ''' - return self._url.hosts() - - def encoded_string(self) -> str: - """The URL's encoded string representation via __str__(). - - This returns the punycode-encoded host version of the URL as a string. - """ - return str(self) - - def unicode_string(self) -> str: - """The URL as a unicode string, unlike `__str__()` this will not punycode encode the hosts.""" - return self._url.unicode_string() - - def __str__(self) -> str: - """The URL as a string, this will punycode encode the host if required.""" - return str(self._url) - - def __repr__(self) -> str: - return f'{self.__class__.__name__}({str(self._url)!r})' - - def __deepcopy__(self, memo: dict) -> Self: - return self.__class__(self._url) - - def __eq__(self, other: Any) -> bool: - return self.__class__ is other.__class__ and self._url == other._url - - def __hash__(self) -> int: - return hash(self._url) - - def __len__(self) -> int: - return len(str(self._url)) - - @classmethod - def build( - cls, - *, - scheme: str, - hosts: list[MultiHostHost] | None = None, - username: str | None = None, - password: str | None = None, - host: str | None = None, - port: int | None = None, - path: str | None = None, - query: str | None = None, - fragment: str | None = None, - ) -> Self: - """Build a new `MultiHostUrl` instance from its component parts. - - This method takes either `hosts` - a list of `MultiHostHost` typed dicts, or the individual components - `username`, `password`, `host` and `port`. - - Args: - scheme: The scheme part of the URL. - hosts: Multiple hosts to build the URL from. - username: The username part of the URL. - password: The password part of the URL. - host: The host part of the URL. - port: The port part of the URL. - path: The path part of the URL. - query: The query part of the URL, or omit for no query. - fragment: The fragment part of the URL, or omit for no fragment. - - Returns: - An instance of `MultiHostUrl` - """ - return cls( - _CoreMultiHostUrl.build( - scheme=scheme, - hosts=hosts, - username=username, - password=password, - host=host, - port=port, - path=path, - query=query, - fragment=fragment, - ) - ) - - @classmethod - def serialize_url(cls, url: Any, info: core_schema.SerializationInfo) -> str | Self: - if not isinstance(url, cls): - raise PydanticSerializationUnexpectedValue( - f"Expected `{cls}` but got `{type(url)}` with value `'{url}'` - serialized value may not be as expected." - ) - if info.mode == 'json': - return str(url) - return url - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[_BaseMultiHostUrl], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - def wrap_val(v, h): - if isinstance(v, source): - return v - if isinstance(v, _BaseMultiHostUrl): - v = str(v) - core_url = h(v) - instance = source.__new__(source) - instance._url = core_url - return instance - - return core_schema.no_info_wrap_validator_function( - wrap_val, - schema=core_schema.multi_host_url_schema(**cls._constraints.defined_constraints), - serialization=core_schema.plain_serializer_function_ser_schema( - cls.serialize_url, info_arg=True, when_used='always' - ), - ) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - # we use the url schema for json schema generation, but we might have to extract it from - # the function-wrap schema we use as a tool for validation on initialization - inner_schema = core_schema['schema'] if core_schema['type'] == 'function-wrap' else core_schema - return handler(inner_schema) - - __pydantic_serializer__ = SchemaSerializer(core_schema.any_schema(serialization=core_schema.to_string_ser_schema())) - - -@lru_cache -def _build_type_adapter(cls: type[_BaseUrl | _BaseMultiHostUrl]) -> TypeAdapter: - return TypeAdapter(cls) - - -class AnyUrl(_BaseUrl): - """Base type for all URLs. - - * Any scheme allowed - * Top-level domain (TLD) not required - * Host not required - - Assuming an input URL of `http://samuel:pass@example.com:8000/the/path/?query=here#fragment=is;this=bit`, - the types export the following properties: - - - `scheme`: the URL scheme (`http`), always set. - - `host`: the URL host (`example.com`). - - `username`: optional username if included (`samuel`). - - `password`: optional password if included (`pass`). - - `port`: optional port (`8000`). - - `path`: optional path (`/the/path/`). - - `query`: optional URL query (for example, `GET` arguments or "search string", such as `query=here`). - - `fragment`: optional fragment (`fragment=is;this=bit`). - """ - - -# Note: all single host urls inherit from `AnyUrl` to preserve compatibility with pre-v2.10 code -# Where urls were annotated variants of `AnyUrl`, which was an alias to `pydantic_core.Url` - - -class AnyHttpUrl(AnyUrl): - """A type that will accept any http or https URL. - - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints(allowed_schemes=['http', 'https']) - - -class HttpUrl(AnyUrl): - """A type that will accept any http or https URL. - - * TLD not required - * Host not required - * Max length 2083 - - ```python - from pydantic import BaseModel, HttpUrl, ValidationError - - class MyModel(BaseModel): - url: HttpUrl - - m = MyModel(url='http://www.example.com') # (1)! - print(m.url) - #> http://www.example.com/ - - try: - MyModel(url='ftp://invalid.url') - except ValidationError as e: - print(e) - ''' - 1 validation error for MyModel - url - URL scheme should be 'http' or 'https' [type=url_scheme, input_value='ftp://invalid.url', input_type=str] - ''' - - try: - MyModel(url='not a url') - except ValidationError as e: - print(e) - ''' - 1 validation error for MyModel - url - Input should be a valid URL, relative URL without a base [type=url_parsing, input_value='not a url', input_type=str] - ''' - ``` - - 1. Note: mypy would prefer `m = MyModel(url=HttpUrl('http://www.example.com'))`, but Pydantic will convert the string to an HttpUrl instance anyway. - - "International domains" (e.g. a URL where the host or TLD includes non-ascii characters) will be encoded via - [punycode](https://en.wikipedia.org/wiki/Punycode) (see - [this article](https://www.xudongz.com/blog/2017/idn-phishing/) for a good description of why this is important): - - ```python - from pydantic import BaseModel, HttpUrl - - class MyModel(BaseModel): - url: HttpUrl - - m1 = MyModel(url='http://puny£code.com') - print(m1.url) - #> http://xn--punycode-eja.com/ - m2 = MyModel(url='https://www.аррӏе.com/') - print(m2.url) - #> https://www.xn--80ak6aa92e.com/ - m3 = MyModel(url='https://www.example.珠宝/') - print(m3.url) - #> https://www.example.xn--pbt977c/ - ``` - - - !!! warning "Underscores in Hostnames" - In Pydantic, underscores are allowed in all parts of a domain except the TLD. - Technically this might be wrong - in theory the hostname cannot have underscores, but subdomains can. - - To explain this; consider the following two cases: - - - `exam_ple.co.uk`: the hostname is `exam_ple`, which should not be allowed since it contains an underscore. - - `foo_bar.example.com` the hostname is `example`, which should be allowed since the underscore is in the subdomain. - - Without having an exhaustive list of TLDs, it would be impossible to differentiate between these two. Therefore - underscores are allowed, but you can always do further validation in a validator if desired. - - Also, Chrome, Firefox, and Safari all currently accept `http://exam_ple.com` as a URL, so we're in good - (or at least big) company. - """ - - _constraints = UrlConstraints(max_length=2083, allowed_schemes=['http', 'https']) - - -class AnyWebsocketUrl(AnyUrl): - """A type that will accept any ws or wss URL. - - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints(allowed_schemes=['ws', 'wss']) - - -class WebsocketUrl(AnyUrl): - """A type that will accept any ws or wss URL. - - * TLD not required - * Host not required - * Max length 2083 - """ - - _constraints = UrlConstraints(max_length=2083, allowed_schemes=['ws', 'wss']) - - -class FileUrl(AnyUrl): - """A type that will accept any file URL. - - * Host not required - """ - - _constraints = UrlConstraints(allowed_schemes=['file']) - - -class FtpUrl(AnyUrl): - """A type that will accept ftp URL. - - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints(allowed_schemes=['ftp']) - - -class PostgresDsn(_BaseMultiHostUrl): - """A type that will accept any Postgres DSN. - - * User info required - * TLD not required - * Host required - * Supports multiple hosts - - If further validation is required, these properties can be used by validators to enforce specific behaviour: - - ```python - from pydantic import ( - BaseModel, - HttpUrl, - PostgresDsn, - ValidationError, - field_validator, - ) - - class MyModel(BaseModel): - url: HttpUrl - - m = MyModel(url='http://www.example.com') - - # the repr() method for a url will display all properties of the url - print(repr(m.url)) - #> HttpUrl('http://www.example.com/') - print(m.url.scheme) - #> http - print(m.url.host) - #> www.example.com - print(m.url.port) - #> 80 - - class MyDatabaseModel(BaseModel): - db: PostgresDsn - - @field_validator('db') - def check_db_name(cls, v): - assert v.path and len(v.path) > 1, 'database must be provided' - return v - - m = MyDatabaseModel(db='postgres://user:pass@localhost:5432/foobar') - print(m.db) - #> postgres://user:pass@localhost:5432/foobar - - try: - MyDatabaseModel(db='postgres://user:pass@localhost:5432') - except ValidationError as e: - print(e) - ''' - 1 validation error for MyDatabaseModel - db - Assertion failed, database must be provided - assert (None) - + where None = PostgresDsn('postgres://user:pass@localhost:5432').path [type=assertion_error, input_value='postgres://user:pass@localhost:5432', input_type=str] - ''' - ``` - """ - - _constraints = UrlConstraints( - host_required=True, - allowed_schemes=[ - 'postgres', - 'postgresql', - 'postgresql+asyncpg', - 'postgresql+pg8000', - 'postgresql+psycopg', - 'postgresql+psycopg2', - 'postgresql+psycopg2cffi', - 'postgresql+py-postgresql', - 'postgresql+pygresql', - ], - ) - - @property - def host(self) -> str: - """The required URL host.""" - return self._url.host # pyright: ignore[reportAttributeAccessIssue] - - -class CockroachDsn(AnyUrl): - """A type that will accept any Cockroach DSN. - - * User info required - * TLD not required - * Host required - """ - - _constraints = UrlConstraints( - host_required=True, - allowed_schemes=[ - 'cockroachdb', - 'cockroachdb+psycopg2', - 'cockroachdb+asyncpg', - ], - ) - - @property - def host(self) -> str: - """The required URL host.""" - return self._url.host # pyright: ignore[reportReturnType] - - -class AmqpDsn(AnyUrl): - """A type that will accept any AMQP DSN. - - * User info required - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints(allowed_schemes=['amqp', 'amqps']) - - -class RedisDsn(AnyUrl): - """A type that will accept any Redis DSN. - - * User info required - * TLD not required - * Host required (e.g., `rediss://:pass@localhost`) - """ - - _constraints = UrlConstraints( - allowed_schemes=['redis', 'rediss'], - default_host='localhost', - default_port=6379, - default_path='/0', - host_required=True, - ) - - @property - def host(self) -> str: - """The required URL host.""" - return self._url.host # pyright: ignore[reportReturnType] - - -class MongoDsn(_BaseMultiHostUrl): - """A type that will accept any MongoDB DSN. - - * User info not required - * Database name not required - * Port not required - * User info may be passed without user part (e.g., `mongodb://mongodb0.example.com:27017`). - - !!! warning - If a port isn't specified, the default MongoDB port `27017` will be used. If this behavior is - undesirable, you can use the following: - - ```python - from typing import Annotated - - from pydantic_core import MultiHostUrl - - from pydantic import UrlConstraints - - MongoDsnNoDefaultPort = Annotated[ - MultiHostUrl, - UrlConstraints(allowed_schemes=['mongodb', 'mongodb+srv']), - ] - ``` - """ - - _constraints = UrlConstraints(allowed_schemes=['mongodb', 'mongodb+srv'], default_port=27017) - - -class KafkaDsn(AnyUrl): - """A type that will accept any Kafka DSN. - - * User info required - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints(allowed_schemes=['kafka'], default_host='localhost', default_port=9092) - - -class NatsDsn(_BaseMultiHostUrl): - """A type that will accept any NATS DSN. - - NATS is a connective technology built for the ever increasingly hyper-connected world. - It is a single technology that enables applications to securely communicate across - any combination of cloud vendors, on-premise, edge, web and mobile, and devices. - More: https://nats.io - """ - - _constraints = UrlConstraints( - allowed_schemes=['nats', 'tls', 'ws', 'wss'], default_host='localhost', default_port=4222 - ) - - -class MySQLDsn(AnyUrl): - """A type that will accept any MySQL DSN. - - * User info required - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints( - allowed_schemes=[ - 'mysql', - 'mysql+mysqlconnector', - 'mysql+aiomysql', - 'mysql+asyncmy', - 'mysql+mysqldb', - 'mysql+pymysql', - 'mysql+cymysql', - 'mysql+pyodbc', - ], - default_port=3306, - host_required=True, - ) - - -class MariaDBDsn(AnyUrl): - """A type that will accept any MariaDB DSN. - - * User info required - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints( - allowed_schemes=['mariadb', 'mariadb+mariadbconnector', 'mariadb+pymysql'], - default_port=3306, - ) - - -class ClickHouseDsn(AnyUrl): - """A type that will accept any ClickHouse DSN. - - * User info required - * TLD not required - * Host not required - """ - - _constraints = UrlConstraints( - allowed_schemes=[ - 'clickhouse+native', - 'clickhouse+asynch', - 'clickhouse+http', - 'clickhouse', - 'clickhouses', - 'clickhousedb', - ], - default_host='localhost', - default_port=9000, - ) - - -class SnowflakeDsn(AnyUrl): - """A type that will accept any Snowflake DSN. - - * User info required - * TLD not required - * Host required - """ - - _constraints = UrlConstraints( - allowed_schemes=['snowflake'], - host_required=True, - ) - - @property - def host(self) -> str: - """The required URL host.""" - return self._url.host # pyright: ignore[reportReturnType] - - -def import_email_validator() -> None: - global email_validator - try: - import email_validator - except ImportError as e: - raise ImportError("email-validator is not installed, run `pip install 'pydantic[email]'`") from e - if not version('email-validator').partition('.')[0] == '2': - raise ImportError('email-validator version >= 2.0 required, run pip install -U email-validator') - - -if TYPE_CHECKING: - EmailStr = Annotated[str, ...] -else: - - class EmailStr: - """ - Info: - To use this type, you need to install the optional - [`email-validator`](https://github.com/JoshData/python-email-validator) package: - - ```bash - pip install email-validator - ``` - - Validate email addresses. - - ```python - from pydantic import BaseModel, EmailStr - - class Model(BaseModel): - email: EmailStr - - print(Model(email='contact@mail.com')) - #> email='contact@mail.com' - ``` - """ # noqa: D212 - - @classmethod - def __get_pydantic_core_schema__( - cls, - _source: type[Any], - _handler: GetCoreSchemaHandler, - ) -> core_schema.CoreSchema: - import_email_validator() - return core_schema.no_info_after_validator_function(cls._validate, core_schema.str_schema()) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = handler(core_schema) - field_schema.update(type='string', format='email') - return field_schema - - @classmethod - def _validate(cls, input_value: str, /) -> str: - return validate_email(input_value)[1] - - -class NameEmail(_repr.Representation): - """ - Info: - To use this type, you need to install the optional - [`email-validator`](https://github.com/JoshData/python-email-validator) package: - - ```bash - pip install email-validator - ``` - - Validate a name and email address combination, as specified by - [RFC 5322](https://datatracker.ietf.org/doc/html/rfc5322#section-3.4). - - The `NameEmail` has two properties: `name` and `email`. - In case the `name` is not provided, it's inferred from the email address. - - ```python - from pydantic import BaseModel, NameEmail - - class User(BaseModel): - email: NameEmail - - user = User(email='Fred Bloggs ') - print(user.email) - #> Fred Bloggs - print(user.email.name) - #> Fred Bloggs - - user = User(email='fred.bloggs@example.com') - print(user.email) - #> fred.bloggs - print(user.email.name) - #> fred.bloggs - ``` - """ # noqa: D212 - - __slots__ = 'name', 'email' - - def __init__(self, name: str, email: str): - self.name = name - self.email = email - - def __eq__(self, other: Any) -> bool: - return isinstance(other, NameEmail) and (self.name, self.email) == (other.name, other.email) - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = handler(core_schema) - field_schema.update(type='string', format='name-email') - return field_schema - - @classmethod - def __get_pydantic_core_schema__( - cls, - _source: type[Any], - _handler: GetCoreSchemaHandler, - ) -> core_schema.CoreSchema: - import_email_validator() - - return core_schema.no_info_after_validator_function( - cls._validate, - core_schema.json_or_python_schema( - json_schema=core_schema.str_schema(), - python_schema=core_schema.union_schema( - [core_schema.is_instance_schema(cls), core_schema.str_schema()], - custom_error_type='name_email_type', - custom_error_message='Input is not a valid NameEmail', - ), - serialization=core_schema.to_string_ser_schema(), - ), - ) - - @classmethod - def _validate(cls, input_value: Self | str, /) -> Self: - if isinstance(input_value, str): - name, email = validate_email(input_value) - return cls(name, email) - else: - return input_value - - def __str__(self) -> str: - if '@' in self.name: - return f'"{self.name}" <{self.email}>' - - return f'{self.name} <{self.email}>' - - -IPvAnyAddressType: TypeAlias = 'IPv4Address | IPv6Address' -IPvAnyInterfaceType: TypeAlias = 'IPv4Interface | IPv6Interface' -IPvAnyNetworkType: TypeAlias = 'IPv4Network | IPv6Network' - -if TYPE_CHECKING: - IPvAnyAddress = IPvAnyAddressType - IPvAnyInterface = IPvAnyInterfaceType - IPvAnyNetwork = IPvAnyNetworkType -else: - - class IPvAnyAddress: - """Validate an IPv4 or IPv6 address. - - ```python - from pydantic import BaseModel - from pydantic.networks import IPvAnyAddress - - class IpModel(BaseModel): - ip: IPvAnyAddress - - print(IpModel(ip='127.0.0.1')) - #> ip=IPv4Address('127.0.0.1') - - try: - IpModel(ip='http://www.example.com') - except ValueError as e: - print(e.errors()) - ''' - [ - { - 'type': 'ip_any_address', - 'loc': ('ip',), - 'msg': 'value is not a valid IPv4 or IPv6 address', - 'input': 'http://www.example.com', - } - ] - ''' - ``` - """ - - __slots__ = () - - def __new__(cls, value: Any) -> IPvAnyAddressType: - """Validate an IPv4 or IPv6 address.""" - try: - return IPv4Address(value) - except ValueError: - pass - - try: - return IPv6Address(value) - except ValueError: - raise PydanticCustomError('ip_any_address', 'value is not a valid IPv4 or IPv6 address') - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = {} - field_schema.update(type='string', format='ipvanyaddress') - return field_schema - - @classmethod - def __get_pydantic_core_schema__( - cls, - _source: type[Any], - _handler: GetCoreSchemaHandler, - ) -> core_schema.CoreSchema: - return core_schema.no_info_plain_validator_function( - cls._validate, serialization=core_schema.to_string_ser_schema() - ) - - @classmethod - def _validate(cls, input_value: Any, /) -> IPvAnyAddressType: - return cls(input_value) # type: ignore[return-value] - - class IPvAnyInterface: - """Validate an IPv4 or IPv6 interface.""" - - __slots__ = () - - def __new__(cls, value: NetworkType) -> IPvAnyInterfaceType: - """Validate an IPv4 or IPv6 interface.""" - try: - return IPv4Interface(value) - except ValueError: - pass - - try: - return IPv6Interface(value) - except ValueError: - raise PydanticCustomError('ip_any_interface', 'value is not a valid IPv4 or IPv6 interface') - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = {} - field_schema.update(type='string', format='ipvanyinterface') - return field_schema - - @classmethod - def __get_pydantic_core_schema__( - cls, - _source: type[Any], - _handler: GetCoreSchemaHandler, - ) -> core_schema.CoreSchema: - return core_schema.no_info_plain_validator_function( - cls._validate, serialization=core_schema.to_string_ser_schema() - ) - - @classmethod - def _validate(cls, input_value: NetworkType, /) -> IPvAnyInterfaceType: - return cls(input_value) # type: ignore[return-value] - - class IPvAnyNetwork: - """Validate an IPv4 or IPv6 network.""" - - __slots__ = () - - def __new__(cls, value: NetworkType) -> IPvAnyNetworkType: - """Validate an IPv4 or IPv6 network.""" - # Assume IP Network is defined with a default value for `strict` argument. - # Define your own class if you want to specify network address check strictness. - try: - return IPv4Network(value) - except ValueError: - pass - - try: - return IPv6Network(value) - except ValueError: - raise PydanticCustomError('ip_any_network', 'value is not a valid IPv4 or IPv6 network') - - @classmethod - def __get_pydantic_json_schema__( - cls, core_schema: core_schema.CoreSchema, handler: _schema_generation_shared.GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = {} - field_schema.update(type='string', format='ipvanynetwork') - return field_schema - - @classmethod - def __get_pydantic_core_schema__( - cls, - _source: type[Any], - _handler: GetCoreSchemaHandler, - ) -> core_schema.CoreSchema: - return core_schema.no_info_plain_validator_function( - cls._validate, serialization=core_schema.to_string_ser_schema() - ) - - @classmethod - def _validate(cls, input_value: NetworkType, /) -> IPvAnyNetworkType: - return cls(input_value) # type: ignore[return-value] - - -def _build_pretty_email_regex() -> re.Pattern[str]: - name_chars = r'[\w!#$%&\'*+\-/=?^_`{|}~]' - unquoted_name_group = rf'((?:{name_chars}+\s+)*{name_chars}+)' - quoted_name_group = r'"((?:[^"]|\")+)"' - email_group = r'<(.+)>' - return re.compile(rf'\s*(?:{unquoted_name_group}|{quoted_name_group})?\s*{email_group}\s*') - - -pretty_email_regex = _build_pretty_email_regex() - -MAX_EMAIL_LENGTH = 2048 -"""Maximum length for an email. -A somewhat arbitrary but very generous number compared to what is allowed by most implementations. -""" - - -def validate_email(value: str) -> tuple[str, str]: - """Email address validation using [email-validator](https://pypi.org/project/email-validator/). - - Returns: - A tuple containing the local part of the email (or the name for "pretty" email addresses) - and the normalized email. - - Raises: - PydanticCustomError: If the email is invalid. - - Note: - Note that: - - * Raw IP address (literal) domain parts are not allowed. - * `"John Doe "` style "pretty" email addresses are processed. - * Spaces are striped from the beginning and end of addresses, but no error is raised. - """ - if email_validator is None: - import_email_validator() - - if len(value) > MAX_EMAIL_LENGTH: - raise PydanticCustomError( - 'value_error', - 'value is not a valid email address: {reason}', - {'reason': f'Length must not exceed {MAX_EMAIL_LENGTH} characters'}, - ) - - m = pretty_email_regex.fullmatch(value) - name: str | None = None - if m: - unquoted_name, quoted_name, value = m.groups() - name = unquoted_name or quoted_name - - email = value.strip() - - try: - parts = email_validator.validate_email(email, check_deliverability=False) - except email_validator.EmailNotValidError as e: - raise PydanticCustomError( - 'value_error', 'value is not a valid email address: {reason}', {'reason': str(e.args[0])} - ) from e - - email = parts.normalized - assert email is not None - name = name or parts.local_part - return name, email - - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/parse.py b/functions/python/data-processor/pydantic/parse.py deleted file mode 100644 index 68b7f0464..000000000 --- a/functions/python/data-processor/pydantic/parse.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `parse` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/plugin/__init__.py b/functions/python/data-processor/pydantic/plugin/__init__.py deleted file mode 100644 index 840d20a09..000000000 --- a/functions/python/data-processor/pydantic/plugin/__init__.py +++ /dev/null @@ -1,193 +0,0 @@ -"""!!! abstract "Usage Documentation" - [Build a Plugin](../concepts/plugins.md#build-a-plugin) - -Plugin interface for Pydantic plugins, and related types. -""" - -from __future__ import annotations - -from typing import Any, Callable, Literal, NamedTuple - -from pydantic_core import CoreConfig, CoreSchema, ValidationError -from typing_extensions import Protocol, TypeAlias - -from pydantic.config import ExtraValues - -__all__ = ( - 'PydanticPluginProtocol', - 'BaseValidateHandlerProtocol', - 'ValidatePythonHandlerProtocol', - 'ValidateJsonHandlerProtocol', - 'ValidateStringsHandlerProtocol', - 'NewSchemaReturns', - 'SchemaTypePath', - 'SchemaKind', -) - -NewSchemaReturns: TypeAlias = 'tuple[ValidatePythonHandlerProtocol | None, ValidateJsonHandlerProtocol | None, ValidateStringsHandlerProtocol | None]' - - -class SchemaTypePath(NamedTuple): - """Path defining where `schema_type` was defined, or where `TypeAdapter` was called.""" - - module: str - name: str - - -SchemaKind: TypeAlias = Literal['BaseModel', 'TypeAdapter', 'dataclass', 'create_model', 'validate_call'] - - -class PydanticPluginProtocol(Protocol): - """Protocol defining the interface for Pydantic plugins.""" - - def new_schema_validator( - self, - schema: CoreSchema, - schema_type: Any, - schema_type_path: SchemaTypePath, - schema_kind: SchemaKind, - config: CoreConfig | None, - plugin_settings: dict[str, object], - ) -> tuple[ - ValidatePythonHandlerProtocol | None, ValidateJsonHandlerProtocol | None, ValidateStringsHandlerProtocol | None - ]: - """This method is called for each plugin every time a new [`SchemaValidator`][pydantic_core.SchemaValidator] - is created. - - It should return an event handler for each of the three validation methods, or `None` if the plugin does not - implement that method. - - Args: - schema: The schema to validate against. - schema_type: The original type which the schema was created from, e.g. the model class. - schema_type_path: Path defining where `schema_type` was defined, or where `TypeAdapter` was called. - schema_kind: The kind of schema to validate against. - config: The config to use for validation. - plugin_settings: Any plugin settings. - - Returns: - A tuple of optional event handlers for each of the three validation methods - - `validate_python`, `validate_json`, `validate_strings`. - """ - raise NotImplementedError('Pydantic plugins should implement `new_schema_validator`.') - - -class BaseValidateHandlerProtocol(Protocol): - """Base class for plugin callbacks protocols. - - You shouldn't implement this protocol directly, instead use one of the subclasses with adds the correctly - typed `on_error` method. - """ - - on_enter: Callable[..., None] - """`on_enter` is changed to be more specific on all subclasses""" - - def on_success(self, result: Any) -> None: - """Callback to be notified of successful validation. - - Args: - result: The result of the validation. - """ - return - - def on_error(self, error: ValidationError) -> None: - """Callback to be notified of validation errors. - - Args: - error: The validation error. - """ - return - - def on_exception(self, exception: Exception) -> None: - """Callback to be notified of validation exceptions. - - Args: - exception: The exception raised during validation. - """ - return - - -class ValidatePythonHandlerProtocol(BaseValidateHandlerProtocol, Protocol): - """Event handler for `SchemaValidator.validate_python`.""" - - def on_enter( - self, - input: Any, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - from_attributes: bool | None = None, - context: Any | None = None, - self_instance: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> None: - """Callback to be notified of validation start, and create an instance of the event handler. - - Args: - input: The input to be validated. - strict: Whether to validate the object in strict mode. - extra: Whether to ignore, allow, or forbid extra data during model validation. - from_attributes: Whether to validate objects as inputs by extracting attributes. - context: The context to use for validation, this is passed to functional validators. - self_instance: An instance of a model to set attributes on from validation, this is used when running - validation from the `__init__` method of a model. - by_alias: Whether to use the field's alias to match the input data to an attribute. - by_name: Whether to use the field's name to match the input data to an attribute. - """ - - -class ValidateJsonHandlerProtocol(BaseValidateHandlerProtocol, Protocol): - """Event handler for `SchemaValidator.validate_json`.""" - - def on_enter( - self, - input: str | bytes | bytearray, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - context: Any | None = None, - self_instance: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> None: - """Callback to be notified of validation start, and create an instance of the event handler. - - Args: - input: The JSON data to be validated. - strict: Whether to validate the object in strict mode. - extra: Whether to ignore, allow, or forbid extra data during model validation. - context: The context to use for validation, this is passed to functional validators. - self_instance: An instance of a model to set attributes on from validation, this is used when running - validation from the `__init__` method of a model. - by_alias: Whether to use the field's alias to match the input data to an attribute. - by_name: Whether to use the field's name to match the input data to an attribute. - """ - - -StringInput: TypeAlias = 'dict[str, StringInput]' - - -class ValidateStringsHandlerProtocol(BaseValidateHandlerProtocol, Protocol): - """Event handler for `SchemaValidator.validate_strings`.""" - - def on_enter( - self, - input: StringInput, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - context: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> None: - """Callback to be notified of validation start, and create an instance of the event handler. - - Args: - input: The string data to be validated. - strict: Whether to validate the object in strict mode. - extra: Whether to ignore, allow, or forbid extra data during model validation. - context: The context to use for validation, this is passed to functional validators. - by_alias: Whether to use the field's alias to match the input data to an attribute. - by_name: Whether to use the field's name to match the input data to an attribute. - """ diff --git a/functions/python/data-processor/pydantic/plugin/_loader.py b/functions/python/data-processor/pydantic/plugin/_loader.py deleted file mode 100644 index e8226fafe..000000000 --- a/functions/python/data-processor/pydantic/plugin/_loader.py +++ /dev/null @@ -1,58 +0,0 @@ -from __future__ import annotations - -import importlib.metadata as importlib_metadata -import os -import warnings -from collections.abc import Iterable -from typing import TYPE_CHECKING, Final - -if TYPE_CHECKING: - from . import PydanticPluginProtocol - - -PYDANTIC_ENTRY_POINT_GROUP: Final[str] = 'pydantic' - -# cache of plugins -_plugins: dict[str, PydanticPluginProtocol] | None = None -# return no plugins while loading plugins to avoid recursion and errors while importing plugins -# this means that if plugins use pydantic -_loading_plugins: bool = False - - -def get_plugins() -> Iterable[PydanticPluginProtocol]: - """Load plugins for Pydantic. - - Inspired by: https://github.com/pytest-dev/pluggy/blob/1.3.0/src/pluggy/_manager.py#L376-L402 - """ - disabled_plugins = os.getenv('PYDANTIC_DISABLE_PLUGINS') - global _plugins, _loading_plugins - if _loading_plugins: - # this happens when plugins themselves use pydantic, we return no plugins - return () - elif disabled_plugins in ('__all__', '1', 'true'): - return () - elif _plugins is None: - _plugins = {} - # set _loading_plugins so any plugins that use pydantic don't themselves use plugins - _loading_plugins = True - try: - for dist in importlib_metadata.distributions(): - for entry_point in dist.entry_points: - if entry_point.group != PYDANTIC_ENTRY_POINT_GROUP: - continue - if entry_point.value in _plugins: - continue - if disabled_plugins is not None and entry_point.name in disabled_plugins.split(','): - continue - try: - _plugins[entry_point.value] = entry_point.load() - except (ImportError, AttributeError) as e: - warnings.warn( - f'{e.__class__.__name__} while loading the `{entry_point.name}` Pydantic plugin, ' - f'this plugin will not be installed.\n\n{e!r}', - stacklevel=2, - ) - finally: - _loading_plugins = False - - return _plugins.values() diff --git a/functions/python/data-processor/pydantic/plugin/_schema_validator.py b/functions/python/data-processor/pydantic/plugin/_schema_validator.py deleted file mode 100644 index 492cc39cb..000000000 --- a/functions/python/data-processor/pydantic/plugin/_schema_validator.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Pluggable schema validator for pydantic.""" - -from __future__ import annotations - -import functools -from collections.abc import Iterable -from typing import TYPE_CHECKING, Any, Callable, Literal, TypeVar - -from pydantic_core import CoreConfig, CoreSchema, SchemaValidator, ValidationError -from typing_extensions import ParamSpec - -if TYPE_CHECKING: - from . import BaseValidateHandlerProtocol, PydanticPluginProtocol, SchemaKind, SchemaTypePath - - -P = ParamSpec('P') -R = TypeVar('R') -Event = Literal['on_validate_python', 'on_validate_json', 'on_validate_strings'] -events: list[Event] = list(Event.__args__) # type: ignore - - -def create_schema_validator( - schema: CoreSchema, - schema_type: Any, - schema_type_module: str, - schema_type_name: str, - schema_kind: SchemaKind, - config: CoreConfig | None = None, - plugin_settings: dict[str, Any] | None = None, - _use_prebuilt: bool = True, -) -> SchemaValidator | PluggableSchemaValidator: - """Create a `SchemaValidator` or `PluggableSchemaValidator` if plugins are installed. - - Returns: - If plugins are installed then return `PluggableSchemaValidator`, otherwise return `SchemaValidator`. - """ - from . import SchemaTypePath - from ._loader import get_plugins - - plugins = get_plugins() - if plugins: - return PluggableSchemaValidator( - schema, - schema_type, - SchemaTypePath(schema_type_module, schema_type_name), - schema_kind, - config, - plugins, - plugin_settings or {}, - _use_prebuilt=_use_prebuilt, - ) - else: - return SchemaValidator(schema, config, _use_prebuilt=_use_prebuilt) - - -class PluggableSchemaValidator: - """Pluggable schema validator.""" - - __slots__ = '_schema_validator', 'validate_json', 'validate_python', 'validate_strings' - - def __init__( - self, - schema: CoreSchema, - schema_type: Any, - schema_type_path: SchemaTypePath, - schema_kind: SchemaKind, - config: CoreConfig | None, - plugins: Iterable[PydanticPluginProtocol], - plugin_settings: dict[str, Any], - _use_prebuilt: bool = True, - ) -> None: - self._schema_validator = SchemaValidator(schema, config, _use_prebuilt=_use_prebuilt) - - python_event_handlers: list[BaseValidateHandlerProtocol] = [] - json_event_handlers: list[BaseValidateHandlerProtocol] = [] - strings_event_handlers: list[BaseValidateHandlerProtocol] = [] - for plugin in plugins: - try: - p, j, s = plugin.new_schema_validator( - schema, schema_type, schema_type_path, schema_kind, config, plugin_settings - ) - except TypeError as e: # pragma: no cover - raise TypeError(f'Error using plugin `{plugin.__module__}:{plugin.__class__.__name__}`: {e}') from e - if p is not None: - python_event_handlers.append(p) - if j is not None: - json_event_handlers.append(j) - if s is not None: - strings_event_handlers.append(s) - - self.validate_python = build_wrapper(self._schema_validator.validate_python, python_event_handlers) - self.validate_json = build_wrapper(self._schema_validator.validate_json, json_event_handlers) - self.validate_strings = build_wrapper(self._schema_validator.validate_strings, strings_event_handlers) - - def __getattr__(self, name: str) -> Any: - return getattr(self._schema_validator, name) - - -def build_wrapper(func: Callable[P, R], event_handlers: list[BaseValidateHandlerProtocol]) -> Callable[P, R]: - if not event_handlers: - return func - else: - on_enters = tuple(h.on_enter for h in event_handlers if filter_handlers(h, 'on_enter')) - on_successes = tuple(h.on_success for h in event_handlers if filter_handlers(h, 'on_success')) - on_errors = tuple(h.on_error for h in event_handlers if filter_handlers(h, 'on_error')) - on_exceptions = tuple(h.on_exception for h in event_handlers if filter_handlers(h, 'on_exception')) - - @functools.wraps(func) - def wrapper(*args: P.args, **kwargs: P.kwargs) -> R: - for on_enter_handler in on_enters: - on_enter_handler(*args, **kwargs) - - try: - result = func(*args, **kwargs) - except ValidationError as error: - for on_error_handler in on_errors: - on_error_handler(error) - raise - except Exception as exception: - for on_exception_handler in on_exceptions: - on_exception_handler(exception) - raise - else: - for on_success_handler in on_successes: - on_success_handler(result) - return result - - return wrapper - - -def filter_handlers(handler_cls: BaseValidateHandlerProtocol, method_name: str) -> bool: - """Filter out handler methods which are not implemented by the plugin directly - e.g. those that are missing - or are inherited from the protocol. - """ - handler = getattr(handler_cls, method_name, None) - if handler is None: - return False - elif handler.__module__ == 'pydantic.plugin': - # this is the original handler, from the protocol due to runtime inheritance - # we don't want to call it - return False - else: - return True diff --git a/functions/python/data-processor/pydantic/py.typed b/functions/python/data-processor/pydantic/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/pydantic/root_model.py b/functions/python/data-processor/pydantic/root_model.py deleted file mode 100644 index d2900433e..000000000 --- a/functions/python/data-processor/pydantic/root_model.py +++ /dev/null @@ -1,157 +0,0 @@ -"""RootModel class and type definitions.""" - -from __future__ import annotations as _annotations - -from copy import copy, deepcopy -from typing import TYPE_CHECKING, Any, Generic, Literal, TypeVar - -from pydantic_core import PydanticUndefined -from typing_extensions import Self, dataclass_transform - -from . import PydanticUserError -from ._internal import _model_construction, _repr -from .main import BaseModel, _object_setattr - -if TYPE_CHECKING: - from .fields import Field as PydanticModelField - from .fields import PrivateAttr as PydanticModelPrivateAttr - - # dataclass_transform could be applied to RootModel directly, but `ModelMetaclass`'s dataclass_transform - # takes priority (at least with pyright). We trick type checkers into thinking we apply dataclass_transform - # on a new metaclass. - @dataclass_transform(kw_only_default=False, field_specifiers=(PydanticModelField, PydanticModelPrivateAttr)) - class _RootModelMetaclass(_model_construction.ModelMetaclass): ... -else: - _RootModelMetaclass = _model_construction.ModelMetaclass - -__all__ = ('RootModel',) - -RootModelRootType = TypeVar('RootModelRootType') - - -class RootModel(BaseModel, Generic[RootModelRootType], metaclass=_RootModelMetaclass): - """!!! abstract "Usage Documentation" - [`RootModel` and Custom Root Types](../concepts/models.md#rootmodel-and-custom-root-types) - - A Pydantic `BaseModel` for the root object of the model. - - Attributes: - root: The root object of the model. - __pydantic_root_model__: Whether the model is a RootModel. - __pydantic_private__: Private fields in the model. - __pydantic_extra__: Extra fields in the model. - - """ - - __pydantic_root_model__ = True - __pydantic_private__ = None - __pydantic_extra__ = None - - root: RootModelRootType - - def __init_subclass__(cls, **kwargs): - extra = cls.model_config.get('extra') - if extra is not None: - raise PydanticUserError( - "`RootModel` does not support setting `model_config['extra']`", code='root-model-extra' - ) - super().__init_subclass__(**kwargs) - - def __init__(self, /, root: RootModelRootType = PydanticUndefined, **data) -> None: # type: ignore - __tracebackhide__ = True - if data: - if root is not PydanticUndefined: - raise ValueError( - '"RootModel.__init__" accepts either a single positional argument or arbitrary keyword arguments' - ) - root = data # type: ignore - self.__pydantic_validator__.validate_python(root, self_instance=self) - - __init__.__pydantic_base_init__ = True # pyright: ignore[reportFunctionMemberAccess] - - @classmethod - def model_construct(cls, root: RootModelRootType, _fields_set: set[str] | None = None) -> Self: # type: ignore - """Create a new model using the provided root object and update fields set. - - Args: - root: The root object of the model. - _fields_set: The set of fields to be updated. - - Returns: - The new model. - - Raises: - NotImplemented: If the model is not a subclass of `RootModel`. - """ - return super().model_construct(root=root, _fields_set=_fields_set) - - def __getstate__(self) -> dict[Any, Any]: - return { - '__dict__': self.__dict__, - '__pydantic_fields_set__': self.__pydantic_fields_set__, - } - - def __setstate__(self, state: dict[Any, Any]) -> None: - _object_setattr(self, '__pydantic_fields_set__', state['__pydantic_fields_set__']) - _object_setattr(self, '__dict__', state['__dict__']) - - def __copy__(self) -> Self: - """Returns a shallow copy of the model.""" - cls = type(self) - m = cls.__new__(cls) - new_dict = copy(self.__dict__) - new_dict['root'] = copy(self.__dict__['root']) - _object_setattr(m, '__dict__', new_dict) - _object_setattr(m, '__pydantic_fields_set__', copy(self.__pydantic_fields_set__)) - return m - - def __deepcopy__(self, memo: dict[int, Any] | None = None) -> Self: - """Returns a deep copy of the model.""" - cls = type(self) - m = cls.__new__(cls) - _object_setattr(m, '__dict__', deepcopy(self.__dict__, memo=memo)) - # This next line doesn't need a deepcopy because __pydantic_fields_set__ is a set[str], - # and attempting a deepcopy would be marginally slower. - _object_setattr(m, '__pydantic_fields_set__', copy(self.__pydantic_fields_set__)) - return m - - if TYPE_CHECKING: - - def model_dump( # type: ignore - self, - *, - mode: Literal['json', 'python'] | str = 'python', - include: Any = None, - exclude: Any = None, - context: dict[str, Any] | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - serialize_as_any: bool = False, - ) -> Any: - """This method is included just to get a more accurate return type for type checkers. - It is included in this `if TYPE_CHECKING:` block since no override is actually necessary. - - See the documentation of `BaseModel.model_dump` for more details about the arguments. - - Generally, this method will have a return type of `RootModelRootType`, assuming that `RootModelRootType` is - not a `BaseModel` subclass. If `RootModelRootType` is a `BaseModel` subclass, then the return - type will likely be `dict[str, Any]`, as `model_dump` calls are recursive. The return type could - even be something different, in the case of a custom serializer. - Thus, `Any` is used here to catch all of these cases. - """ - ... - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, RootModel): - return NotImplemented - return self.__pydantic_fields__['root'].annotation == other.__pydantic_fields__[ - 'root' - ].annotation and super().__eq__(other) - - def __repr_args__(self) -> _repr.ReprArgs: - yield 'root', self.root diff --git a/functions/python/data-processor/pydantic/schema.py b/functions/python/data-processor/pydantic/schema.py deleted file mode 100644 index a3245a61a..000000000 --- a/functions/python/data-processor/pydantic/schema.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `schema` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/tools.py b/functions/python/data-processor/pydantic/tools.py deleted file mode 100644 index fdc68c4f4..000000000 --- a/functions/python/data-processor/pydantic/tools.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `tools` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/type_adapter.py b/functions/python/data-processor/pydantic/type_adapter.py deleted file mode 100644 index d962305f2..000000000 --- a/functions/python/data-processor/pydantic/type_adapter.py +++ /dev/null @@ -1,801 +0,0 @@ -"""Type adapter specification.""" - -from __future__ import annotations as _annotations - -import sys -import types -from collections.abc import Callable, Iterable -from dataclasses import is_dataclass -from types import FrameType -from typing import ( - Any, - Generic, - Literal, - TypeVar, - cast, - final, - overload, -) - -from pydantic_core import CoreSchema, SchemaSerializer, SchemaValidator, Some -from typing_extensions import ParamSpec, is_typeddict - -from pydantic.errors import PydanticUserError -from pydantic.main import BaseModel, IncEx - -from ._internal import _config, _generate_schema, _mock_val_ser, _namespace_utils, _repr, _typing_extra, _utils -from .config import ConfigDict, ExtraValues -from .errors import PydanticUndefinedAnnotation -from .json_schema import ( - DEFAULT_REF_TEMPLATE, - GenerateJsonSchema, - JsonSchemaKeyT, - JsonSchemaMode, - JsonSchemaValue, -) -from .plugin._schema_validator import PluggableSchemaValidator, create_schema_validator - -T = TypeVar('T') -R = TypeVar('R') -P = ParamSpec('P') -TypeAdapterT = TypeVar('TypeAdapterT', bound='TypeAdapter') - - -def _getattr_no_parents(obj: Any, attribute: str) -> Any: - """Returns the attribute value without attempting to look up attributes from parent types.""" - if hasattr(obj, '__dict__'): - try: - return obj.__dict__[attribute] - except KeyError: - pass - - slots = getattr(obj, '__slots__', None) - if slots is not None and attribute in slots: - return getattr(obj, attribute) - else: - raise AttributeError(attribute) - - -def _type_has_config(type_: Any) -> bool: - """Returns whether the type has config.""" - type_ = _typing_extra.annotated_type(type_) or type_ - try: - return issubclass(type_, BaseModel) or is_dataclass(type_) or is_typeddict(type_) - except TypeError: - # type is not a class - return False - - -@final -class TypeAdapter(Generic[T]): - """!!! abstract "Usage Documentation" - [`TypeAdapter`](../concepts/type_adapter.md) - - Type adapters provide a flexible way to perform validation and serialization based on a Python type. - - A `TypeAdapter` instance exposes some of the functionality from `BaseModel` instance methods - for types that do not have such methods (such as dataclasses, primitive types, and more). - - **Note:** `TypeAdapter` instances are not types, and cannot be used as type annotations for fields. - - Args: - type: The type associated with the `TypeAdapter`. - config: Configuration for the `TypeAdapter`, should be a dictionary conforming to - [`ConfigDict`][pydantic.config.ConfigDict]. - - !!! note - You cannot provide a configuration when instantiating a `TypeAdapter` if the type you're using - has its own config that cannot be overridden (ex: `BaseModel`, `TypedDict`, and `dataclass`). A - [`type-adapter-config-unused`](../errors/usage_errors.md#type-adapter-config-unused) error will - be raised in this case. - _parent_depth: Depth at which to search for the [parent frame][frame-objects]. This frame is used when - resolving forward annotations during schema building, by looking for the globals and locals of this - frame. Defaults to 2, which will result in the frame where the `TypeAdapter` was instantiated. - - !!! note - This parameter is named with an underscore to suggest its private nature and discourage use. - It may be deprecated in a minor version, so we only recommend using it if you're comfortable - with potential change in behavior/support. It's default value is 2 because internally, - the `TypeAdapter` class makes another call to fetch the frame. - module: The module that passes to plugin if provided. - - Attributes: - core_schema: The core schema for the type. - validator: The schema validator for the type. - serializer: The schema serializer for the type. - pydantic_complete: Whether the core schema for the type is successfully built. - - ??? tip "Compatibility with `mypy`" - Depending on the type used, `mypy` might raise an error when instantiating a `TypeAdapter`. As a workaround, you can explicitly - annotate your variable: - - ```py - from typing import Union - - from pydantic import TypeAdapter - - ta: TypeAdapter[Union[str, int]] = TypeAdapter(Union[str, int]) # type: ignore[arg-type] - ``` - - ??? info "Namespace management nuances and implementation details" - - Here, we collect some notes on namespace management, and subtle differences from `BaseModel`: - - `BaseModel` uses its own `__module__` to find out where it was defined - and then looks for symbols to resolve forward references in those globals. - On the other hand, `TypeAdapter` can be initialized with arbitrary objects, - which may not be types and thus do not have a `__module__` available. - So instead we look at the globals in our parent stack frame. - - It is expected that the `ns_resolver` passed to this function will have the correct - namespace for the type we're adapting. See the source code for `TypeAdapter.__init__` - and `TypeAdapter.rebuild` for various ways to construct this namespace. - - This works for the case where this function is called in a module that - has the target of forward references in its scope, but - does not always work for more complex cases. - - For example, take the following: - - ```python {title="a.py"} - IntList = list[int] - OuterDict = dict[str, 'IntList'] - ``` - - ```python {test="skip" title="b.py"} - from a import OuterDict - - from pydantic import TypeAdapter - - IntList = int # replaces the symbol the forward reference is looking for - v = TypeAdapter(OuterDict) - v({'x': 1}) # should fail but doesn't - ``` - - If `OuterDict` were a `BaseModel`, this would work because it would resolve - the forward reference within the `a.py` namespace. - But `TypeAdapter(OuterDict)` can't determine what module `OuterDict` came from. - - In other words, the assumption that _all_ forward references exist in the - module we are being called from is not technically always true. - Although most of the time it is and it works fine for recursive models and such, - `BaseModel`'s behavior isn't perfect either and _can_ break in similar ways, - so there is no right or wrong between the two. - - But at the very least this behavior is _subtly_ different from `BaseModel`'s. - """ - - core_schema: CoreSchema - validator: SchemaValidator | PluggableSchemaValidator - serializer: SchemaSerializer - pydantic_complete: bool - - @overload - def __init__( - self, - type: type[T], - *, - config: ConfigDict | None = ..., - _parent_depth: int = ..., - module: str | None = ..., - ) -> None: ... - - # This second overload is for unsupported special forms (such as Annotated, Union, etc.) - # Currently there is no way to type this correctly - # See https://github.com/python/typing/pull/1618 - @overload - def __init__( - self, - type: Any, - *, - config: ConfigDict | None = ..., - _parent_depth: int = ..., - module: str | None = ..., - ) -> None: ... - - def __init__( - self, - type: Any, - *, - config: ConfigDict | None = None, - _parent_depth: int = 2, - module: str | None = None, - ) -> None: - if _type_has_config(type) and config is not None: - raise PydanticUserError( - 'Cannot use `config` when the type is a BaseModel, dataclass or TypedDict.' - ' These types can have their own config and setting the config via the `config`' - ' parameter to TypeAdapter will not override it, thus the `config` you passed to' - ' TypeAdapter becomes meaningless, which is probably not what you want.', - code='type-adapter-config-unused', - ) - - self._type = type - self._config = config - self._parent_depth = _parent_depth - self.pydantic_complete = False - - parent_frame = self._fetch_parent_frame() - if isinstance(type, types.FunctionType): - # Special case functions, which are *not* pushed to the `NsResolver` stack and without this special case - # would only have access to the parent namespace where the `TypeAdapter` was instantiated (if the function is defined - # in another module, we need to look at that module's globals). - if parent_frame is not None: - # `f_locals` is the namespace where the type adapter was instantiated (~ to `f_globals` if at the module level): - parent_ns = parent_frame.f_locals - else: # pragma: no cover - parent_ns = None - globalns, localns = _namespace_utils.ns_for_function( - type, - parent_namespace=parent_ns, - ) - parent_namespace = None - else: - if parent_frame is not None: - globalns = parent_frame.f_globals - # Do not provide a local ns if the type adapter happens to be instantiated at the module level: - localns = parent_frame.f_locals if parent_frame.f_locals is not globalns else {} - else: # pragma: no cover - globalns = {} - localns = {} - parent_namespace = localns - - self._module_name = module or cast(str, globalns.get('__name__', '')) - self._init_core_attrs( - ns_resolver=_namespace_utils.NsResolver( - namespaces_tuple=_namespace_utils.NamespacesTuple(locals=localns, globals=globalns), - parent_namespace=parent_namespace, - ), - force=False, - ) - - def _fetch_parent_frame(self) -> FrameType | None: - frame = sys._getframe(self._parent_depth) - if frame.f_globals.get('__name__') == 'typing': - # Because `TypeAdapter` is generic, explicitly parametrizing the class results - # in a `typing._GenericAlias` instance, which proxies instantiation calls to the - # "real" `TypeAdapter` class and thus adding an extra frame to the call. To avoid - # pulling anything from the `typing` module, use the correct frame (the one before): - return frame.f_back - - return frame - - def _init_core_attrs( - self, ns_resolver: _namespace_utils.NsResolver, force: bool, raise_errors: bool = False - ) -> bool: - """Initialize the core schema, validator, and serializer for the type. - - Args: - ns_resolver: The namespace resolver to use when building the core schema for the adapted type. - force: Whether to force the construction of the core schema, validator, and serializer. - If `force` is set to `False` and `_defer_build` is `True`, the core schema, validator, and serializer will be set to mocks. - raise_errors: Whether to raise errors if initializing any of the core attrs fails. - - Returns: - `True` if the core schema, validator, and serializer were successfully initialized, otherwise `False`. - - Raises: - PydanticUndefinedAnnotation: If `PydanticUndefinedAnnotation` occurs in`__get_pydantic_core_schema__` - and `raise_errors=True`. - """ - if not force and self._defer_build: - _mock_val_ser.set_type_adapter_mocks(self) - self.pydantic_complete = False - return False - - try: - self.core_schema = _getattr_no_parents(self._type, '__pydantic_core_schema__') - self.validator = _getattr_no_parents(self._type, '__pydantic_validator__') - self.serializer = _getattr_no_parents(self._type, '__pydantic_serializer__') - - # TODO: we don't go through the rebuild logic here directly because we don't want - # to repeat all of the namespace fetching logic that we've already done - # so we simply skip to the block below that does the actual schema generation - if ( - isinstance(self.core_schema, _mock_val_ser.MockCoreSchema) - or isinstance(self.validator, _mock_val_ser.MockValSer) - or isinstance(self.serializer, _mock_val_ser.MockValSer) - ): - raise AttributeError() - except AttributeError: - config_wrapper = _config.ConfigWrapper(self._config) - - schema_generator = _generate_schema.GenerateSchema(config_wrapper, ns_resolver=ns_resolver) - - try: - core_schema = schema_generator.generate_schema(self._type) - except PydanticUndefinedAnnotation: - if raise_errors: - raise - _mock_val_ser.set_type_adapter_mocks(self) - return False - - try: - self.core_schema = schema_generator.clean_schema(core_schema) - except _generate_schema.InvalidSchemaError: - _mock_val_ser.set_type_adapter_mocks(self) - return False - - core_config = config_wrapper.core_config(None) - - self.validator = create_schema_validator( - schema=self.core_schema, - schema_type=self._type, - schema_type_module=self._module_name, - schema_type_name=str(self._type), - schema_kind='TypeAdapter', - config=core_config, - plugin_settings=config_wrapper.plugin_settings, - ) - self.serializer = SchemaSerializer(self.core_schema, core_config) - - self.pydantic_complete = True - return True - - @property - def _defer_build(self) -> bool: - config = self._config if self._config is not None else self._model_config - if config: - return config.get('defer_build') is True - return False - - @property - def _model_config(self) -> ConfigDict | None: - type_: Any = _typing_extra.annotated_type(self._type) or self._type # Eg FastAPI heavily uses Annotated - if _utils.lenient_issubclass(type_, BaseModel): - return type_.model_config - return getattr(type_, '__pydantic_config__', None) - - def __repr__(self) -> str: - return f'TypeAdapter({_repr.display_as_type(self._type)})' - - def rebuild( - self, - *, - force: bool = False, - raise_errors: bool = True, - _parent_namespace_depth: int = 2, - _types_namespace: _namespace_utils.MappingNamespace | None = None, - ) -> bool | None: - """Try to rebuild the pydantic-core schema for the adapter's type. - - This may be necessary when one of the annotations is a ForwardRef which could not be resolved during - the initial attempt to build the schema, and automatic rebuilding fails. - - Args: - force: Whether to force the rebuilding of the type adapter's schema, defaults to `False`. - raise_errors: Whether to raise errors, defaults to `True`. - _parent_namespace_depth: Depth at which to search for the [parent frame][frame-objects]. This - frame is used when resolving forward annotations during schema rebuilding, by looking for - the locals of this frame. Defaults to 2, which will result in the frame where the method - was called. - _types_namespace: An explicit types namespace to use, instead of using the local namespace - from the parent frame. Defaults to `None`. - - Returns: - Returns `None` if the schema is already "complete" and rebuilding was not required. - If rebuilding _was_ required, returns `True` if rebuilding was successful, otherwise `False`. - """ - if not force and self.pydantic_complete: - return None - - if _types_namespace is not None: - rebuild_ns = _types_namespace - elif _parent_namespace_depth > 0: - rebuild_ns = _typing_extra.parent_frame_namespace(parent_depth=_parent_namespace_depth, force=True) or {} - else: - rebuild_ns = {} - - # we have to manually fetch globals here because there's no type on the stack of the NsResolver - # and so we skip the globalns = get_module_ns_of(typ) call that would normally happen - globalns = sys._getframe(max(_parent_namespace_depth - 1, 1)).f_globals - ns_resolver = _namespace_utils.NsResolver( - namespaces_tuple=_namespace_utils.NamespacesTuple(locals=rebuild_ns, globals=globalns), - parent_namespace=rebuild_ns, - ) - return self._init_core_attrs(ns_resolver=ns_resolver, force=True, raise_errors=raise_errors) - - def validate_python( - self, - object: Any, - /, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - from_attributes: bool | None = None, - context: Any | None = None, - experimental_allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> T: - """Validate a Python object against the model. - - Args: - object: The Python object to validate against the model. - strict: Whether to strictly check types. - extra: Whether to ignore, allow, or forbid extra data during model validation. - See the [`extra` configuration value][pydantic.ConfigDict.extra] for details. - from_attributes: Whether to extract data from object attributes. - context: Additional context to pass to the validator. - experimental_allow_partial: **Experimental** whether to enable - [partial validation](../concepts/experimental.md#partial-validation), e.g. to process streams. - * False / 'off': Default behavior, no partial validation. - * True / 'on': Enable partial validation. - * 'trailing-strings': Enable partial validation and allow trailing strings in the input. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - !!! note - When using `TypeAdapter` with a Pydantic `dataclass`, the use of the `from_attributes` - argument is not supported. - - Returns: - The validated object. - """ - if by_alias is False and by_name is not True: - raise PydanticUserError( - 'At least one of `by_alias` or `by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return self.validator.validate_python( - object, - strict=strict, - extra=extra, - from_attributes=from_attributes, - context=context, - allow_partial=experimental_allow_partial, - by_alias=by_alias, - by_name=by_name, - ) - - def validate_json( - self, - data: str | bytes | bytearray, - /, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - context: Any | None = None, - experimental_allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> T: - """!!! abstract "Usage Documentation" - [JSON Parsing](../concepts/json.md#json-parsing) - - Validate a JSON string or bytes against the model. - - Args: - data: The JSON data to validate against the model. - strict: Whether to strictly check types. - extra: Whether to ignore, allow, or forbid extra data during model validation. - See the [`extra` configuration value][pydantic.ConfigDict.extra] for details. - context: Additional context to use during validation. - experimental_allow_partial: **Experimental** whether to enable - [partial validation](../concepts/experimental.md#partial-validation), e.g. to process streams. - * False / 'off': Default behavior, no partial validation. - * True / 'on': Enable partial validation. - * 'trailing-strings': Enable partial validation and allow trailing strings in the input. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Returns: - The validated object. - """ - if by_alias is False and by_name is not True: - raise PydanticUserError( - 'At least one of `by_alias` or `by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return self.validator.validate_json( - data, - strict=strict, - extra=extra, - context=context, - allow_partial=experimental_allow_partial, - by_alias=by_alias, - by_name=by_name, - ) - - def validate_strings( - self, - obj: Any, - /, - *, - strict: bool | None = None, - extra: ExtraValues | None = None, - context: Any | None = None, - experimental_allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> T: - """Validate object contains string data against the model. - - Args: - obj: The object contains string data to validate. - strict: Whether to strictly check types. - extra: Whether to ignore, allow, or forbid extra data during model validation. - See the [`extra` configuration value][pydantic.ConfigDict.extra] for details. - context: Additional context to use during validation. - experimental_allow_partial: **Experimental** whether to enable - [partial validation](../concepts/experimental.md#partial-validation), e.g. to process streams. - * False / 'off': Default behavior, no partial validation. - * True / 'on': Enable partial validation. - * 'trailing-strings': Enable partial validation and allow trailing strings in the input. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Returns: - The validated object. - """ - if by_alias is False and by_name is not True: - raise PydanticUserError( - 'At least one of `by_alias` or `by_name` must be set to True.', - code='validate-by-alias-and-name-false', - ) - - return self.validator.validate_strings( - obj, - strict=strict, - extra=extra, - context=context, - allow_partial=experimental_allow_partial, - by_alias=by_alias, - by_name=by_name, - ) - - def get_default_value(self, *, strict: bool | None = None, context: Any | None = None) -> Some[T] | None: - """Get the default value for the wrapped type. - - Args: - strict: Whether to strictly check types. - context: Additional context to pass to the validator. - - Returns: - The default value wrapped in a `Some` if there is one or None if not. - """ - return self.validator.get_default_value(strict=strict, context=context) - - def dump_python( - self, - instance: T, - /, - *, - mode: Literal['json', 'python'] = 'python', - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - context: Any | None = None, - ) -> Any: - """Dump an instance of the adapted type to a Python object. - - Args: - instance: The Python object to serialize. - mode: The output format. - include: Fields to include in the output. - exclude: Fields to exclude from the output. - by_alias: Whether to use alias names for field names. - exclude_unset: Whether to exclude unset fields. - exclude_defaults: Whether to exclude fields with default values. - exclude_none: Whether to exclude fields with None values. - exclude_computed_fields: Whether to exclude computed fields. - While this can be useful for round-tripping, it is usually recommended to use the dedicated - `round_trip` parameter instead. - round_trip: Whether to output the serialized data in a way that is compatible with deserialization. - warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered. If not provided, - a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - context: Additional context to pass to the serializer. - - Returns: - The serialized object. - """ - return self.serializer.to_python( - instance, - mode=mode, - by_alias=by_alias, - include=include, - exclude=exclude, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - exclude_computed_fields=exclude_computed_fields, - round_trip=round_trip, - warnings=warnings, - fallback=fallback, - serialize_as_any=serialize_as_any, - polymorphic_serialization=polymorphic_serialization, - context=context, - ) - - def dump_json( - self, - instance: T, - /, - *, - indent: int | None = None, - ensure_ascii: bool = False, - include: IncEx | None = None, - exclude: IncEx | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - context: Any | None = None, - ) -> bytes: - """!!! abstract "Usage Documentation" - [JSON Serialization](../concepts/json.md#json-serialization) - - Serialize an instance of the adapted type to JSON. - - Args: - instance: The instance to be serialized. - indent: Number of spaces for JSON indentation. - ensure_ascii: If `True`, the output is guaranteed to have all incoming non-ASCII characters escaped. - If `False` (the default), these characters will be output as-is. - include: Fields to include. - exclude: Fields to exclude. - by_alias: Whether to use alias names for field names. - exclude_unset: Whether to exclude unset fields. - exclude_defaults: Whether to exclude fields with default values. - exclude_none: Whether to exclude fields with a value of `None`. - exclude_computed_fields: Whether to exclude computed fields. - While this can be useful for round-tripping, it is usually recommended to use the dedicated - `round_trip` parameter instead. - round_trip: Whether to serialize and deserialize the instance to ensure round-tripping. - warnings: How to handle serialization errors. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered. If not provided, - a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - context: Additional context to pass to the serializer. - - Returns: - The JSON representation of the given instance as bytes. - """ - return self.serializer.to_json( - instance, - indent=indent, - ensure_ascii=ensure_ascii, - include=include, - exclude=exclude, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - exclude_computed_fields=exclude_computed_fields, - round_trip=round_trip, - warnings=warnings, - fallback=fallback, - serialize_as_any=serialize_as_any, - polymorphic_serialization=polymorphic_serialization, - context=context, - ) - - def json_schema( - self, - *, - by_alias: bool = True, - ref_template: str = DEFAULT_REF_TEMPLATE, - union_format: Literal['any_of', 'primitive_type_array'] = 'any_of', - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, - mode: JsonSchemaMode = 'validation', - ) -> dict[str, Any]: - """Generate a JSON schema for the adapted type. - - Args: - by_alias: Whether to use alias names for field names. - ref_template: The format string used for generating $ref strings. - union_format: The format to use when combining schemas from unions together. Can be one of: - - - `'any_of'`: Use the [`anyOf`](https://json-schema.org/understanding-json-schema/reference/combining#anyOf) - keyword to combine schemas (the default). - - `'primitive_type_array'`: Use the [`type`](https://json-schema.org/understanding-json-schema/reference/type) - keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive - type (`string`, `boolean`, `null`, `integer` or `number`) or contains constraints/metadata, falls back to - `any_of`. - schema_generator: To override the logic used to generate the JSON schema, as a subclass of - `GenerateJsonSchema` with your desired modifications - mode: The mode in which to generate the schema. - schema_generator: The generator class used for creating the schema. - mode: The mode to use for schema generation. - - Returns: - The JSON schema for the model as a dictionary. - """ - schema_generator_instance = schema_generator( - by_alias=by_alias, ref_template=ref_template, union_format=union_format - ) - if isinstance(self.core_schema, _mock_val_ser.MockCoreSchema): - self.core_schema.rebuild() - assert not isinstance(self.core_schema, _mock_val_ser.MockCoreSchema), 'this is a bug! please report it' - return schema_generator_instance.generate(self.core_schema, mode=mode) - - @staticmethod - def json_schemas( - inputs: Iterable[tuple[JsonSchemaKeyT, JsonSchemaMode, TypeAdapter[Any]]], - /, - *, - by_alias: bool = True, - title: str | None = None, - description: str | None = None, - ref_template: str = DEFAULT_REF_TEMPLATE, - union_format: Literal['any_of', 'primitive_type_array'] = 'any_of', - schema_generator: type[GenerateJsonSchema] = GenerateJsonSchema, - ) -> tuple[dict[tuple[JsonSchemaKeyT, JsonSchemaMode], JsonSchemaValue], JsonSchemaValue]: - """Generate a JSON schema including definitions from multiple type adapters. - - Args: - inputs: Inputs to schema generation. The first two items will form the keys of the (first) - output mapping; the type adapters will provide the core schemas that get converted into - definitions in the output JSON schema. - by_alias: Whether to use alias names. - title: The title for the schema. - description: The description for the schema. - ref_template: The format string used for generating $ref strings. - union_format: The format to use when combining schemas from unions together. Can be one of: - - - `'any_of'`: Use the [`anyOf`](https://json-schema.org/understanding-json-schema/reference/combining#anyOf) - keyword to combine schemas (the default). - - `'primitive_type_array'`: Use the [`type`](https://json-schema.org/understanding-json-schema/reference/type) - keyword as an array of strings, containing each type of the combination. If any of the schemas is not a primitive - type (`string`, `boolean`, `null`, `integer` or `number`) or contains constraints/metadata, falls back to - `any_of`. - schema_generator: The generator class used for creating the schema. - - Returns: - A tuple where: - - - The first element is a dictionary whose keys are tuples of JSON schema key type and JSON mode, and - whose values are the JSON schema corresponding to that pair of inputs. (These schemas may have - JsonRef references to definitions that are defined in the second returned element.) - - The second element is a JSON schema containing all definitions referenced in the first returned - element, along with the optional title and description keys. - - """ - schema_generator_instance = schema_generator( - by_alias=by_alias, ref_template=ref_template, union_format=union_format - ) - - inputs_ = [] - for key, mode, adapter in inputs: - # This is the same pattern we follow for model json schemas - we attempt a core schema rebuild if we detect a mock - if isinstance(adapter.core_schema, _mock_val_ser.MockCoreSchema): - adapter.core_schema.rebuild() - assert not isinstance(adapter.core_schema, _mock_val_ser.MockCoreSchema), ( - 'this is a bug! please report it' - ) - inputs_.append((key, mode, adapter.core_schema)) - - json_schemas_map, definitions = schema_generator_instance.generate_definitions(inputs_) - - json_schema: dict[str, Any] = {} - if definitions: - json_schema['$defs'] = definitions - if title: - json_schema['title'] = title - if description: - json_schema['description'] = description - - return json_schemas_map, json_schema diff --git a/functions/python/data-processor/pydantic/types.py b/functions/python/data-processor/pydantic/types.py deleted file mode 100644 index 467bc8285..000000000 --- a/functions/python/data-processor/pydantic/types.py +++ /dev/null @@ -1,3310 +0,0 @@ -"""The types module contains custom types used by pydantic.""" - -from __future__ import annotations as _annotations - -import base64 -import dataclasses as _dataclasses -import re -from collections.abc import Hashable, Iterator -from datetime import date, datetime -from decimal import Decimal -from enum import Enum -from pathlib import Path -from re import Pattern -from types import ModuleType -from typing import ( - TYPE_CHECKING, - Annotated, - Any, - Callable, - ClassVar, - Generic, - Literal, - TypeVar, - Union, - cast, -) -from uuid import UUID - -import annotated_types -from annotated_types import BaseMetadata, MaxLen, MinLen -from pydantic_core import CoreSchema, PydanticCustomError, SchemaSerializer, core_schema -from typing_extensions import Protocol, TypeAlias, TypeAliasType, deprecated, get_args, get_origin - -from ._internal import _fields, _internal_dataclass, _utils, _validators -from ._migration import getattr_migration -from .annotated_handlers import GetCoreSchemaHandler, GetJsonSchemaHandler -from .errors import PydanticUserError -from .json_schema import JsonSchemaValue -from .warnings import PydanticDeprecatedSince20 - -if TYPE_CHECKING: - from ._internal._core_metadata import CoreMetadata - -__all__ = ( - 'Strict', - 'StrictStr', - 'SocketPath', - 'conbytes', - 'conlist', - 'conset', - 'confrozenset', - 'constr', - 'ImportString', - 'conint', - 'PositiveInt', - 'NegativeInt', - 'NonNegativeInt', - 'NonPositiveInt', - 'confloat', - 'PositiveFloat', - 'NegativeFloat', - 'NonNegativeFloat', - 'NonPositiveFloat', - 'FiniteFloat', - 'condecimal', - 'UUID1', - 'UUID3', - 'UUID4', - 'UUID5', - 'UUID6', - 'UUID7', - 'UUID8', - 'FilePath', - 'DirectoryPath', - 'NewPath', - 'Json', - 'Secret', - 'SecretStr', - 'SecretBytes', - 'StrictBool', - 'StrictBytes', - 'StrictInt', - 'StrictFloat', - 'PaymentCardNumber', - 'ByteSize', - 'PastDate', - 'FutureDate', - 'PastDatetime', - 'FutureDatetime', - 'condate', - 'AwareDatetime', - 'NaiveDatetime', - 'AllowInfNan', - 'EncoderProtocol', - 'EncodedBytes', - 'EncodedStr', - 'Base64Encoder', - 'Base64Bytes', - 'Base64Str', - 'Base64UrlBytes', - 'Base64UrlStr', - 'GetPydanticSchema', - 'StringConstraints', - 'Tag', - 'Discriminator', - 'JsonValue', - 'OnErrorOmit', - 'FailFast', -) - - -T = TypeVar('T') - - -@_dataclasses.dataclass -class Strict(_fields.PydanticMetadata, BaseMetadata): - """!!! abstract "Usage Documentation" - [Strict Mode with `Annotated` `Strict`](../concepts/strict_mode.md#strict-mode-with-annotated-strict) - - A field metadata class to indicate that a field should be validated in strict mode. - Use this class as an annotation via [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated), as seen below. - - Attributes: - strict: Whether to validate the field in strict mode. - - Example: - ```python - from typing import Annotated - - from pydantic.types import Strict - - StrictBool = Annotated[bool, Strict()] - ``` - """ - - strict: bool = True - - def __hash__(self) -> int: - return hash(self.strict) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BOOLEAN TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -StrictBool = Annotated[bool, Strict()] -"""A boolean that must be either ``True`` or ``False``.""" - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ INTEGER TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -def conint( - *, - strict: bool | None = None, - gt: int | None = None, - ge: int | None = None, - lt: int | None = None, - le: int | None = None, - multiple_of: int | None = None, -) -> type[int]: - """ - !!! warning "Discouraged" - This function is **discouraged** in favor of using - [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) with - [`Field`][pydantic.fields.Field] instead. - - This function will be **deprecated** in Pydantic 3.0. - - The reason is that `conint` returns a type, which doesn't play well with static analysis tools. - - === ":x: Don't do this" - ```python - from pydantic import BaseModel, conint - - class Foo(BaseModel): - bar: conint(strict=True, gt=0) - ``` - - === ":white_check_mark: Do this" - ```python - from typing import Annotated - - from pydantic import BaseModel, Field - - class Foo(BaseModel): - bar: Annotated[int, Field(strict=True, gt=0)] - ``` - - A wrapper around `int` that allows for additional constraints. - - Args: - strict: Whether to validate the integer in strict mode. Defaults to `None`. - gt: The value must be greater than this. - ge: The value must be greater than or equal to this. - lt: The value must be less than this. - le: The value must be less than or equal to this. - multiple_of: The value must be a multiple of this. - - Returns: - The wrapped integer type. - - ```python - from pydantic import BaseModel, ValidationError, conint - - class ConstrainedExample(BaseModel): - constrained_int: conint(gt=1) - - m = ConstrainedExample(constrained_int=2) - print(repr(m)) - #> ConstrainedExample(constrained_int=2) - - try: - ConstrainedExample(constrained_int=0) - except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than', - 'loc': ('constrained_int',), - 'msg': 'Input should be greater than 1', - 'input': 0, - 'ctx': {'gt': 1}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than', - } - ] - ''' - ``` - - """ # noqa: D212 - return Annotated[ # pyright: ignore[reportReturnType] - int, - Strict(strict) if strict is not None else None, - annotated_types.Interval(gt=gt, ge=ge, lt=lt, le=le), - annotated_types.MultipleOf(multiple_of) if multiple_of is not None else None, - ] - - -PositiveInt = Annotated[int, annotated_types.Gt(0)] -"""An integer that must be greater than zero. - -```python -from pydantic import BaseModel, PositiveInt, ValidationError - -class Model(BaseModel): - positive_int: PositiveInt - -m = Model(positive_int=1) -print(repr(m)) -#> Model(positive_int=1) - -try: - Model(positive_int=-1) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than', - 'loc': ('positive_int',), - 'msg': 'Input should be greater than 0', - 'input': -1, - 'ctx': {'gt': 0}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than', - } - ] - ''' -``` -""" -NegativeInt = Annotated[int, annotated_types.Lt(0)] -"""An integer that must be less than zero. - -```python -from pydantic import BaseModel, NegativeInt, ValidationError - -class Model(BaseModel): - negative_int: NegativeInt - -m = Model(negative_int=-1) -print(repr(m)) -#> Model(negative_int=-1) - -try: - Model(negative_int=1) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'less_than', - 'loc': ('negative_int',), - 'msg': 'Input should be less than 0', - 'input': 1, - 'ctx': {'lt': 0}, - 'url': 'https://errors.pydantic.dev/2/v/less_than', - } - ] - ''' -``` -""" -NonPositiveInt = Annotated[int, annotated_types.Le(0)] -"""An integer that must be less than or equal to zero. - -```python -from pydantic import BaseModel, NonPositiveInt, ValidationError - -class Model(BaseModel): - non_positive_int: NonPositiveInt - -m = Model(non_positive_int=0) -print(repr(m)) -#> Model(non_positive_int=0) - -try: - Model(non_positive_int=1) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'less_than_equal', - 'loc': ('non_positive_int',), - 'msg': 'Input should be less than or equal to 0', - 'input': 1, - 'ctx': {'le': 0}, - 'url': 'https://errors.pydantic.dev/2/v/less_than_equal', - } - ] - ''' -``` -""" -NonNegativeInt = Annotated[int, annotated_types.Ge(0)] -"""An integer that must be greater than or equal to zero. - -```python -from pydantic import BaseModel, NonNegativeInt, ValidationError - -class Model(BaseModel): - non_negative_int: NonNegativeInt - -m = Model(non_negative_int=0) -print(repr(m)) -#> Model(non_negative_int=0) - -try: - Model(non_negative_int=-1) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than_equal', - 'loc': ('non_negative_int',), - 'msg': 'Input should be greater than or equal to 0', - 'input': -1, - 'ctx': {'ge': 0}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than_equal', - } - ] - ''' -``` -""" -StrictInt = Annotated[int, Strict()] -"""An integer that must be validated in strict mode. - -```python -from pydantic import BaseModel, StrictInt, ValidationError - -class StrictIntModel(BaseModel): - strict_int: StrictInt - -try: - StrictIntModel(strict_int=3.14159) -except ValidationError as e: - print(e) - ''' - 1 validation error for StrictIntModel - strict_int - Input should be a valid integer [type=int_type, input_value=3.14159, input_type=float] - ''' -``` -""" - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FLOAT TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -@_dataclasses.dataclass -class AllowInfNan(_fields.PydanticMetadata): - """A field metadata class to indicate that a field should allow `-inf`, `inf`, and `nan`. - - Use this class as an annotation via [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated), as seen below. - - Attributes: - allow_inf_nan: Whether to allow `-inf`, `inf`, and `nan`. Defaults to `True`. - - Example: - ```python - from typing import Annotated - - from pydantic.types import AllowInfNan - - LaxFloat = Annotated[float, AllowInfNan()] - ``` - """ - - allow_inf_nan: bool = True - - def __hash__(self) -> int: - return hash(self.allow_inf_nan) - - -def confloat( - *, - strict: bool | None = None, - gt: float | None = None, - ge: float | None = None, - lt: float | None = None, - le: float | None = None, - multiple_of: float | None = None, - allow_inf_nan: bool | None = None, -) -> type[float]: - """ - !!! warning "Discouraged" - This function is **discouraged** in favor of using - [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) with - [`Field`][pydantic.fields.Field] instead. - - This function will be **deprecated** in Pydantic 3.0. - - The reason is that `confloat` returns a type, which doesn't play well with static analysis tools. - - === ":x: Don't do this" - ```python - from pydantic import BaseModel, confloat - - class Foo(BaseModel): - bar: confloat(strict=True, gt=0) - ``` - - === ":white_check_mark: Do this" - ```python - from typing import Annotated - - from pydantic import BaseModel, Field - - class Foo(BaseModel): - bar: Annotated[float, Field(strict=True, gt=0)] - ``` - - A wrapper around `float` that allows for additional constraints. - - Args: - strict: Whether to validate the float in strict mode. - gt: The value must be greater than this. - ge: The value must be greater than or equal to this. - lt: The value must be less than this. - le: The value must be less than or equal to this. - multiple_of: The value must be a multiple of this. - allow_inf_nan: Whether to allow `-inf`, `inf`, and `nan`. - - Returns: - The wrapped float type. - - ```python - from pydantic import BaseModel, ValidationError, confloat - - class ConstrainedExample(BaseModel): - constrained_float: confloat(gt=1.0) - - m = ConstrainedExample(constrained_float=1.1) - print(repr(m)) - #> ConstrainedExample(constrained_float=1.1) - - try: - ConstrainedExample(constrained_float=0.9) - except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than', - 'loc': ('constrained_float',), - 'msg': 'Input should be greater than 1', - 'input': 0.9, - 'ctx': {'gt': 1.0}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than', - } - ] - ''' - ``` - """ # noqa: D212 - return Annotated[ # pyright: ignore[reportReturnType] - float, - Strict(strict) if strict is not None else None, - annotated_types.Interval(gt=gt, ge=ge, lt=lt, le=le), - annotated_types.MultipleOf(multiple_of) if multiple_of is not None else None, - AllowInfNan(allow_inf_nan) if allow_inf_nan is not None else None, - ] - - -PositiveFloat = Annotated[float, annotated_types.Gt(0)] -"""A float that must be greater than zero. - -```python -from pydantic import BaseModel, PositiveFloat, ValidationError - -class Model(BaseModel): - positive_float: PositiveFloat - -m = Model(positive_float=1.0) -print(repr(m)) -#> Model(positive_float=1.0) - -try: - Model(positive_float=-1.0) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than', - 'loc': ('positive_float',), - 'msg': 'Input should be greater than 0', - 'input': -1.0, - 'ctx': {'gt': 0.0}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than', - } - ] - ''' -``` -""" -NegativeFloat = Annotated[float, annotated_types.Lt(0)] -"""A float that must be less than zero. - -```python -from pydantic import BaseModel, NegativeFloat, ValidationError - -class Model(BaseModel): - negative_float: NegativeFloat - -m = Model(negative_float=-1.0) -print(repr(m)) -#> Model(negative_float=-1.0) - -try: - Model(negative_float=1.0) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'less_than', - 'loc': ('negative_float',), - 'msg': 'Input should be less than 0', - 'input': 1.0, - 'ctx': {'lt': 0.0}, - 'url': 'https://errors.pydantic.dev/2/v/less_than', - } - ] - ''' -``` -""" -NonPositiveFloat = Annotated[float, annotated_types.Le(0)] -"""A float that must be less than or equal to zero. - -```python -from pydantic import BaseModel, NonPositiveFloat, ValidationError - -class Model(BaseModel): - non_positive_float: NonPositiveFloat - -m = Model(non_positive_float=0.0) -print(repr(m)) -#> Model(non_positive_float=0.0) - -try: - Model(non_positive_float=1.0) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'less_than_equal', - 'loc': ('non_positive_float',), - 'msg': 'Input should be less than or equal to 0', - 'input': 1.0, - 'ctx': {'le': 0.0}, - 'url': 'https://errors.pydantic.dev/2/v/less_than_equal', - } - ] - ''' -``` -""" -NonNegativeFloat = Annotated[float, annotated_types.Ge(0)] -"""A float that must be greater than or equal to zero. - -```python -from pydantic import BaseModel, NonNegativeFloat, ValidationError - -class Model(BaseModel): - non_negative_float: NonNegativeFloat - -m = Model(non_negative_float=0.0) -print(repr(m)) -#> Model(non_negative_float=0.0) - -try: - Model(non_negative_float=-1.0) -except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than_equal', - 'loc': ('non_negative_float',), - 'msg': 'Input should be greater than or equal to 0', - 'input': -1.0, - 'ctx': {'ge': 0.0}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than_equal', - } - ] - ''' -``` -""" -StrictFloat = Annotated[float, Strict(True)] -"""A float that must be validated in strict mode. - -```python -from pydantic import BaseModel, StrictFloat, ValidationError - -class StrictFloatModel(BaseModel): - strict_float: StrictFloat - -try: - StrictFloatModel(strict_float='1.0') -except ValidationError as e: - print(e) - ''' - 1 validation error for StrictFloatModel - strict_float - Input should be a valid number [type=float_type, input_value='1.0', input_type=str] - ''' -``` -""" -FiniteFloat = Annotated[float, AllowInfNan(False)] -"""A float that must be finite (not ``-inf``, ``inf``, or ``nan``). - -```python -from pydantic import BaseModel, FiniteFloat - -class Model(BaseModel): - finite: FiniteFloat - -m = Model(finite=1.0) -print(m) -#> finite=1.0 -``` -""" - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BYTES TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -def conbytes( - *, - min_length: int | None = None, - max_length: int | None = None, - strict: bool | None = None, -) -> type[bytes]: - """A wrapper around `bytes` that allows for additional constraints. - - Args: - min_length: The minimum length of the bytes. - max_length: The maximum length of the bytes. - strict: Whether to validate the bytes in strict mode. - - Returns: - The wrapped bytes type. - """ - return Annotated[ # pyright: ignore[reportReturnType] - bytes, - Strict(strict) if strict is not None else None, - annotated_types.Len(min_length or 0, max_length), - ] - - -StrictBytes = Annotated[bytes, Strict()] -"""A bytes that must be validated in strict mode.""" - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ STRING TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -@_dataclasses.dataclass(frozen=True) -class StringConstraints(annotated_types.GroupedMetadata): - """!!! abstract "Usage Documentation" - [String types](./standard_library_types.md#strings) - - A field metadata class to apply constraints to `str` types. - Use this class as an annotation via [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated), as seen below. - - Attributes: - strip_whitespace: Whether to remove leading and trailing whitespace. - to_upper: Whether to convert the string to uppercase. - to_lower: Whether to convert the string to lowercase. - strict: Whether to validate the string in strict mode. - min_length: The minimum length of the string. - max_length: The maximum length of the string. - pattern: A regex pattern that the string must match. - ascii_only: Whether the string should contain only ASCII characters. - - Example: - ```python - from typing import Annotated - - from pydantic.types import StringConstraints - - ConstrainedStr = Annotated[str, StringConstraints(min_length=1, max_length=10)] - ``` - """ - - strip_whitespace: bool | None = None - to_upper: bool | None = None - to_lower: bool | None = None - strict: bool | None = None - min_length: int | None = None - max_length: int | None = None - pattern: str | Pattern[str] | None = None - ascii_only: bool | None = None - - def __iter__(self) -> Iterator[BaseMetadata]: - if self.min_length is not None: - yield MinLen(self.min_length) - if self.max_length is not None: - yield MaxLen(self.max_length) - if self.strict is not None: - yield Strict(self.strict) - if ( - self.strip_whitespace is not None - or self.pattern is not None - or self.to_lower is not None - or self.to_upper is not None - or self.ascii_only is not None - ): - yield _fields.pydantic_general_metadata( - strip_whitespace=self.strip_whitespace, - to_upper=self.to_upper, - to_lower=self.to_lower, - pattern=self.pattern, - ascii_only=self.ascii_only, - ) - - -def constr( - *, - strip_whitespace: bool | None = None, - to_upper: bool | None = None, - to_lower: bool | None = None, - strict: bool | None = None, - min_length: int | None = None, - max_length: int | None = None, - pattern: str | Pattern[str] | None = None, - ascii_only: bool | None = None, -) -> type[str]: - """ - !!! warning "Discouraged" - This function is **discouraged** in favor of using - [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) with - [`StringConstraints`][pydantic.types.StringConstraints] instead. - - This function will be **deprecated** in Pydantic 3.0. - - The reason is that `constr` returns a type, which doesn't play well with static analysis tools. - - === ":x: Don't do this" - ```python - from pydantic import BaseModel, constr - - class Foo(BaseModel): - bar: constr(strip_whitespace=True, to_upper=True, pattern=r'^[A-Z]+$') - ``` - - === ":white_check_mark: Do this" - ```python - from typing import Annotated - - from pydantic import BaseModel, StringConstraints - - class Foo(BaseModel): - bar: Annotated[ - str, - StringConstraints( - strip_whitespace=True, to_upper=True, pattern=r'^[A-Z]+$' - ), - ] - ``` - - A wrapper around `str` that allows for additional constraints. - - ```python - from pydantic import BaseModel, constr - - class Foo(BaseModel): - bar: constr(strip_whitespace=True, to_upper=True) - - foo = Foo(bar=' hello ') - print(foo) - #> bar='HELLO' - ``` - - Args: - strip_whitespace: Whether to remove leading and trailing whitespace. - to_upper: Whether to turn all characters to uppercase. - to_lower: Whether to turn all characters to lowercase. - strict: Whether to validate the string in strict mode. - min_length: The minimum length of the string. - max_length: The maximum length of the string. - pattern: A regex pattern to validate the string against. - ascii_only: Whether the string should contain only ASCII characters. - - Returns: - The wrapped string type. - """ # noqa: D212 - return Annotated[ # pyright: ignore[reportReturnType] - str, - StringConstraints( - strip_whitespace=strip_whitespace, - to_upper=to_upper, - to_lower=to_lower, - strict=strict, - min_length=min_length, - max_length=max_length, - pattern=pattern, - ascii_only=ascii_only, - ), - ] - - -StrictStr = Annotated[str, Strict()] -"""A string that must be validated in strict mode.""" - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~ COLLECTION TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -HashableItemType = TypeVar('HashableItemType', bound=Hashable) - - -def conset( - item_type: type[HashableItemType], *, min_length: int | None = None, max_length: int | None = None -) -> type[set[HashableItemType]]: - """A wrapper around `typing.Set` that allows for additional constraints. - - Args: - item_type: The type of the items in the set. - min_length: The minimum length of the set. - max_length: The maximum length of the set. - - Returns: - The wrapped set type. - """ - return Annotated[set[item_type], annotated_types.Len(min_length or 0, max_length)] # pyright: ignore[reportReturnType] - - -def confrozenset( - item_type: type[HashableItemType], *, min_length: int | None = None, max_length: int | None = None -) -> type[frozenset[HashableItemType]]: - """A wrapper around `typing.FrozenSet` that allows for additional constraints. - - Args: - item_type: The type of the items in the frozenset. - min_length: The minimum length of the frozenset. - max_length: The maximum length of the frozenset. - - Returns: - The wrapped frozenset type. - """ - return Annotated[frozenset[item_type], annotated_types.Len(min_length or 0, max_length)] # pyright: ignore[reportReturnType] - - -AnyItemType = TypeVar('AnyItemType') - - -def conlist( - item_type: type[AnyItemType], - *, - min_length: int | None = None, - max_length: int | None = None, - unique_items: bool | None = None, -) -> type[list[AnyItemType]]: - """A wrapper around [`list`][] that adds validation. - - Args: - item_type: The type of the items in the list. - min_length: The minimum length of the list. Defaults to None. - max_length: The maximum length of the list. Defaults to None. - unique_items: Whether the items in the list must be unique. Defaults to None. - !!! warning Deprecated - The `unique_items` parameter is deprecated, use `Set` instead. - See [this issue](https://github.com/pydantic/pydantic-core/issues/296) for more details. - - Returns: - The wrapped list type. - """ - if unique_items is not None: - raise PydanticUserError( - ( - '`unique_items` is removed, use `Set` instead' - '(this feature is discussed in https://github.com/pydantic/pydantic-core/issues/296)' - ), - code='removed-kwargs', - ) - return Annotated[list[item_type], annotated_types.Len(min_length or 0, max_length)] # pyright: ignore[reportReturnType] - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~ IMPORT STRING TYPE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -AnyType = TypeVar('AnyType') -if TYPE_CHECKING: - ImportString = Annotated[AnyType, ...] -else: - - class ImportString: - """A type that can be used to import a Python object from a string. - - `ImportString` expects a string and loads the Python object importable at that dotted path. - Attributes of modules may be separated from the module by `:` or `.`, e.g. if `'math:cos'` is provided, - the resulting field value would be the function `cos`. If a `.` is used and both an attribute and submodule - are present at the same path, the module will be preferred. - - On model instantiation, pointers will be evaluated and imported. There is - some nuance to this behavior, demonstrated in the examples below. - - ```python - import math - - from pydantic import BaseModel, Field, ImportString, ValidationError - - class ImportThings(BaseModel): - obj: ImportString - - # A string value will cause an automatic import - my_cos = ImportThings(obj='math.cos') - - # You can use the imported function as you would expect - cos_of_0 = my_cos.obj(0) - assert cos_of_0 == 1 - - # A string whose value cannot be imported will raise an error - try: - ImportThings(obj='foo.bar') - except ValidationError as e: - print(e) - ''' - 1 validation error for ImportThings - obj - Invalid python path: No module named 'foo' [type=import_error, input_value='foo.bar', input_type=str] - ''' - - # Actual python objects can be assigned as well - my_cos = ImportThings(obj=math.cos) - my_cos_2 = ImportThings(obj='math.cos') - my_cos_3 = ImportThings(obj='math:cos') - assert my_cos == my_cos_2 == my_cos_3 - - # You can set default field value either as Python object: - class ImportThingsDefaultPyObj(BaseModel): - obj: ImportString = math.cos - - # or as a string value (but only if used with `validate_default=True`) - class ImportThingsDefaultString(BaseModel): - obj: ImportString = Field(default='math.cos', validate_default=True) - - my_cos_default1 = ImportThingsDefaultPyObj() - my_cos_default2 = ImportThingsDefaultString() - assert my_cos_default1.obj == my_cos_default2.obj == math.cos - - # note: this will not work! - class ImportThingsMissingValidateDefault(BaseModel): - obj: ImportString = 'math.cos' - - my_cos_default3 = ImportThingsMissingValidateDefault() - assert my_cos_default3.obj == 'math.cos' # just string, not evaluated - ``` - - Serializing an `ImportString` type to json is also possible. - - ```python - from pydantic import BaseModel, ImportString - - class ImportThings(BaseModel): - obj: ImportString - - # Create an instance - m = ImportThings(obj='math.cos') - print(m) - #> obj= - print(m.model_dump_json()) - #> {"obj":"math.cos"} - ``` - """ - - @classmethod - def __class_getitem__(cls, item: AnyType) -> AnyType: - return Annotated[item, cls()] - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - serializer = core_schema.plain_serializer_function_ser_schema(cls._serialize, when_used='json') - if cls is source: - # Treat bare usage of ImportString (`schema is None`) as the same as ImportString[Any] - return core_schema.no_info_plain_validator_function( - function=_validators.import_string, serialization=serializer - ) - else: - return core_schema.no_info_before_validator_function( - function=_validators.import_string, schema=handler(source), serialization=serializer - ) - - @classmethod - def __get_pydantic_json_schema__(cls, cs: CoreSchema, handler: GetJsonSchemaHandler) -> JsonSchemaValue: - return handler(core_schema.str_schema()) - - @staticmethod - def _serialize(v: Any) -> str: - if isinstance(v, ModuleType): - return v.__name__ - elif hasattr(v, '__module__') and hasattr(v, '__name__'): - return f'{v.__module__}.{v.__name__}' - # Handle special cases for sys.XXX streams - # if we see more of these, we should consider a more general solution - elif hasattr(v, 'name'): - if v.name == '': - return 'sys.stdout' - elif v.name == '': - return 'sys.stdin' - elif v.name == '': - return 'sys.stderr' - return v - - def __repr__(self) -> str: - return 'ImportString' - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DECIMAL TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -def condecimal( - *, - strict: bool | None = None, - gt: int | Decimal | None = None, - ge: int | Decimal | None = None, - lt: int | Decimal | None = None, - le: int | Decimal | None = None, - multiple_of: int | Decimal | None = None, - max_digits: int | None = None, - decimal_places: int | None = None, - allow_inf_nan: bool | None = None, -) -> type[Decimal]: - """ - !!! warning "Discouraged" - This function is **discouraged** in favor of using - [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated) with - [`Field`][pydantic.fields.Field] instead. - - This function will be **deprecated** in Pydantic 3.0. - - The reason is that `condecimal` returns a type, which doesn't play well with static analysis tools. - - === ":x: Don't do this" - ```python - from pydantic import BaseModel, condecimal - - class Foo(BaseModel): - bar: condecimal(strict=True, allow_inf_nan=True) - ``` - - === ":white_check_mark: Do this" - ```python - from decimal import Decimal - from typing import Annotated - - from pydantic import BaseModel, Field - - class Foo(BaseModel): - bar: Annotated[Decimal, Field(strict=True, allow_inf_nan=True)] - ``` - - A wrapper around Decimal that adds validation. - - Args: - strict: Whether to validate the value in strict mode. Defaults to `None`. - gt: The value must be greater than this. Defaults to `None`. - ge: The value must be greater than or equal to this. Defaults to `None`. - lt: The value must be less than this. Defaults to `None`. - le: The value must be less than or equal to this. Defaults to `None`. - multiple_of: The value must be a multiple of this. Defaults to `None`. - max_digits: The maximum number of digits. Defaults to `None`. - decimal_places: The number of decimal places. Defaults to `None`. - allow_inf_nan: Whether to allow infinity and NaN. Defaults to `None`. - - ```python - from decimal import Decimal - - from pydantic import BaseModel, ValidationError, condecimal - - class ConstrainedExample(BaseModel): - constrained_decimal: condecimal(gt=Decimal('1.0')) - - m = ConstrainedExample(constrained_decimal=Decimal('1.1')) - print(repr(m)) - #> ConstrainedExample(constrained_decimal=Decimal('1.1')) - - try: - ConstrainedExample(constrained_decimal=Decimal('0.9')) - except ValidationError as e: - print(e.errors()) - ''' - [ - { - 'type': 'greater_than', - 'loc': ('constrained_decimal',), - 'msg': 'Input should be greater than 1.0', - 'input': Decimal('0.9'), - 'ctx': {'gt': Decimal('1.0')}, - 'url': 'https://errors.pydantic.dev/2/v/greater_than', - } - ] - ''' - ``` - """ # noqa: D212 - return Annotated[ # pyright: ignore[reportReturnType] - Decimal, - Strict(strict) if strict is not None else None, - annotated_types.Interval(gt=gt, ge=ge, lt=lt, le=le), - annotated_types.MultipleOf(multiple_of) if multiple_of is not None else None, - _fields.pydantic_general_metadata(max_digits=max_digits, decimal_places=decimal_places), - AllowInfNan(allow_inf_nan) if allow_inf_nan is not None else None, - ] - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ UUID TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -@_dataclasses.dataclass(**_internal_dataclass.slots_true) -class UuidVersion: - """A field metadata class to indicate a [UUID](https://docs.python.org/3/library/uuid.html) version. - - Use this class as an annotation via [`Annotated`](https://docs.python.org/3/library/typing.html#typing.Annotated), as seen below. - - Attributes: - uuid_version: The version of the UUID. Must be one of 1, 3, 4, 5, 6, 7 or 8. - - Example: - ```python - from typing import Annotated - from uuid import UUID - - from pydantic.types import UuidVersion - - UUID1 = Annotated[UUID, UuidVersion(1)] - ``` - """ - - uuid_version: Literal[1, 3, 4, 5, 6, 7, 8] - - def __get_pydantic_json_schema__( - self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = handler(core_schema) - field_schema.pop('anyOf', None) # remove the bytes/str union - field_schema.update(type='string', format=f'uuid{self.uuid_version}') - return field_schema - - def __get_pydantic_core_schema__(self, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source) - _check_annotated_type(schema['type'], 'uuid', self.__class__.__name__) - schema['version'] = self.uuid_version # type: ignore - return schema - - def __hash__(self) -> int: - return hash(self.uuid_version) - - -UUID1 = Annotated[UUID, UuidVersion(1)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 1. - -```python -import uuid - -from pydantic import UUID1, BaseModel - -class Model(BaseModel): - uuid1: UUID1 - -Model(uuid1=uuid.uuid1()) -``` -""" -UUID3 = Annotated[UUID, UuidVersion(3)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 3. - -```python -import uuid - -from pydantic import UUID3, BaseModel - -class Model(BaseModel): - uuid3: UUID3 - -Model(uuid3=uuid.uuid3(uuid.NAMESPACE_DNS, 'pydantic.org')) -``` -""" -UUID4 = Annotated[UUID, UuidVersion(4)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 4. - -```python -import uuid - -from pydantic import UUID4, BaseModel - -class Model(BaseModel): - uuid4: UUID4 - -Model(uuid4=uuid.uuid4()) -``` -""" -UUID5 = Annotated[UUID, UuidVersion(5)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 5. - -```python -import uuid - -from pydantic import UUID5, BaseModel - -class Model(BaseModel): - uuid5: UUID5 - -Model(uuid5=uuid.uuid5(uuid.NAMESPACE_DNS, 'pydantic.org')) -``` -""" -UUID6 = Annotated[UUID, UuidVersion(6)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 6. - -```python -import uuid - -from pydantic import UUID6, BaseModel - -class Model(BaseModel): - uuid6: UUID6 - -Model(uuid6=uuid.UUID('1efea953-c2d6-6790-aa0a-69db8c87df97')) -``` -""" -UUID7 = Annotated[UUID, UuidVersion(7)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 7. - -```python -import uuid - -from pydantic import UUID7, BaseModel - -class Model(BaseModel): - uuid7: UUID7 - -Model(uuid7=uuid.UUID('0194fdcb-1c47-7a09-b52c-561154de0b4a')) -``` -""" -UUID8 = Annotated[UUID, UuidVersion(8)] -"""A [UUID](https://docs.python.org/3/library/uuid.html) that must be version 8. - -```python -import uuid - -from pydantic import UUID8, BaseModel - -class Model(BaseModel): - uuid8: UUID8 - -Model(uuid8=uuid.UUID('81a0b92e-6078-8551-9c81-8ccb666bdab8')) -``` -""" - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PATH TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -@_dataclasses.dataclass -class PathType: - path_type: Literal['file', 'dir', 'new', 'socket'] - - def __get_pydantic_json_schema__( - self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = handler(core_schema) - format_conversion = {'file': 'file-path', 'dir': 'directory-path'} - field_schema.update(format=format_conversion.get(self.path_type, 'path'), type='string') - return field_schema - - def __get_pydantic_core_schema__(self, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - function_lookup = { - 'file': cast(core_schema.WithInfoValidatorFunction, self.validate_file), - 'dir': cast(core_schema.WithInfoValidatorFunction, self.validate_directory), - 'new': cast(core_schema.WithInfoValidatorFunction, self.validate_new), - 'socket': cast(core_schema.WithInfoValidatorFunction, self.validate_socket), - } - - return core_schema.with_info_after_validator_function( - function_lookup[self.path_type], - handler(source), - ) - - @staticmethod - def validate_file(path: Path, _: core_schema.ValidationInfo) -> Path: - if path.is_file(): - return path - else: - raise PydanticCustomError('path_not_file', 'Path does not point to a file') - - @staticmethod - def validate_socket(path: Path, _: core_schema.ValidationInfo) -> Path: - if path.is_socket(): - return path - else: - raise PydanticCustomError('path_not_socket', 'Path does not point to a socket') - - @staticmethod - def validate_directory(path: Path, _: core_schema.ValidationInfo) -> Path: - if path.is_dir(): - return path - else: - raise PydanticCustomError('path_not_directory', 'Path does not point to a directory') - - @staticmethod - def validate_new(path: Path, _: core_schema.ValidationInfo) -> Path: - if path.exists(): - raise PydanticCustomError('path_exists', 'Path already exists') - elif not path.parent.exists(): - raise PydanticCustomError('parent_does_not_exist', 'Parent directory does not exist') - else: - return path - - def __hash__(self) -> int: - return hash(self.path_type) - - -FilePath = Annotated[Path, PathType('file')] -"""A path that must point to a file. - -```python -from pathlib import Path - -from pydantic import BaseModel, FilePath, ValidationError - -class Model(BaseModel): - f: FilePath - -path = Path('text.txt') -path.touch() -m = Model(f='text.txt') -print(m.model_dump()) -#> {'f': PosixPath('text.txt')} -path.unlink() - -path = Path('directory') -path.mkdir(exist_ok=True) -try: - Model(f='directory') # directory -except ValidationError as e: - print(e) - ''' - 1 validation error for Model - f - Path does not point to a file [type=path_not_file, input_value='directory', input_type=str] - ''' -path.rmdir() - -try: - Model(f='not-exists-file') -except ValidationError as e: - print(e) - ''' - 1 validation error for Model - f - Path does not point to a file [type=path_not_file, input_value='not-exists-file', input_type=str] - ''' -``` -""" -DirectoryPath = Annotated[Path, PathType('dir')] -"""A path that must point to a directory. - -```python -from pathlib import Path - -from pydantic import BaseModel, DirectoryPath, ValidationError - -class Model(BaseModel): - f: DirectoryPath - -path = Path('directory/') -path.mkdir() -m = Model(f='directory/') -print(m.model_dump()) -#> {'f': PosixPath('directory')} -path.rmdir() - -path = Path('file.txt') -path.touch() -try: - Model(f='file.txt') # file -except ValidationError as e: - print(e) - ''' - 1 validation error for Model - f - Path does not point to a directory [type=path_not_directory, input_value='file.txt', input_type=str] - ''' -path.unlink() - -try: - Model(f='not-exists-directory') -except ValidationError as e: - print(e) - ''' - 1 validation error for Model - f - Path does not point to a directory [type=path_not_directory, input_value='not-exists-directory', input_type=str] - ''' -``` -""" -NewPath = Annotated[Path, PathType('new')] -"""A path for a new file or directory that must not already exist. The parent directory must already exist.""" - -SocketPath = Annotated[Path, PathType('socket')] -"""A path to an existing socket file""" - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ JSON TYPE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -if TYPE_CHECKING: - # Json[list[str]] will be recognized by type checkers as list[str] - Json = Annotated[AnyType, ...] - -else: - - class Json: - """A special type wrapper which loads JSON before parsing. - - You can use the `Json` data type to make Pydantic first load a raw JSON string before - validating the loaded data into the parametrized type: - - ```python - from typing import Any - - from pydantic import BaseModel, Json, ValidationError - - class AnyJsonModel(BaseModel): - json_obj: Json[Any] - - class ConstrainedJsonModel(BaseModel): - json_obj: Json[list[int]] - - print(AnyJsonModel(json_obj='{"b": 1}')) - #> json_obj={'b': 1} - print(ConstrainedJsonModel(json_obj='[1, 2, 3]')) - #> json_obj=[1, 2, 3] - - try: - ConstrainedJsonModel(json_obj=12) - except ValidationError as e: - print(e) - ''' - 1 validation error for ConstrainedJsonModel - json_obj - JSON input should be string, bytes or bytearray [type=json_type, input_value=12, input_type=int] - ''' - - try: - ConstrainedJsonModel(json_obj='[a, b]') - except ValidationError as e: - print(e) - ''' - 1 validation error for ConstrainedJsonModel - json_obj - Invalid JSON: expected value at line 1 column 2 [type=json_invalid, input_value='[a, b]', input_type=str] - ''' - - try: - ConstrainedJsonModel(json_obj='["a", "b"]') - except ValidationError as e: - print(e) - ''' - 2 validation errors for ConstrainedJsonModel - json_obj.0 - Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='a', input_type=str] - json_obj.1 - Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='b', input_type=str] - ''' - ``` - - When you dump the model using `model_dump` or `model_dump_json`, the dumped value will be the result of validation, - not the original JSON string. However, you can use the argument `round_trip=True` to get the original JSON string back: - - ```python - from pydantic import BaseModel, Json - - class ConstrainedJsonModel(BaseModel): - json_obj: Json[list[int]] - - print(ConstrainedJsonModel(json_obj='[1, 2, 3]').model_dump_json()) - #> {"json_obj":[1,2,3]} - print( - ConstrainedJsonModel(json_obj='[1, 2, 3]').model_dump_json(round_trip=True) - ) - #> {"json_obj":"[1,2,3]"} - ``` - """ - - @classmethod - def __class_getitem__(cls, item: AnyType) -> AnyType: - return Annotated[item, cls()] - - @classmethod - def __get_pydantic_core_schema__(cls, source: Any, handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - if cls is source: - return core_schema.json_schema(None) - else: - return core_schema.json_schema(handler(source)) - - def __repr__(self) -> str: - return 'Json' - - def __hash__(self) -> int: - return hash(type(self)) - - def __eq__(self, other: Any) -> bool: - return type(other) is type(self) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SECRET TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -# The `Secret` class being conceptually immutable, make the type variable covariant: -SecretType = TypeVar('SecretType', covariant=True) - - -class _SecretBase(Generic[SecretType]): - def __init__(self, secret_value: SecretType) -> None: - self._secret_value: SecretType = secret_value - - def get_secret_value(self) -> SecretType: - """Get the secret value. - - Returns: - The secret value. - """ - return self._secret_value - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and self.get_secret_value() == other.get_secret_value() - - def __hash__(self) -> int: - return hash(self.get_secret_value()) - - def __str__(self) -> str: - return str(self._display()) - - def __repr__(self) -> str: - return f'{self.__class__.__name__}({self._display()!r})' - - def _display(self) -> str | bytes: - raise NotImplementedError - - -def _serialize_secret(value: Secret[SecretType], info: core_schema.SerializationInfo) -> str | Secret[SecretType]: - if info.mode == 'json': - return str(value) - else: - return value - - -class Secret(_SecretBase[SecretType]): - """A generic base class used for defining a field with sensitive information that you do not want to be visible in logging or tracebacks. - - You may either directly parametrize `Secret` with a type, or subclass from `Secret` with a parametrized type. The benefit of subclassing - is that you can define a custom `_display` method, which will be used for `repr()` and `str()` methods. The examples below demonstrate both - ways of using `Secret` to create a new secret type. - - 1. Directly parametrizing `Secret` with a type: - - ```python - from pydantic import BaseModel, Secret - - SecretBool = Secret[bool] - - class Model(BaseModel): - secret_bool: SecretBool - - m = Model(secret_bool=True) - print(m.model_dump()) - #> {'secret_bool': Secret('**********')} - - print(m.model_dump_json()) - #> {"secret_bool":"**********"} - - print(m.secret_bool.get_secret_value()) - #> True - ``` - - 2. Subclassing from parametrized `Secret`: - - ```python - from datetime import date - - from pydantic import BaseModel, Secret - - class SecretDate(Secret[date]): - def _display(self) -> str: - return '****/**/**' - - class Model(BaseModel): - secret_date: SecretDate - - m = Model(secret_date=date(2022, 1, 1)) - print(m.model_dump()) - #> {'secret_date': SecretDate('****/**/**')} - - print(m.model_dump_json()) - #> {"secret_date":"****/**/**"} - - print(m.secret_date.get_secret_value()) - #> 2022-01-01 - ``` - - The value returned by the `_display` method will be used for `repr()` and `str()`. - - You can enforce constraints on the underlying type through annotations: - For example: - - ```python - from typing import Annotated - - from pydantic import BaseModel, Field, Secret, ValidationError - - SecretPosInt = Secret[Annotated[int, Field(gt=0, strict=True)]] - - class Model(BaseModel): - sensitive_int: SecretPosInt - - m = Model(sensitive_int=42) - print(m.model_dump()) - #> {'sensitive_int': Secret('**********')} - - try: - m = Model(sensitive_int=-42) # (1)! - except ValidationError as exc_info: - print(exc_info.errors(include_url=False, include_input=False)) - ''' - [ - { - 'type': 'greater_than', - 'loc': ('sensitive_int',), - 'msg': 'Input should be greater than 0', - 'ctx': {'gt': 0}, - } - ] - ''' - - try: - m = Model(sensitive_int='42') # (2)! - except ValidationError as exc_info: - print(exc_info.errors(include_url=False, include_input=False)) - ''' - [ - { - 'type': 'int_type', - 'loc': ('sensitive_int',), - 'msg': 'Input should be a valid integer', - } - ] - ''' - ``` - - 1. The input value is not greater than 0, so it raises a validation error. - 2. The input value is not an integer, so it raises a validation error because the `SecretPosInt` type has strict mode enabled. - """ - - def _display(self) -> str | bytes: - return '**********' if self.get_secret_value() else '' - - @classmethod - def __get_pydantic_core_schema__(cls, source: type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - inner_type = None - # if origin_type is Secret, then cls is a GenericAlias, and we can extract the inner type directly - origin_type = get_origin(source) - if origin_type is not None: - inner_type = get_args(source)[0] - # otherwise, we need to get the inner type from the base class - else: - bases = getattr(cls, '__orig_bases__', getattr(cls, '__bases__', [])) - for base in bases: - if get_origin(base) is Secret: - inner_type = get_args(base)[0] - if bases == [] or inner_type is None: - raise TypeError( - f"Can't get secret type from {cls.__name__}. " - 'Please use Secret[], or subclass from Secret[] instead.' - ) - - inner_schema = handler.generate_schema(inner_type) # type: ignore - - def validate_secret_value(value, handler) -> Secret[SecretType]: - if isinstance(value, Secret): - value = value.get_secret_value() - validated_inner = handler(value) - return cls(validated_inner) - - return core_schema.json_or_python_schema( - python_schema=core_schema.no_info_wrap_validator_function( - validate_secret_value, - inner_schema, - ), - json_schema=core_schema.no_info_after_validator_function(lambda x: cls(x), inner_schema), - serialization=core_schema.plain_serializer_function_ser_schema( - _serialize_secret, - info_arg=True, - when_used='always', - ), - ) - - __pydantic_serializer__ = SchemaSerializer( - core_schema.any_schema( - serialization=core_schema.plain_serializer_function_ser_schema( - _serialize_secret, - info_arg=True, - when_used='always', - ) - ) - ) - - -def _secret_display(value: SecretType) -> str: # type: ignore - return '**********' if value else '' - - -def _serialize_secret_field( - value: _SecretField[SecretType], info: core_schema.SerializationInfo -) -> str | _SecretField[SecretType]: - if info.mode == 'json': - # we want the output to always be string without the `b'` prefix for bytes, - # hence we just use `secret_display` - return _secret_display(value.get_secret_value()) - else: - return value - - -class _SecretField(_SecretBase[SecretType]): - _inner_schema: ClassVar[CoreSchema] - _error_kind: ClassVar[str] - - @classmethod - def __get_pydantic_core_schema__(cls, source: type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - def get_json_schema(_core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler) -> JsonSchemaValue: - json_schema = handler(cls._inner_schema) - _utils.update_not_none( - json_schema, - type='string', - writeOnly=True, - format='password', - ) - return json_schema - - def get_secret_schema(strict: bool) -> CoreSchema: - inner_schema = {**cls._inner_schema, 'strict': strict} - json_schema = core_schema.no_info_after_validator_function( - source, # construct the type - inner_schema, # pyright: ignore[reportArgumentType] - ) - return core_schema.json_or_python_schema( - python_schema=core_schema.union_schema( - [ - core_schema.is_instance_schema(source), - json_schema, - ], - custom_error_type=cls._error_kind, - ), - json_schema=json_schema, - serialization=core_schema.plain_serializer_function_ser_schema( - _serialize_secret_field, - info_arg=True, - when_used='always', - ), - ) - - return core_schema.lax_or_strict_schema( - lax_schema=get_secret_schema(strict=False), - strict_schema=get_secret_schema(strict=True), - metadata={'pydantic_js_functions': [get_json_schema]}, - ) - - __pydantic_serializer__ = SchemaSerializer( - core_schema.any_schema( - serialization=core_schema.plain_serializer_function_ser_schema( - _serialize_secret_field, - info_arg=True, - when_used='always', - ) - ) - ) - - -class SecretStr(_SecretField[str]): - """A string used for storing sensitive information that you do not want to be visible in logging or tracebacks. - - When the secret value is nonempty, it is displayed as `'**********'` instead of the underlying value in - calls to `repr()` and `str()`. If the value _is_ empty, it is displayed as `''`. - - ```python - from pydantic import BaseModel, SecretStr - - class User(BaseModel): - username: str - password: SecretStr - - user = User(username='scolvin', password='password1') - - print(user) - #> username='scolvin' password=SecretStr('**********') - print(user.password.get_secret_value()) - #> password1 - print((SecretStr('password'), SecretStr(''))) - #> (SecretStr('**********'), SecretStr('')) - ``` - - As seen above, by default, [`SecretStr`][pydantic.types.SecretStr] (and [`SecretBytes`][pydantic.types.SecretBytes]) - will be serialized as `**********` when serializing to json. - - You can use the [`field_serializer`][pydantic.functional_serializers.field_serializer] to dump the - secret as plain-text when serializing to json. - - ```python - from pydantic import BaseModel, SecretBytes, SecretStr, field_serializer - - class Model(BaseModel): - password: SecretStr - password_bytes: SecretBytes - - @field_serializer('password', 'password_bytes', when_used='json') - def dump_secret(self, v): - return v.get_secret_value() - - model = Model(password='IAmSensitive', password_bytes=b'IAmSensitiveBytes') - print(model) - #> password=SecretStr('**********') password_bytes=SecretBytes(b'**********') - print(model.password) - #> ********** - print(model.model_dump()) - ''' - { - 'password': SecretStr('**********'), - 'password_bytes': SecretBytes(b'**********'), - } - ''' - print(model.model_dump_json()) - #> {"password":"IAmSensitive","password_bytes":"IAmSensitiveBytes"} - ``` - """ - - _inner_schema: ClassVar[CoreSchema] = core_schema.str_schema() - _error_kind: ClassVar[str] = 'string_type' - - def __len__(self) -> int: - return len(self._secret_value) - - def _display(self) -> str: - return _secret_display(self._secret_value) - - -class SecretBytes(_SecretField[bytes]): - """A bytes used for storing sensitive information that you do not want to be visible in logging or tracebacks. - - It displays `b'**********'` instead of the string value on `repr()` and `str()` calls. - When the secret value is nonempty, it is displayed as `b'**********'` instead of the underlying value in - calls to `repr()` and `str()`. If the value _is_ empty, it is displayed as `b''`. - - ```python - from pydantic import BaseModel, SecretBytes - - class User(BaseModel): - username: str - password: SecretBytes - - user = User(username='scolvin', password=b'password1') - #> username='scolvin' password=SecretBytes(b'**********') - print(user.password.get_secret_value()) - #> b'password1' - print((SecretBytes(b'password'), SecretBytes(b''))) - #> (SecretBytes(b'**********'), SecretBytes(b'')) - ``` - """ - - _inner_schema: ClassVar[CoreSchema] = core_schema.bytes_schema() - _error_kind: ClassVar[str] = 'bytes_type' - - def __len__(self) -> int: - return len(self._secret_value) - - def _display(self) -> bytes: - return _secret_display(self._secret_value).encode() - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PAYMENT CARD TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class PaymentCardBrand(str, Enum): - amex = 'American Express' - mastercard = 'Mastercard' - visa = 'Visa' - other = 'other' - - def __str__(self) -> str: - return self.value - - -@deprecated( - 'The `PaymentCardNumber` class is deprecated, use `pydantic_extra_types` instead. ' - 'See https://docs.pydantic.dev/latest/api/pydantic_extra_types_payment/#pydantic_extra_types.payment.PaymentCardNumber.', - category=PydanticDeprecatedSince20, -) -class PaymentCardNumber(str): - """Based on: https://en.wikipedia.org/wiki/Payment_card_number.""" - - strip_whitespace: ClassVar[bool] = True - min_length: ClassVar[int] = 12 - max_length: ClassVar[int] = 19 - bin: str - last4: str - brand: PaymentCardBrand - - def __init__(self, card_number: str): - self.validate_digits(card_number) - - card_number = self.validate_luhn_check_digit(card_number) - - self.bin = card_number[:6] - self.last4 = card_number[-4:] - self.brand = self.validate_brand(card_number) - - @classmethod - def __get_pydantic_core_schema__(cls, source: type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - return core_schema.with_info_after_validator_function( - cls.validate, - core_schema.str_schema( - min_length=cls.min_length, max_length=cls.max_length, strip_whitespace=cls.strip_whitespace - ), - ) - - @classmethod - def validate(cls, input_value: str, /, _: core_schema.ValidationInfo) -> PaymentCardNumber: - """Validate the card number and return a `PaymentCardNumber` instance.""" - return cls(input_value) - - @property - def masked(self) -> str: - """Mask all but the last 4 digits of the card number. - - Returns: - A masked card number string. - """ - num_masked = len(self) - 10 # len(bin) + len(last4) == 10 - return f'{self.bin}{"*" * num_masked}{self.last4}' - - @classmethod - def validate_digits(cls, card_number: str) -> None: - """Validate that the card number is all digits.""" - if not card_number.isdigit(): - raise PydanticCustomError('payment_card_number_digits', 'Card number is not all digits') - - @classmethod - def validate_luhn_check_digit(cls, card_number: str) -> str: - """Based on: https://en.wikipedia.org/wiki/Luhn_algorithm.""" - sum_ = int(card_number[-1]) - length = len(card_number) - parity = length % 2 - for i in range(length - 1): - digit = int(card_number[i]) - if i % 2 == parity: - digit *= 2 - if digit > 9: - digit -= 9 - sum_ += digit - valid = sum_ % 10 == 0 - if not valid: - raise PydanticCustomError('payment_card_number_luhn', 'Card number is not luhn valid') - return card_number - - @staticmethod - def validate_brand(card_number: str) -> PaymentCardBrand: - """Validate length based on BIN for major brands: - https://en.wikipedia.org/wiki/Payment_card_number#Issuer_identification_number_(IIN). - """ - if card_number[0] == '4': - brand = PaymentCardBrand.visa - elif 51 <= int(card_number[:2]) <= 55: - brand = PaymentCardBrand.mastercard - elif card_number[:2] in {'34', '37'}: - brand = PaymentCardBrand.amex - else: - brand = PaymentCardBrand.other - - required_length: None | int | str = None - if brand in PaymentCardBrand.mastercard: - required_length = 16 - valid = len(card_number) == required_length - elif brand == PaymentCardBrand.visa: - required_length = '13, 16 or 19' - valid = len(card_number) in {13, 16, 19} - elif brand == PaymentCardBrand.amex: - required_length = 15 - valid = len(card_number) == required_length - else: - valid = True - - if not valid: - raise PydanticCustomError( - 'payment_card_number_brand', - 'Length for a {brand} card must be {required_length}', - {'brand': brand, 'required_length': required_length}, - ) - return brand - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BYTE SIZE TYPE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class ByteSize(int): - """Converts a string representing a number of bytes with units (such as `'1KB'` or `'11.5MiB'`) into an integer. - - You can use the `ByteSize` data type to (case-insensitively) convert a string representation of a number of bytes into - an integer, and also to print out human-readable strings representing a number of bytes. - - In conformance with [IEC 80000-13 Standard](https://en.wikipedia.org/wiki/ISO/IEC_80000) we interpret `'1KB'` to mean 1000 bytes, - and `'1KiB'` to mean 1024 bytes. In general, including a middle `'i'` will cause the unit to be interpreted as a power of 2, - rather than a power of 10 (so, for example, `'1 MB'` is treated as `1_000_000` bytes, whereas `'1 MiB'` is treated as `1_048_576` bytes). - - !!! info - Note that `1b` will be parsed as "1 byte" and not "1 bit". - - ```python - from pydantic import BaseModel, ByteSize - - class MyModel(BaseModel): - size: ByteSize - - print(MyModel(size=52000).size) - #> 52000 - print(MyModel(size='3000 KiB').size) - #> 3072000 - - m = MyModel(size='50 PB') - print(m.size.human_readable()) - #> 44.4PiB - print(m.size.human_readable(decimal=True)) - #> 50.0PB - print(m.size.human_readable(separator=' ')) - #> 44.4 PiB - - print(m.size.to('TiB')) - #> 45474.73508864641 - ``` - """ - - byte_sizes = { - 'b': 1, - 'kb': 10**3, - 'mb': 10**6, - 'gb': 10**9, - 'tb': 10**12, - 'pb': 10**15, - 'eb': 10**18, - 'kib': 2**10, - 'mib': 2**20, - 'gib': 2**30, - 'tib': 2**40, - 'pib': 2**50, - 'eib': 2**60, - 'bit': 1 / 8, - 'kbit': 10**3 / 8, - 'mbit': 10**6 / 8, - 'gbit': 10**9 / 8, - 'tbit': 10**12 / 8, - 'pbit': 10**15 / 8, - 'ebit': 10**18 / 8, - 'kibit': 2**10 / 8, - 'mibit': 2**20 / 8, - 'gibit': 2**30 / 8, - 'tibit': 2**40 / 8, - 'pibit': 2**50 / 8, - 'eibit': 2**60 / 8, - } - byte_sizes.update({k.lower()[0]: v for k, v in byte_sizes.items() if 'i' not in k}) - - byte_string_pattern = r'^\s*(\d*\.?\d+)\s*(\w+)?' - byte_string_re = re.compile(byte_string_pattern, re.IGNORECASE) - - @classmethod - def __get_pydantic_core_schema__(cls, source: type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - return core_schema.with_info_after_validator_function( - function=cls._validate, - schema=core_schema.union_schema( - [ - core_schema.str_schema(pattern=cls.byte_string_pattern), - core_schema.int_schema(ge=0), - ], - custom_error_type='byte_size', - custom_error_message='could not parse value and unit from byte string', - ), - serialization=core_schema.plain_serializer_function_ser_schema( - int, return_schema=core_schema.int_schema(ge=0) - ), - ) - - @classmethod - def _validate(cls, input_value: Any, /, _: core_schema.ValidationInfo) -> ByteSize: - try: - return cls(int(input_value)) - except ValueError: - pass - - str_match = cls.byte_string_re.match(str(input_value)) - if str_match is None: - raise PydanticCustomError('byte_size', 'could not parse value and unit from byte string') - - scalar, unit = str_match.groups() - if unit is None: - unit = 'b' - - try: - unit_mult = cls.byte_sizes[unit.lower()] - except KeyError: - raise PydanticCustomError('byte_size_unit', 'could not interpret byte unit: {unit}', {'unit': unit}) - - return cls(int(float(scalar) * unit_mult)) - - def human_readable(self, decimal: bool = False, separator: str = '') -> str: - """Converts a byte size to a human readable string. - - Args: - decimal: If True, use decimal units (e.g. 1000 bytes per KB). If False, use binary units - (e.g. 1024 bytes per KiB). - separator: A string used to split the value and unit. Defaults to an empty string (''). - - Returns: - A human readable string representation of the byte size. - """ - if decimal: - divisor = 1000 - units = 'B', 'KB', 'MB', 'GB', 'TB', 'PB' - final_unit = 'EB' - else: - divisor = 1024 - units = 'B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB' - final_unit = 'EiB' - - num = float(self) - for unit in units: - if abs(num) < divisor: - if unit == 'B': - return f'{num:0.0f}{separator}{unit}' - else: - return f'{num:0.1f}{separator}{unit}' - num /= divisor - - return f'{num:0.1f}{separator}{final_unit}' - - def to(self, unit: str) -> float: - """Converts a byte size to another unit, including both byte and bit units. - - Args: - unit: The unit to convert to. Must be one of the following: B, KB, MB, GB, TB, PB, EB, - KiB, MiB, GiB, TiB, PiB, EiB (byte units) and - bit, kbit, mbit, gbit, tbit, pbit, ebit, - kibit, mibit, gibit, tibit, pibit, eibit (bit units). - - Returns: - The byte size in the new unit. - """ - try: - unit_div = self.byte_sizes[unit.lower()] - except KeyError: - raise PydanticCustomError('byte_size_unit', 'Could not interpret byte unit: {unit}', {'unit': unit}) - - return self / unit_div - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DATE TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -def _check_annotated_type(annotated_type: str, expected_type: str, annotation: str) -> None: - if annotated_type != expected_type: - raise PydanticUserError(f"'{annotation}' cannot annotate '{annotated_type}'.", code='invalid-annotated-type') - - -if TYPE_CHECKING: - PastDate = Annotated[date, ...] - FutureDate = Annotated[date, ...] -else: - - class PastDate: - """A date in the past.""" - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - if cls is source: - # used directly as a type - return core_schema.date_schema(now_op='past') - else: - schema = handler(source) - _check_annotated_type(schema['type'], 'date', cls.__name__) - schema['now_op'] = 'past' - return schema - - def __repr__(self) -> str: - return 'PastDate' - - class FutureDate: - """A date in the future.""" - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - if cls is source: - # used directly as a type - return core_schema.date_schema(now_op='future') - else: - schema = handler(source) - _check_annotated_type(schema['type'], 'date', cls.__name__) - schema['now_op'] = 'future' - return schema - - def __repr__(self) -> str: - return 'FutureDate' - - -def condate( - *, - strict: bool | None = None, - gt: date | None = None, - ge: date | None = None, - lt: date | None = None, - le: date | None = None, -) -> type[date]: - """A wrapper for date that adds constraints. - - Args: - strict: Whether to validate the date value in strict mode. Defaults to `None`. - gt: The value must be greater than this. Defaults to `None`. - ge: The value must be greater than or equal to this. Defaults to `None`. - lt: The value must be less than this. Defaults to `None`. - le: The value must be less than or equal to this. Defaults to `None`. - - Returns: - A date type with the specified constraints. - """ - return Annotated[ # pyright: ignore[reportReturnType] - date, - Strict(strict) if strict is not None else None, - annotated_types.Interval(gt=gt, ge=ge, lt=lt, le=le), - ] - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DATETIME TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -if TYPE_CHECKING: - AwareDatetime = Annotated[datetime, ...] - NaiveDatetime = Annotated[datetime, ...] - PastDatetime = Annotated[datetime, ...] - FutureDatetime = Annotated[datetime, ...] - -else: - - class AwareDatetime: - """A datetime that requires timezone info.""" - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - if cls is source: - # used directly as a type - return core_schema.datetime_schema(tz_constraint='aware') - else: - schema = handler(source) - _check_annotated_type(schema['type'], 'datetime', cls.__name__) - schema['tz_constraint'] = 'aware' - return schema - - def __repr__(self) -> str: - return 'AwareDatetime' - - class NaiveDatetime: - """A datetime that doesn't require timezone info.""" - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - if cls is source: - # used directly as a type - return core_schema.datetime_schema(tz_constraint='naive') - else: - schema = handler(source) - _check_annotated_type(schema['type'], 'datetime', cls.__name__) - schema['tz_constraint'] = 'naive' - return schema - - def __repr__(self) -> str: - return 'NaiveDatetime' - - class PastDatetime: - """A datetime that must be in the past.""" - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - if cls is source: - # used directly as a type - return core_schema.datetime_schema(now_op='past') - else: - schema = handler(source) - _check_annotated_type(schema['type'], 'datetime', cls.__name__) - schema['now_op'] = 'past' - return schema - - def __repr__(self) -> str: - return 'PastDatetime' - - class FutureDatetime: - """A datetime that must be in the future.""" - - @classmethod - def __get_pydantic_core_schema__( - cls, source: type[Any], handler: GetCoreSchemaHandler - ) -> core_schema.CoreSchema: - if cls is source: - # used directly as a type - return core_schema.datetime_schema(now_op='future') - else: - schema = handler(source) - _check_annotated_type(schema['type'], 'datetime', cls.__name__) - schema['now_op'] = 'future' - return schema - - def __repr__(self) -> str: - return 'FutureDatetime' - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ Encoded TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class EncoderProtocol(Protocol): - """Protocol for encoding and decoding data to and from bytes.""" - - @classmethod - def decode(cls, data: bytes) -> bytes: - """Decode the data using the encoder. - - Args: - data: The data to decode. - - Returns: - The decoded data. - """ - ... - - @classmethod - def encode(cls, value: bytes) -> bytes: - """Encode the data using the encoder. - - Args: - value: The data to encode. - - Returns: - The encoded data. - """ - ... - - @classmethod - def get_json_format(cls) -> str: - """Get the JSON format for the encoded data. - - Returns: - The JSON format for the encoded data. - """ - ... - - -class Base64Encoder(EncoderProtocol): - """Standard (non-URL-safe) Base64 encoder.""" - - @classmethod - def decode(cls, data: bytes) -> bytes: - """Decode the data from base64 encoded bytes to original bytes data. - - Args: - data: The data to decode. - - Returns: - The decoded data. - """ - try: - return base64.b64decode(data) - except ValueError as e: - raise PydanticCustomError('base64_decode', "Base64 decoding error: '{error}'", {'error': str(e)}) - - @classmethod - def encode(cls, value: bytes) -> bytes: - """Encode the data from bytes to a base64 encoded bytes. - - Args: - value: The data to encode. - - Returns: - The encoded data. - """ - return base64.b64encode(value) - - @classmethod - def get_json_format(cls) -> Literal['base64']: - """Get the JSON format for the encoded data. - - Returns: - The JSON format for the encoded data. - """ - return 'base64' - - -class Base64UrlEncoder(EncoderProtocol): - """URL-safe Base64 encoder.""" - - @classmethod - def decode(cls, data: bytes) -> bytes: - """Decode the data from base64 encoded bytes to original bytes data. - - Args: - data: The data to decode. - - Returns: - The decoded data. - """ - try: - return base64.urlsafe_b64decode(data) - except ValueError as e: - raise PydanticCustomError('base64_decode', "Base64 decoding error: '{error}'", {'error': str(e)}) - - @classmethod - def encode(cls, value: bytes) -> bytes: - """Encode the data from bytes to a base64 encoded bytes. - - Args: - value: The data to encode. - - Returns: - The encoded data. - """ - return base64.urlsafe_b64encode(value) - - @classmethod - def get_json_format(cls) -> Literal['base64url']: - """Get the JSON format for the encoded data. - - Returns: - The JSON format for the encoded data. - """ - return 'base64url' - - -@_dataclasses.dataclass(**_internal_dataclass.slots_true) -class EncodedBytes: - """A bytes type that is encoded and decoded using the specified encoder. - - `EncodedBytes` needs an encoder that implements `EncoderProtocol` to operate. - - ```python - from typing import Annotated - - from pydantic import BaseModel, EncodedBytes, EncoderProtocol, ValidationError - - class MyEncoder(EncoderProtocol): - @classmethod - def decode(cls, data: bytes) -> bytes: - if data == b'**undecodable**': - raise ValueError('Cannot decode data') - return data[13:] - - @classmethod - def encode(cls, value: bytes) -> bytes: - return b'**encoded**: ' + value - - @classmethod - def get_json_format(cls) -> str: - return 'my-encoder' - - MyEncodedBytes = Annotated[bytes, EncodedBytes(encoder=MyEncoder)] - - class Model(BaseModel): - my_encoded_bytes: MyEncodedBytes - - # Initialize the model with encoded data - m = Model(my_encoded_bytes=b'**encoded**: some bytes') - - # Access decoded value - print(m.my_encoded_bytes) - #> b'some bytes' - - # Serialize into the encoded form - print(m.model_dump()) - #> {'my_encoded_bytes': b'**encoded**: some bytes'} - - # Validate encoded data - try: - Model(my_encoded_bytes=b'**undecodable**') - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - my_encoded_bytes - Value error, Cannot decode data [type=value_error, input_value=b'**undecodable**', input_type=bytes] - ''' - ``` - """ - - encoder: type[EncoderProtocol] - - def __get_pydantic_json_schema__( - self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = handler(core_schema) - field_schema.update(type='string', format=self.encoder.get_json_format()) - return field_schema - - def __get_pydantic_core_schema__(self, source: type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source) - _check_annotated_type(schema['type'], 'bytes', self.__class__.__name__) - return core_schema.with_info_after_validator_function( - function=self.decode, - schema=schema, - serialization=core_schema.plain_serializer_function_ser_schema(function=self.encode), - ) - - def decode(self, data: bytes, _: core_schema.ValidationInfo) -> bytes: - """Decode the data using the specified encoder. - - Args: - data: The data to decode. - - Returns: - The decoded data. - """ - return self.encoder.decode(data) - - def encode(self, value: bytes) -> bytes: - """Encode the data using the specified encoder. - - Args: - value: The data to encode. - - Returns: - The encoded data. - """ - return self.encoder.encode(value) - - def __hash__(self) -> int: - return hash(self.encoder) - - -@_dataclasses.dataclass(**_internal_dataclass.slots_true) -class EncodedStr: - """A str type that is encoded and decoded using the specified encoder. - - `EncodedStr` needs an encoder that implements `EncoderProtocol` to operate. - - ```python - from typing import Annotated - - from pydantic import BaseModel, EncodedStr, EncoderProtocol, ValidationError - - class MyEncoder(EncoderProtocol): - @classmethod - def decode(cls, data: bytes) -> bytes: - if data == b'**undecodable**': - raise ValueError('Cannot decode data') - return data[13:] - - @classmethod - def encode(cls, value: bytes) -> bytes: - return b'**encoded**: ' + value - - @classmethod - def get_json_format(cls) -> str: - return 'my-encoder' - - MyEncodedStr = Annotated[str, EncodedStr(encoder=MyEncoder)] - - class Model(BaseModel): - my_encoded_str: MyEncodedStr - - # Initialize the model with encoded data - m = Model(my_encoded_str='**encoded**: some str') - - # Access decoded value - print(m.my_encoded_str) - #> some str - - # Serialize into the encoded form - print(m.model_dump()) - #> {'my_encoded_str': '**encoded**: some str'} - - # Validate encoded data - try: - Model(my_encoded_str='**undecodable**') - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - my_encoded_str - Value error, Cannot decode data [type=value_error, input_value='**undecodable**', input_type=str] - ''' - ``` - """ - - encoder: type[EncoderProtocol] - - def __get_pydantic_json_schema__( - self, core_schema: core_schema.CoreSchema, handler: GetJsonSchemaHandler - ) -> JsonSchemaValue: - field_schema = handler(core_schema) - field_schema.update(type='string', format=self.encoder.get_json_format()) - return field_schema - - def __get_pydantic_core_schema__(self, source: type[Any], handler: GetCoreSchemaHandler) -> core_schema.CoreSchema: - schema = handler(source) - _check_annotated_type(schema['type'], 'str', self.__class__.__name__) - return core_schema.with_info_after_validator_function( - function=self.decode_str, - schema=schema, - serialization=core_schema.plain_serializer_function_ser_schema(function=self.encode_str), - ) - - def decode_str(self, data: str, _: core_schema.ValidationInfo) -> str: - """Decode the data using the specified encoder. - - Args: - data: The data to decode. - - Returns: - The decoded data. - """ - return self.encoder.decode(data.encode()).decode() - - def encode_str(self, value: str) -> str: - """Encode the data using the specified encoder. - - Args: - value: The data to encode. - - Returns: - The encoded data. - """ - return self.encoder.encode(value.encode()).decode() # noqa: UP008 - - def __hash__(self) -> int: - return hash(self.encoder) - - -Base64Bytes = Annotated[bytes, EncodedBytes(encoder=Base64Encoder)] -"""A bytes type that is encoded and decoded using the standard (non-URL-safe) base64 encoder. - -Note: - Under the hood, `Base64Bytes` uses the standard library [`base64.b64encode()`][base64.b64encode] and [`base64.b64decode()`][base64.b64decode] functions. - - As a result, attempting to decode url-safe base64 data using the `Base64Bytes` type may fail or produce an incorrect - decoding. - -/// version-changed | v2.10 -`Base64Bytes` now uses [`base64.b64encode()`][base64.b64encode] and [`base64.b64decode()`][base64.b64decode] -instead of [`base64.encodebytes()`][base64.encodebytes] and [`base64.decodebytes()`][base64.decodebytes]. - -These methods are considered legacy implementation. If you'd still like to use these legacy encoders/decoders, -you can achieve this by creating a custom annotated type, like follows: -```python -import base64 -from typing import Annotated, Literal - -from pydantic_core import PydanticCustomError - -from pydantic import EncodedBytes, EncoderProtocol - -class LegacyBase64Encoder(EncoderProtocol): - @classmethod - def decode(cls, data: bytes) -> bytes: - try: - return base64.decodebytes(data) - except ValueError as e: - raise PydanticCustomError( - 'base64_decode', - "Base64 decoding error: '{error}'", - {'error': str(e)}, - ) - - @classmethod - def encode(cls, value: bytes) -> bytes: - return base64.encodebytes(value) - - @classmethod - def get_json_format(cls) -> Literal['base64']: - return 'base64' - -LegacyBase64Bytes = Annotated[bytes, EncodedBytes(encoder=LegacyBase64Encoder)] -``` -/// - -```python -from pydantic import Base64Bytes, BaseModel, ValidationError - -class Model(BaseModel): - base64_bytes: Base64Bytes - -# Initialize the model with base64 data -m = Model(base64_bytes=b'VGhpcyBpcyB0aGUgd2F5') - -# Access decoded value -print(m.base64_bytes) -#> b'This is the way' - -# Serialize into the base64 form -print(m.model_dump()) -#> {'base64_bytes': b'VGhpcyBpcyB0aGUgd2F5'} - -# Validate base64 data -try: - print(Model(base64_bytes=b'undecodable').base64_bytes) -except ValidationError as e: - print(e) - ''' - 1 validation error for Model - base64_bytes - Base64 decoding error: 'Incorrect padding' [type=base64_decode, input_value=b'undecodable', input_type=bytes] - ''' -``` -""" -Base64Str = Annotated[str, EncodedStr(encoder=Base64Encoder)] -"""A string type that is encoded and decoded using the standard (non-URL-safe) base64 encoder. - -Note: - Under the hood, `Base64Str` uses the standard library [`base64.b64encode()`][base64.b64encode] and [`base64.b64decode()`][base64.b64decode] functions. - - As a result, attempting to decode url-safe base64 data using the `Base64Str` type may fail or produce an incorrect - decoding. - -/// version-changed | v2.10 -`Base64Str` now uses [`base64.b64encode()`][base64.b64encode] and [`base64.b64decode()`][base64.b64decode] -instead of [`base64.encodebytes()`][base64.encodebytes] and [`base64.decodebytes()`][base64.decodebytes]. - -These methods are considered legacy implementation. See the documentation about the [`Base64Bytes`][pydantic.types.Base64Bytes] type -for more information on how to replicate the old behavior with the legacy encoders/decoders. -/// - - -```python -from pydantic import Base64Str, BaseModel, ValidationError - -class Model(BaseModel): - base64_str: Base64Str - -# Initialize the model with base64 data -m = Model(base64_str='VGhlc2UgYXJlbid0IHRoZSBkcm9pZHMgeW91J3JlIGxvb2tpbmcgZm9y') - -# Access decoded value -print(m.base64_str) -#> These aren't the droids you're looking for - -# Serialize into the base64 form -print(m.model_dump()) -#> {'base64_str': 'VGhlc2UgYXJlbid0IHRoZSBkcm9pZHMgeW91J3JlIGxvb2tpbmcgZm9y'} - -# Validate base64 data -try: - print(Model(base64_str='undecodable').base64_str) -except ValidationError as e: - print(e) - ''' - 1 validation error for Model - base64_str - Base64 decoding error: 'Incorrect padding' [type=base64_decode, input_value='undecodable', input_type=str] - ''' -``` -""" -Base64UrlBytes = Annotated[bytes, EncodedBytes(encoder=Base64UrlEncoder)] -"""A bytes type that is encoded and decoded using the URL-safe base64 encoder. - -Note: - Under the hood, `Base64UrlBytes` use standard library `base64.urlsafe_b64encode` and `base64.urlsafe_b64decode` - functions. - - As a result, the `Base64UrlBytes` type can be used to faithfully decode "vanilla" base64 data - (using `'+'` and `'/'`). - -```python -from pydantic import Base64UrlBytes, BaseModel - -class Model(BaseModel): - base64url_bytes: Base64UrlBytes - -# Initialize the model with base64 data -m = Model(base64url_bytes=b'SHc_dHc-TXc==') -print(m) -#> base64url_bytes=b'Hw?tw>Mw' -``` -""" -Base64UrlStr = Annotated[str, EncodedStr(encoder=Base64UrlEncoder)] -"""A str type that is encoded and decoded using the URL-safe base64 encoder. - -Note: - Under the hood, `Base64UrlStr` use standard library `base64.urlsafe_b64encode` and `base64.urlsafe_b64decode` - functions. - - As a result, the `Base64UrlStr` type can be used to faithfully decode "vanilla" base64 data (using `'+'` and `'/'`). - -```python -from pydantic import Base64UrlStr, BaseModel - -class Model(BaseModel): - base64url_str: Base64UrlStr - -# Initialize the model with base64 data -m = Model(base64url_str='SHc_dHc-TXc==') -print(m) -#> base64url_str='Hw?tw>Mw' -``` -""" - - -__getattr__ = getattr_migration(__name__) - - -@_dataclasses.dataclass(**_internal_dataclass.slots_true) -class GetPydanticSchema: - """!!! abstract "Usage Documentation" - [Using `GetPydanticSchema` to Reduce Boilerplate](../concepts/types.md#using-getpydanticschema-to-reduce-boilerplate) - - A convenience class for creating an annotation that provides pydantic custom type hooks. - - This class is intended to eliminate the need to create a custom "marker" which defines the - `__get_pydantic_core_schema__` and `__get_pydantic_json_schema__` custom hook methods. - - For example, to have a field treated by type checkers as `int`, but by pydantic as `Any`, you can do: - ```python - from typing import Annotated, Any - - from pydantic import BaseModel, GetPydanticSchema - - HandleAsAny = GetPydanticSchema(lambda _s, h: h(Any)) - - class Model(BaseModel): - x: Annotated[int, HandleAsAny] # pydantic sees `x: Any` - - print(repr(Model(x='abc').x)) - #> 'abc' - ``` - """ - - get_pydantic_core_schema: Callable[[Any, GetCoreSchemaHandler], CoreSchema] | None = None - get_pydantic_json_schema: Callable[[Any, GetJsonSchemaHandler], JsonSchemaValue] | None = None - - # Note: we may want to consider adding a convenience staticmethod `def for_type(type_: Any) -> GetPydanticSchema:` - # which returns `GetPydanticSchema(lambda _s, h: h(type_))` - - if not TYPE_CHECKING: - # We put `__getattr__` in a non-TYPE_CHECKING block because otherwise, mypy allows arbitrary attribute access - - def __getattr__(self, item: str) -> Any: - """Use this rather than defining `__get_pydantic_core_schema__` etc. to reduce the number of nested calls.""" - if item == '__get_pydantic_core_schema__' and self.get_pydantic_core_schema: - return self.get_pydantic_core_schema - elif item == '__get_pydantic_json_schema__' and self.get_pydantic_json_schema: - return self.get_pydantic_json_schema - else: - return object.__getattribute__(self, item) - - __hash__ = object.__hash__ - - -@_dataclasses.dataclass(**_internal_dataclass.slots_true, frozen=True) -class Tag: - """Provides a way to specify the expected tag to use for a case of a (callable) discriminated union. - - Also provides a way to label a union case in error messages. - - When using a callable `Discriminator`, attach a `Tag` to each case in the `Union` to specify the tag that - should be used to identify that case. For example, in the below example, the `Tag` is used to specify that - if `get_discriminator_value` returns `'apple'`, the input should be validated as an `ApplePie`, and if it - returns `'pumpkin'`, the input should be validated as a `PumpkinPie`. - - The primary role of the `Tag` here is to map the return value from the callable `Discriminator` function to - the appropriate member of the `Union` in question. - - ```python - from typing import Annotated, Any, Literal, Union - - from pydantic import BaseModel, Discriminator, Tag - - class Pie(BaseModel): - time_to_cook: int - num_ingredients: int - - class ApplePie(Pie): - fruit: Literal['apple'] = 'apple' - - class PumpkinPie(Pie): - filling: Literal['pumpkin'] = 'pumpkin' - - def get_discriminator_value(v: Any) -> str: - if isinstance(v, dict): - return v.get('fruit', v.get('filling')) - return getattr(v, 'fruit', getattr(v, 'filling', None)) - - class ThanksgivingDinner(BaseModel): - dessert: Annotated[ - Union[ - Annotated[ApplePie, Tag('apple')], - Annotated[PumpkinPie, Tag('pumpkin')], - ], - Discriminator(get_discriminator_value), - ] - - apple_variation = ThanksgivingDinner.model_validate( - {'dessert': {'fruit': 'apple', 'time_to_cook': 60, 'num_ingredients': 8}} - ) - print(repr(apple_variation)) - ''' - ThanksgivingDinner(dessert=ApplePie(time_to_cook=60, num_ingredients=8, fruit='apple')) - ''' - - pumpkin_variation = ThanksgivingDinner.model_validate( - { - 'dessert': { - 'filling': 'pumpkin', - 'time_to_cook': 40, - 'num_ingredients': 6, - } - } - ) - print(repr(pumpkin_variation)) - ''' - ThanksgivingDinner(dessert=PumpkinPie(time_to_cook=40, num_ingredients=6, filling='pumpkin')) - ''' - ``` - - !!! note - You must specify a `Tag` for every case in a `Tag` that is associated with a - callable `Discriminator`. Failing to do so will result in a `PydanticUserError` with code - [`callable-discriminator-no-tag`](../errors/usage_errors.md#callable-discriminator-no-tag). - - See the [Discriminated Unions] concepts docs for more details on how to use `Tag`s. - - [Discriminated Unions]: ../concepts/unions.md#discriminated-unions - """ - - tag: str - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema: - schema = handler(source_type) - metadata = cast('CoreMetadata', schema.setdefault('metadata', {})) - metadata['pydantic_internal_union_tag_key'] = self.tag - return schema - - -@_dataclasses.dataclass(**_internal_dataclass.slots_true, frozen=True) -class Discriminator: - """!!! abstract "Usage Documentation" - [Discriminated Unions with `Callable` `Discriminator`](../concepts/unions.md#discriminated-unions-with-callable-discriminator) - - Provides a way to use a custom callable as the way to extract the value of a union discriminator. - - This allows you to get validation behavior like you'd get from `Field(discriminator=)`, - but without needing to have a single shared field across all the union choices. This also makes it - possible to handle unions of models and primitive types with discriminated-union-style validation errors. - Finally, this allows you to use a custom callable as the way to identify which member of a union a value - belongs to, while still seeing all the performance benefits of a discriminated union. - - Consider this example, which is much more performant with the use of `Discriminator` and thus a `TaggedUnion` - than it would be as a normal `Union`. - - ```python - from typing import Annotated, Any, Literal, Union - - from pydantic import BaseModel, Discriminator, Tag - - class Pie(BaseModel): - time_to_cook: int - num_ingredients: int - - class ApplePie(Pie): - fruit: Literal['apple'] = 'apple' - - class PumpkinPie(Pie): - filling: Literal['pumpkin'] = 'pumpkin' - - def get_discriminator_value(v: Any) -> str: - if isinstance(v, dict): - return v.get('fruit', v.get('filling')) - return getattr(v, 'fruit', getattr(v, 'filling', None)) - - class ThanksgivingDinner(BaseModel): - dessert: Annotated[ - Union[ - Annotated[ApplePie, Tag('apple')], - Annotated[PumpkinPie, Tag('pumpkin')], - ], - Discriminator(get_discriminator_value), - ] - - apple_variation = ThanksgivingDinner.model_validate( - {'dessert': {'fruit': 'apple', 'time_to_cook': 60, 'num_ingredients': 8}} - ) - print(repr(apple_variation)) - ''' - ThanksgivingDinner(dessert=ApplePie(time_to_cook=60, num_ingredients=8, fruit='apple')) - ''' - - pumpkin_variation = ThanksgivingDinner.model_validate( - { - 'dessert': { - 'filling': 'pumpkin', - 'time_to_cook': 40, - 'num_ingredients': 6, - } - } - ) - print(repr(pumpkin_variation)) - ''' - ThanksgivingDinner(dessert=PumpkinPie(time_to_cook=40, num_ingredients=6, filling='pumpkin')) - ''' - ``` - - See the [Discriminated Unions] concepts docs for more details on how to use `Discriminator`s. - - [Discriminated Unions]: ../concepts/unions.md#discriminated-unions - """ - - discriminator: str | Callable[[Any], Hashable] - """The callable or field name for discriminating the type in a tagged union. - - A `Callable` discriminator must extract the value of the discriminator from the input. - A `str` discriminator must be the name of a field to discriminate against. - """ - custom_error_type: str | None = None - """Type to use in [custom errors](../errors/errors.md) replacing the standard discriminated union - validation errors. - """ - custom_error_message: str | None = None - """Message to use in custom errors.""" - custom_error_context: dict[str, int | str | float] | None = None - """Context to use in custom errors.""" - - def __get_pydantic_core_schema__(self, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema: - if isinstance(self.discriminator, str): - from pydantic import Field - - return handler(Annotated[source_type, Field(discriminator=self.discriminator)]) - else: - original_schema = handler(source_type) - return self._convert_schema(original_schema, handler) - - def _convert_schema( - self, original_schema: core_schema.CoreSchema, handler: GetCoreSchemaHandler | None = None - ) -> core_schema.TaggedUnionSchema: - if handler is not None and original_schema['type'] == 'definition-ref': - # Same logic as `_ApplyInferredDiscriminator._apply_to_root()` - try: - def_schema = handler.resolve_ref_schema(original_schema) - except LookupError: # pragma: no cover - from pydantic._internal._discriminated_union import MissingDefinitionForUnionRef - - raise MissingDefinitionForUnionRef(original_schema['schema_ref']) - - # If using a referenceable union as discriminated (e.g. `type Pet = Cat | Dog; field: Pet = Field(discriminator=...)`): - if def_schema['type'] == 'union': - original_schema = def_schema.copy() - original_schema.pop('ref') - - if original_schema['type'] != 'union': - # This likely indicates that the schema was a single-item union that was simplified. - # In this case, we do the same thing we do in - # `pydantic._internal._discriminated_union._ApplyInferredDiscriminator._apply_to_root`, namely, - # package the generated schema back into a single-item union. - original_schema = core_schema.union_schema([original_schema]) - - tagged_union_choices = {} - for choice in original_schema['choices']: - tag = None - if isinstance(choice, tuple): - choice, tag = choice - metadata = cast('CoreMetadata | None', choice.get('metadata')) - if metadata is not None: - tag = metadata.get('pydantic_internal_union_tag_key') or tag - if tag is None: - # `handler` is None when this method is called from `apply_discriminator()` (deferred discriminators) - if handler is not None and choice['type'] == 'definition-ref': - # If choice was built from a PEP 695 type alias, try to resolve the def: - try: - choice = handler.resolve_ref_schema(choice) - except LookupError: - pass - else: - metadata = cast('CoreMetadata | None', choice.get('metadata')) - if metadata is not None: - tag = metadata.get('pydantic_internal_union_tag_key') - - if tag is None: - raise PydanticUserError( - f'`Tag` not provided for choice {choice} used with `Discriminator`', - code='callable-discriminator-no-tag', - ) - tagged_union_choices[tag] = choice - - # Have to do these verbose checks to ensure falsy values ('' and {}) don't get ignored - custom_error_type = self.custom_error_type - if custom_error_type is None: - custom_error_type = original_schema.get('custom_error_type') - - custom_error_message = self.custom_error_message - if custom_error_message is None: - custom_error_message = original_schema.get('custom_error_message') - - custom_error_context = self.custom_error_context - if custom_error_context is None: - custom_error_context = original_schema.get('custom_error_context') - - custom_error_type = original_schema.get('custom_error_type') if custom_error_type is None else custom_error_type - return core_schema.tagged_union_schema( - tagged_union_choices, - self.discriminator, - custom_error_type=custom_error_type, - custom_error_message=custom_error_message, - custom_error_context=custom_error_context, - strict=original_schema.get('strict'), - ref=original_schema.get('ref'), - metadata=original_schema.get('metadata'), - serialization=original_schema.get('serialization'), - ) - - -_JSON_TYPES = {int, float, str, bool, list, dict, type(None)} - - -def _get_type_name(x: Any) -> str: - type_ = type(x) - if type_ in _JSON_TYPES: - return type_.__name__ - - # Handle proper subclasses; note we don't need to handle None or bool here - if isinstance(x, int): - return 'int' - if isinstance(x, float): - return 'float' - if isinstance(x, str): - return 'str' - if isinstance(x, list): - return 'list' - if isinstance(x, dict): - return 'dict' - - # Fail by returning the type's actual name - return getattr(type_, '__name__', '') - - -class _AllowAnyJson: - @classmethod - def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema: - python_schema = handler(source_type) - return core_schema.json_or_python_schema(json_schema=core_schema.any_schema(), python_schema=python_schema) - - -if TYPE_CHECKING: - # This seems to only be necessary for mypy - JsonValue: TypeAlias = Union[ - list['JsonValue'], - dict[str, 'JsonValue'], - str, - bool, - int, - float, - None, - ] - """A `JsonValue` is used to represent a value that can be serialized to JSON. - - It may be one of: - - * `list['JsonValue']` - * `dict[str, 'JsonValue']` - * `str` - * `bool` - * `int` - * `float` - * `None` - - The following example demonstrates how to use `JsonValue` to validate JSON data, - and what kind of errors to expect when input data is not json serializable. - - ```python - import json - - from pydantic import BaseModel, JsonValue, ValidationError - - class Model(BaseModel): - j: JsonValue - - valid_json_data = {'j': {'a': {'b': {'c': 1, 'd': [2, None]}}}} - invalid_json_data = {'j': {'a': {'b': ...}}} - - print(repr(Model.model_validate(valid_json_data))) - #> Model(j={'a': {'b': {'c': 1, 'd': [2, None]}}}) - print(repr(Model.model_validate_json(json.dumps(valid_json_data)))) - #> Model(j={'a': {'b': {'c': 1, 'd': [2, None]}}}) - - try: - Model.model_validate(invalid_json_data) - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - j.dict.a.dict.b - input was not a valid JSON value [type=invalid-json-value, input_value=Ellipsis, input_type=ellipsis] - ''' - ``` - """ - -else: - JsonValue = TypeAliasType( - 'JsonValue', - Annotated[ - Union[ - Annotated[list['JsonValue'], Tag('list')], - Annotated[dict[str, 'JsonValue'], Tag('dict')], - Annotated[str, Tag('str')], - Annotated[bool, Tag('bool')], - Annotated[int, Tag('int')], - Annotated[float, Tag('float')], - Annotated[None, Tag('NoneType')], - ], - Discriminator( - _get_type_name, - custom_error_type='invalid-json-value', - custom_error_message='input was not a valid JSON value', - ), - _AllowAnyJson, - ], - ) - - -class _OnErrorOmit: - @classmethod - def __get_pydantic_core_schema__(cls, source_type: Any, handler: GetCoreSchemaHandler) -> CoreSchema: - # there is no actual default value here but we use with_default_schema since it already has the on_error - # behavior implemented and it would be no more efficient to implement it on every other validator - # or as a standalone validator - return core_schema.with_default_schema(schema=handler(source_type), on_error='omit') - - -OnErrorOmit = Annotated[T, _OnErrorOmit] -""" -When used as an item in a list, the key type in a dict, optional values of a TypedDict, etc. -this annotation omits the item from the iteration if there is any error validating it. -That is, instead of a [`ValidationError`][pydantic_core.ValidationError] being propagated up and the entire iterable being discarded -any invalid items are discarded and the valid ones are returned. -""" - - -@_dataclasses.dataclass -class FailFast(_fields.PydanticMetadata, BaseMetadata): - """A `FailFast` annotation can be used to specify that validation should stop at the first error. - - This can be useful when you want to validate a large amount of data and you only need to know if it's valid or not. - - You might want to enable this setting if you want to validate your data faster (basically, if you use this, - validation will be more performant with the caveat that you get less information). - - ```python - from typing import Annotated - - from pydantic import BaseModel, FailFast, ValidationError - - class Model(BaseModel): - x: Annotated[list[int], FailFast()] - - # This will raise a single error for the first invalid value and stop validation - try: - obj = Model(x=[1, 2, 'a', 4, 5, 'b', 7, 8, 9, 'c']) - except ValidationError as e: - print(e) - ''' - 1 validation error for Model - x.2 - Input should be a valid integer, unable to parse string as an integer [type=int_parsing, input_value='a', input_type=str] - ''' - ``` - """ - - fail_fast: bool = True diff --git a/functions/python/data-processor/pydantic/typing.py b/functions/python/data-processor/pydantic/typing.py deleted file mode 100644 index 0bda22d02..000000000 --- a/functions/python/data-processor/pydantic/typing.py +++ /dev/null @@ -1,5 +0,0 @@ -"""`typing` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/utils.py b/functions/python/data-processor/pydantic/utils.py deleted file mode 100644 index 8d1e2a81c..000000000 --- a/functions/python/data-processor/pydantic/utils.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `utils` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/v1/__init__.py b/functions/python/data-processor/pydantic/v1/__init__.py deleted file mode 100644 index 6ad3f4666..000000000 --- a/functions/python/data-processor/pydantic/v1/__init__.py +++ /dev/null @@ -1,131 +0,0 @@ -# flake8: noqa -from pydantic.v1 import dataclasses -from pydantic.v1.annotated_types import create_model_from_namedtuple, create_model_from_typeddict -from pydantic.v1.class_validators import root_validator, validator -from pydantic.v1.config import BaseConfig, ConfigDict, Extra -from pydantic.v1.decorator import validate_arguments -from pydantic.v1.env_settings import BaseSettings -from pydantic.v1.error_wrappers import ValidationError -from pydantic.v1.errors import * -from pydantic.v1.fields import Field, PrivateAttr, Required -from pydantic.v1.main import * -from pydantic.v1.networks import * -from pydantic.v1.parse import Protocol -from pydantic.v1.tools import * -from pydantic.v1.types import * -from pydantic.v1.version import VERSION, compiled - -__version__ = VERSION - -# WARNING __all__ from pydantic.errors is not included here, it will be removed as an export here in v2 -# please use "from pydantic.v1.errors import ..." instead -__all__ = [ - # annotated types utils - 'create_model_from_namedtuple', - 'create_model_from_typeddict', - # dataclasses - 'dataclasses', - # class_validators - 'root_validator', - 'validator', - # config - 'BaseConfig', - 'ConfigDict', - 'Extra', - # decorator - 'validate_arguments', - # env_settings - 'BaseSettings', - # error_wrappers - 'ValidationError', - # fields - 'Field', - 'Required', - # main - 'BaseModel', - 'create_model', - 'validate_model', - # network - 'AnyUrl', - 'AnyHttpUrl', - 'FileUrl', - 'HttpUrl', - 'stricturl', - 'EmailStr', - 'NameEmail', - 'IPvAnyAddress', - 'IPvAnyInterface', - 'IPvAnyNetwork', - 'PostgresDsn', - 'CockroachDsn', - 'AmqpDsn', - 'RedisDsn', - 'MongoDsn', - 'KafkaDsn', - 'validate_email', - # parse - 'Protocol', - # tools - 'parse_file_as', - 'parse_obj_as', - 'parse_raw_as', - 'schema_of', - 'schema_json_of', - # types - 'NoneStr', - 'NoneBytes', - 'StrBytes', - 'NoneStrBytes', - 'StrictStr', - 'ConstrainedBytes', - 'conbytes', - 'ConstrainedList', - 'conlist', - 'ConstrainedSet', - 'conset', - 'ConstrainedFrozenSet', - 'confrozenset', - 'ConstrainedStr', - 'constr', - 'PyObject', - 'ConstrainedInt', - 'conint', - 'PositiveInt', - 'NegativeInt', - 'NonNegativeInt', - 'NonPositiveInt', - 'ConstrainedFloat', - 'confloat', - 'PositiveFloat', - 'NegativeFloat', - 'NonNegativeFloat', - 'NonPositiveFloat', - 'FiniteFloat', - 'ConstrainedDecimal', - 'condecimal', - 'ConstrainedDate', - 'condate', - 'UUID1', - 'UUID3', - 'UUID4', - 'UUID5', - 'FilePath', - 'DirectoryPath', - 'Json', - 'JsonWrapper', - 'SecretField', - 'SecretStr', - 'SecretBytes', - 'StrictBool', - 'StrictBytes', - 'StrictInt', - 'StrictFloat', - 'PaymentCardNumber', - 'PrivateAttr', - 'ByteSize', - 'PastDate', - 'FutureDate', - # version - 'compiled', - 'VERSION', -] diff --git a/functions/python/data-processor/pydantic/v1/_hypothesis_plugin.py b/functions/python/data-processor/pydantic/v1/_hypothesis_plugin.py deleted file mode 100644 index b62234d50..000000000 --- a/functions/python/data-processor/pydantic/v1/_hypothesis_plugin.py +++ /dev/null @@ -1,391 +0,0 @@ -""" -Register Hypothesis strategies for Pydantic custom types. - -This enables fully-automatic generation of test data for most Pydantic classes. - -Note that this module has *no* runtime impact on Pydantic itself; instead it -is registered as a setuptools entry point and Hypothesis will import it if -Pydantic is installed. See also: - -https://hypothesis.readthedocs.io/en/latest/strategies.html#registering-strategies-via-setuptools-entry-points -https://hypothesis.readthedocs.io/en/latest/data.html#hypothesis.strategies.register_type_strategy -https://hypothesis.readthedocs.io/en/latest/strategies.html#interaction-with-pytest-cov -https://docs.pydantic.dev/usage/types/#pydantic-types - -Note that because our motivation is to *improve user experience*, the strategies -are always sound (never generate invalid data) but sacrifice completeness for -maintainability (ie may be unable to generate some tricky but valid data). - -Finally, this module makes liberal use of `# type: ignore[]` pragmas. -This is because Hypothesis annotates `register_type_strategy()` with -`(T, SearchStrategy[T])`, but in most cases we register e.g. `ConstrainedInt` -to generate instances of the builtin `int` type which match the constraints. -""" - -import contextlib -import datetime -import ipaddress -import json -import math -from fractions import Fraction -from typing import Callable, Dict, Type, Union, cast, overload - -import hypothesis.strategies as st - -import pydantic -import pydantic.color -import pydantic.types -from pydantic.v1.utils import lenient_issubclass - -# FilePath and DirectoryPath are explicitly unsupported, as we'd have to create -# them on-disk, and that's unsafe in general without being told *where* to do so. -# -# URLs are unsupported because it's easy for users to define their own strategy for -# "normal" URLs, and hard for us to define a general strategy which includes "weird" -# URLs but doesn't also have unpredictable performance problems. -# -# conlist() and conset() are unsupported for now, because the workarounds for -# Cython and Hypothesis to handle parametrized generic types are incompatible. -# We are rethinking Hypothesis compatibility in Pydantic v2. - -# Emails -try: - import email_validator -except ImportError: # pragma: no cover - pass -else: - - def is_valid_email(s: str) -> bool: - # Hypothesis' st.emails() occasionally generates emails like 0@A0--0.ac - # that are invalid according to email-validator, so we filter those out. - try: - email_validator.validate_email(s, check_deliverability=False) - return True - except email_validator.EmailNotValidError: # pragma: no cover - return False - - # Note that these strategies deliberately stay away from any tricky Unicode - # or other encoding issues; we're just trying to generate *something* valid. - st.register_type_strategy(pydantic.EmailStr, st.emails().filter(is_valid_email)) # type: ignore[arg-type] - st.register_type_strategy( - pydantic.NameEmail, - st.builds( - '{} <{}>'.format, # type: ignore[arg-type] - st.from_regex('[A-Za-z0-9_]+( [A-Za-z0-9_]+){0,5}', fullmatch=True), - st.emails().filter(is_valid_email), - ), - ) - -# PyObject - dotted names, in this case taken from the math module. -st.register_type_strategy( - pydantic.PyObject, # type: ignore[arg-type] - st.sampled_from( - [cast(pydantic.PyObject, f'math.{name}') for name in sorted(vars(math)) if not name.startswith('_')] - ), -) - -# CSS3 Colors; as name, hex, rgb(a) tuples or strings, or hsl strings -_color_regexes = ( - '|'.join( - ( - pydantic.color.r_hex_short, - pydantic.color.r_hex_long, - pydantic.color.r_rgb, - pydantic.color.r_rgba, - pydantic.color.r_hsl, - pydantic.color.r_hsla, - ) - ) - # Use more precise regex patterns to avoid value-out-of-range errors - .replace(pydantic.color._r_sl, r'(?:(\d\d?(?:\.\d+)?|100(?:\.0+)?)%)') - .replace(pydantic.color._r_alpha, r'(?:(0(?:\.\d+)?|1(?:\.0+)?|\.\d+|\d{1,2}%))') - .replace(pydantic.color._r_255, r'(?:((?:\d|\d\d|[01]\d\d|2[0-4]\d|25[0-4])(?:\.\d+)?|255(?:\.0+)?))') -) -st.register_type_strategy( - pydantic.color.Color, - st.one_of( - st.sampled_from(sorted(pydantic.color.COLORS_BY_NAME)), - st.tuples( - st.integers(0, 255), - st.integers(0, 255), - st.integers(0, 255), - st.none() | st.floats(0, 1) | st.floats(0, 100).map('{}%'.format), - ), - st.from_regex(_color_regexes, fullmatch=True), - ), -) - - -# Card numbers, valid according to the Luhn algorithm - - -def add_luhn_digit(card_number: str) -> str: - # See https://en.wikipedia.org/wiki/Luhn_algorithm - for digit in '0123456789': - with contextlib.suppress(Exception): - pydantic.PaymentCardNumber.validate_luhn_check_digit(card_number + digit) - return card_number + digit - raise AssertionError('Unreachable') # pragma: no cover - - -card_patterns = ( - # Note that these patterns omit the Luhn check digit; that's added by the function above - '4[0-9]{14}', # Visa - '5[12345][0-9]{13}', # Mastercard - '3[47][0-9]{12}', # American Express - '[0-26-9][0-9]{10,17}', # other (incomplete to avoid overlap) -) -st.register_type_strategy( - pydantic.PaymentCardNumber, - st.from_regex('|'.join(card_patterns), fullmatch=True).map(add_luhn_digit), # type: ignore[arg-type] -) - -# UUIDs -st.register_type_strategy(pydantic.UUID1, st.uuids(version=1)) -st.register_type_strategy(pydantic.UUID3, st.uuids(version=3)) -st.register_type_strategy(pydantic.UUID4, st.uuids(version=4)) -st.register_type_strategy(pydantic.UUID5, st.uuids(version=5)) - -# Secrets -st.register_type_strategy(pydantic.SecretBytes, st.binary().map(pydantic.SecretBytes)) -st.register_type_strategy(pydantic.SecretStr, st.text().map(pydantic.SecretStr)) - -# IP addresses, networks, and interfaces -st.register_type_strategy(pydantic.IPvAnyAddress, st.ip_addresses()) # type: ignore[arg-type] -st.register_type_strategy( - pydantic.IPvAnyInterface, - st.from_type(ipaddress.IPv4Interface) | st.from_type(ipaddress.IPv6Interface), # type: ignore[arg-type] -) -st.register_type_strategy( - pydantic.IPvAnyNetwork, - st.from_type(ipaddress.IPv4Network) | st.from_type(ipaddress.IPv6Network), # type: ignore[arg-type] -) - -# We hook into the con***() functions and the ConstrainedNumberMeta metaclass, -# so here we only have to register subclasses for other constrained types which -# don't go via those mechanisms. Then there are the registration hooks below. -st.register_type_strategy(pydantic.StrictBool, st.booleans()) -st.register_type_strategy(pydantic.StrictStr, st.text()) - - -# FutureDate, PastDate -st.register_type_strategy(pydantic.FutureDate, st.dates(min_value=datetime.date.today() + datetime.timedelta(days=1))) -st.register_type_strategy(pydantic.PastDate, st.dates(max_value=datetime.date.today() - datetime.timedelta(days=1))) - - -# Constrained-type resolver functions -# -# For these ones, we actually want to inspect the type in order to work out a -# satisfying strategy. First up, the machinery for tracking resolver functions: - -RESOLVERS: Dict[type, Callable[[type], st.SearchStrategy]] = {} # type: ignore[type-arg] - - -@overload -def _registered(typ: Type[pydantic.types.T]) -> Type[pydantic.types.T]: - pass - - -@overload -def _registered(typ: pydantic.types.ConstrainedNumberMeta) -> pydantic.types.ConstrainedNumberMeta: - pass - - -def _registered( - typ: Union[Type[pydantic.types.T], pydantic.types.ConstrainedNumberMeta] -) -> Union[Type[pydantic.types.T], pydantic.types.ConstrainedNumberMeta]: - # This function replaces the version in `pydantic.types`, in order to - # effect the registration of new constrained types so that Hypothesis - # can generate valid examples. - pydantic.types._DEFINED_TYPES.add(typ) - for supertype, resolver in RESOLVERS.items(): - if issubclass(typ, supertype): - st.register_type_strategy(typ, resolver(typ)) # type: ignore - return typ - raise NotImplementedError(f'Unknown type {typ!r} has no resolver to register') # pragma: no cover - - -def resolves( - typ: Union[type, pydantic.types.ConstrainedNumberMeta] -) -> Callable[[Callable[..., st.SearchStrategy]], Callable[..., st.SearchStrategy]]: # type: ignore[type-arg] - def inner(f): # type: ignore - assert f not in RESOLVERS - RESOLVERS[typ] = f - return f - - return inner - - -# Type-to-strategy resolver functions - - -@resolves(pydantic.JsonWrapper) -def resolve_json(cls): # type: ignore[no-untyped-def] - try: - inner = st.none() if cls.inner_type is None else st.from_type(cls.inner_type) - except Exception: # pragma: no cover - finite = st.floats(allow_infinity=False, allow_nan=False) - inner = st.recursive( - base=st.one_of(st.none(), st.booleans(), st.integers(), finite, st.text()), - extend=lambda x: st.lists(x) | st.dictionaries(st.text(), x), # type: ignore - ) - inner_type = getattr(cls, 'inner_type', None) - return st.builds( - cls.inner_type.json if lenient_issubclass(inner_type, pydantic.BaseModel) else json.dumps, - inner, - ensure_ascii=st.booleans(), - indent=st.none() | st.integers(0, 16), - sort_keys=st.booleans(), - ) - - -@resolves(pydantic.ConstrainedBytes) -def resolve_conbytes(cls): # type: ignore[no-untyped-def] # pragma: no cover - min_size = cls.min_length or 0 - max_size = cls.max_length - if not cls.strip_whitespace: - return st.binary(min_size=min_size, max_size=max_size) - # Fun with regex to ensure we neither start nor end with whitespace - repeats = '{{{},{}}}'.format( - min_size - 2 if min_size > 2 else 0, - max_size - 2 if (max_size or 0) > 2 else '', - ) - if min_size >= 2: - pattern = rf'\W.{repeats}\W' - elif min_size == 1: - pattern = rf'\W(.{repeats}\W)?' - else: - assert min_size == 0 - pattern = rf'(\W(.{repeats}\W)?)?' - return st.from_regex(pattern.encode(), fullmatch=True) - - -@resolves(pydantic.ConstrainedDecimal) -def resolve_condecimal(cls): # type: ignore[no-untyped-def] - min_value = cls.ge - max_value = cls.le - if cls.gt is not None: - assert min_value is None, 'Set `gt` or `ge`, but not both' - min_value = cls.gt - if cls.lt is not None: - assert max_value is None, 'Set `lt` or `le`, but not both' - max_value = cls.lt - s = st.decimals(min_value, max_value, allow_nan=False, places=cls.decimal_places) - if cls.lt is not None: - s = s.filter(lambda d: d < cls.lt) - if cls.gt is not None: - s = s.filter(lambda d: cls.gt < d) - return s - - -@resolves(pydantic.ConstrainedFloat) -def resolve_confloat(cls): # type: ignore[no-untyped-def] - min_value = cls.ge - max_value = cls.le - exclude_min = False - exclude_max = False - - if cls.gt is not None: - assert min_value is None, 'Set `gt` or `ge`, but not both' - min_value = cls.gt - exclude_min = True - if cls.lt is not None: - assert max_value is None, 'Set `lt` or `le`, but not both' - max_value = cls.lt - exclude_max = True - - if cls.multiple_of is None: - return st.floats(min_value, max_value, exclude_min=exclude_min, exclude_max=exclude_max, allow_nan=False) - - if min_value is not None: - min_value = math.ceil(min_value / cls.multiple_of) - if exclude_min: - min_value = min_value + 1 - if max_value is not None: - assert max_value >= cls.multiple_of, 'Cannot build model with max value smaller than multiple of' - max_value = math.floor(max_value / cls.multiple_of) - if exclude_max: - max_value = max_value - 1 - - return st.integers(min_value, max_value).map(lambda x: x * cls.multiple_of) - - -@resolves(pydantic.ConstrainedInt) -def resolve_conint(cls): # type: ignore[no-untyped-def] - min_value = cls.ge - max_value = cls.le - if cls.gt is not None: - assert min_value is None, 'Set `gt` or `ge`, but not both' - min_value = cls.gt + 1 - if cls.lt is not None: - assert max_value is None, 'Set `lt` or `le`, but not both' - max_value = cls.lt - 1 - - if cls.multiple_of is None or cls.multiple_of == 1: - return st.integers(min_value, max_value) - - # These adjustments and the .map handle integer-valued multiples, while the - # .filter handles trickier cases as for confloat. - if min_value is not None: - min_value = math.ceil(Fraction(min_value) / Fraction(cls.multiple_of)) - if max_value is not None: - max_value = math.floor(Fraction(max_value) / Fraction(cls.multiple_of)) - return st.integers(min_value, max_value).map(lambda x: x * cls.multiple_of) - - -@resolves(pydantic.ConstrainedDate) -def resolve_condate(cls): # type: ignore[no-untyped-def] - if cls.ge is not None: - assert cls.gt is None, 'Set `gt` or `ge`, but not both' - min_value = cls.ge - elif cls.gt is not None: - min_value = cls.gt + datetime.timedelta(days=1) - else: - min_value = datetime.date.min - if cls.le is not None: - assert cls.lt is None, 'Set `lt` or `le`, but not both' - max_value = cls.le - elif cls.lt is not None: - max_value = cls.lt - datetime.timedelta(days=1) - else: - max_value = datetime.date.max - return st.dates(min_value, max_value) - - -@resolves(pydantic.ConstrainedStr) -def resolve_constr(cls): # type: ignore[no-untyped-def] # pragma: no cover - min_size = cls.min_length or 0 - max_size = cls.max_length - - if cls.regex is None and not cls.strip_whitespace: - return st.text(min_size=min_size, max_size=max_size) - - if cls.regex is not None: - strategy = st.from_regex(cls.regex) - if cls.strip_whitespace: - strategy = strategy.filter(lambda s: s == s.strip()) - elif cls.strip_whitespace: - repeats = '{{{},{}}}'.format( - min_size - 2 if min_size > 2 else 0, - max_size - 2 if (max_size or 0) > 2 else '', - ) - if min_size >= 2: - strategy = st.from_regex(rf'\W.{repeats}\W') - elif min_size == 1: - strategy = st.from_regex(rf'\W(.{repeats}\W)?') - else: - assert min_size == 0 - strategy = st.from_regex(rf'(\W(.{repeats}\W)?)?') - - if min_size == 0 and max_size is None: - return strategy - elif max_size is None: - return strategy.filter(lambda s: min_size <= len(s)) - return strategy.filter(lambda s: min_size <= len(s) <= max_size) - - -# Finally, register all previously-defined types, and patch in our new function -for typ in list(pydantic.types._DEFINED_TYPES): - _registered(typ) -pydantic.types._registered = _registered -st.register_type_strategy(pydantic.Json, resolve_json) diff --git a/functions/python/data-processor/pydantic/v1/annotated_types.py b/functions/python/data-processor/pydantic/v1/annotated_types.py deleted file mode 100644 index d9eaaafd5..000000000 --- a/functions/python/data-processor/pydantic/v1/annotated_types.py +++ /dev/null @@ -1,72 +0,0 @@ -import sys -from typing import TYPE_CHECKING, Any, Dict, FrozenSet, NamedTuple, Type - -from pydantic.v1.fields import Required -from pydantic.v1.main import BaseModel, create_model -from pydantic.v1.typing import is_typeddict, is_typeddict_special - -if TYPE_CHECKING: - from typing_extensions import TypedDict - -if sys.version_info < (3, 11): - - def is_legacy_typeddict(typeddict_cls: Type['TypedDict']) -> bool: # type: ignore[valid-type] - return is_typeddict(typeddict_cls) and type(typeddict_cls).__module__ == 'typing' - -else: - - def is_legacy_typeddict(_: Any) -> Any: - return False - - -def create_model_from_typeddict( - # Mypy bug: `Type[TypedDict]` is resolved as `Any` https://github.com/python/mypy/issues/11030 - typeddict_cls: Type['TypedDict'], # type: ignore[valid-type] - **kwargs: Any, -) -> Type['BaseModel']: - """ - Create a `BaseModel` based on the fields of a `TypedDict`. - Since `typing.TypedDict` in Python 3.8 does not store runtime information about optional keys, - we raise an error if this happens (see https://bugs.python.org/issue38834). - """ - field_definitions: Dict[str, Any] - - # Best case scenario: with python 3.9+ or when `TypedDict` is imported from `typing_extensions` - if not hasattr(typeddict_cls, '__required_keys__'): - raise TypeError( - 'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` with Python < 3.9.2. ' - 'Without it, there is no way to differentiate required and optional fields when subclassed.' - ) - - if is_legacy_typeddict(typeddict_cls) and any( - is_typeddict_special(t) for t in typeddict_cls.__annotations__.values() - ): - raise TypeError( - 'You should use `typing_extensions.TypedDict` instead of `typing.TypedDict` with Python < 3.11. ' - 'Without it, there is no way to reflect Required/NotRequired keys.' - ) - - required_keys: FrozenSet[str] = typeddict_cls.__required_keys__ # type: ignore[attr-defined] - field_definitions = { - field_name: (field_type, Required if field_name in required_keys else None) - for field_name, field_type in typeddict_cls.__annotations__.items() - } - - return create_model(typeddict_cls.__name__, **kwargs, **field_definitions) - - -def create_model_from_namedtuple(namedtuple_cls: Type['NamedTuple'], **kwargs: Any) -> Type['BaseModel']: - """ - Create a `BaseModel` based on the fields of a named tuple. - A named tuple can be created with `typing.NamedTuple` and declared annotations - but also with `collections.namedtuple`, in this case we consider all fields - to have type `Any`. - """ - # With python 3.10+, `__annotations__` always exists but can be empty hence the `getattr... or...` logic - namedtuple_annotations: Dict[str, Type[Any]] = getattr(namedtuple_cls, '__annotations__', None) or { - k: Any for k in namedtuple_cls._fields - } - field_definitions: Dict[str, Any] = { - field_name: (field_type, Required) for field_name, field_type in namedtuple_annotations.items() - } - return create_model(namedtuple_cls.__name__, **kwargs, **field_definitions) diff --git a/functions/python/data-processor/pydantic/v1/class_validators.py b/functions/python/data-processor/pydantic/v1/class_validators.py deleted file mode 100644 index 2f68fc860..000000000 --- a/functions/python/data-processor/pydantic/v1/class_validators.py +++ /dev/null @@ -1,361 +0,0 @@ -import warnings -from collections import ChainMap -from functools import partial, partialmethod, wraps -from itertools import chain -from types import FunctionType -from typing import TYPE_CHECKING, Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Type, Union, overload - -from pydantic.v1.errors import ConfigError -from pydantic.v1.typing import AnyCallable -from pydantic.v1.utils import ROOT_KEY, in_ipython - -if TYPE_CHECKING: - from pydantic.v1.typing import AnyClassMethod - - -class Validator: - __slots__ = 'func', 'pre', 'each_item', 'always', 'check_fields', 'skip_on_failure' - - def __init__( - self, - func: AnyCallable, - pre: bool = False, - each_item: bool = False, - always: bool = False, - check_fields: bool = False, - skip_on_failure: bool = False, - ): - self.func = func - self.pre = pre - self.each_item = each_item - self.always = always - self.check_fields = check_fields - self.skip_on_failure = skip_on_failure - - -if TYPE_CHECKING: - from inspect import Signature - - from pydantic.v1.config import BaseConfig - from pydantic.v1.fields import ModelField - from pydantic.v1.types import ModelOrDc - - ValidatorCallable = Callable[[Optional[ModelOrDc], Any, Dict[str, Any], ModelField, Type[BaseConfig]], Any] - ValidatorsList = List[ValidatorCallable] - ValidatorListDict = Dict[str, List[Validator]] - -_FUNCS: Set[str] = set() -VALIDATOR_CONFIG_KEY = '__validator_config__' -ROOT_VALIDATOR_CONFIG_KEY = '__root_validator_config__' - - -def validator( - *fields: str, - pre: bool = False, - each_item: bool = False, - always: bool = False, - check_fields: bool = True, - whole: Optional[bool] = None, - allow_reuse: bool = False, -) -> Callable[[AnyCallable], 'AnyClassMethod']: - """ - Decorate methods on the class indicating that they should be used to validate fields - :param fields: which field(s) the method should be called on - :param pre: whether or not this validator should be called before the standard validators (else after) - :param each_item: for complex objects (sets, lists etc.) whether to validate individual elements rather than the - whole object - :param always: whether this method and other validators should be called even if the value is missing - :param check_fields: whether to check that the fields actually exist on the model - :param allow_reuse: whether to track and raise an error if another validator refers to the decorated function - """ - if not fields: - raise ConfigError('validator with no fields specified') - elif isinstance(fields[0], FunctionType): - raise ConfigError( - "validators should be used with fields and keyword arguments, not bare. " # noqa: Q000 - "E.g. usage should be `@validator('', ...)`" - ) - elif not all(isinstance(field, str) for field in fields): - raise ConfigError( - "validator fields should be passed as separate string args. " # noqa: Q000 - "E.g. usage should be `@validator('', '', ...)`" - ) - - if whole is not None: - warnings.warn( - 'The "whole" keyword argument is deprecated, use "each_item" (inverse meaning, default False) instead', - DeprecationWarning, - ) - assert each_item is False, '"each_item" and "whole" conflict, remove "whole"' - each_item = not whole - - def dec(f: AnyCallable) -> 'AnyClassMethod': - f_cls = _prepare_validator(f, allow_reuse) - setattr( - f_cls, - VALIDATOR_CONFIG_KEY, - ( - fields, - Validator(func=f_cls.__func__, pre=pre, each_item=each_item, always=always, check_fields=check_fields), - ), - ) - return f_cls - - return dec - - -@overload -def root_validator(_func: AnyCallable) -> 'AnyClassMethod': - ... - - -@overload -def root_validator( - *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False -) -> Callable[[AnyCallable], 'AnyClassMethod']: - ... - - -def root_validator( - _func: Optional[AnyCallable] = None, *, pre: bool = False, allow_reuse: bool = False, skip_on_failure: bool = False -) -> Union['AnyClassMethod', Callable[[AnyCallable], 'AnyClassMethod']]: - """ - Decorate methods on a model indicating that they should be used to validate (and perhaps modify) data either - before or after standard model parsing/validation is performed. - """ - if _func: - f_cls = _prepare_validator(_func, allow_reuse) - setattr( - f_cls, ROOT_VALIDATOR_CONFIG_KEY, Validator(func=f_cls.__func__, pre=pre, skip_on_failure=skip_on_failure) - ) - return f_cls - - def dec(f: AnyCallable) -> 'AnyClassMethod': - f_cls = _prepare_validator(f, allow_reuse) - setattr( - f_cls, ROOT_VALIDATOR_CONFIG_KEY, Validator(func=f_cls.__func__, pre=pre, skip_on_failure=skip_on_failure) - ) - return f_cls - - return dec - - -def _prepare_validator(function: AnyCallable, allow_reuse: bool) -> 'AnyClassMethod': - """ - Avoid validators with duplicated names since without this, validators can be overwritten silently - which generally isn't the intended behaviour, don't run in ipython (see #312) or if allow_reuse is False. - """ - f_cls = function if isinstance(function, classmethod) else classmethod(function) - if not in_ipython() and not allow_reuse: - ref = ( - getattr(f_cls.__func__, '__module__', '') - + '.' - + getattr(f_cls.__func__, '__qualname__', f'') - ) - if ref in _FUNCS: - raise ConfigError(f'duplicate validator function "{ref}"; if this is intended, set `allow_reuse=True`') - _FUNCS.add(ref) - return f_cls - - -class ValidatorGroup: - def __init__(self, validators: 'ValidatorListDict') -> None: - self.validators = validators - self.used_validators = {'*'} - - def get_validators(self, name: str) -> Optional[Dict[str, Validator]]: - self.used_validators.add(name) - validators = self.validators.get(name, []) - if name != ROOT_KEY: - validators += self.validators.get('*', []) - if validators: - return {getattr(v.func, '__name__', f''): v for v in validators} - else: - return None - - def check_for_unused(self) -> None: - unused_validators = set( - chain.from_iterable( - ( - getattr(v.func, '__name__', f'') - for v in self.validators[f] - if v.check_fields - ) - for f in (self.validators.keys() - self.used_validators) - ) - ) - if unused_validators: - fn = ', '.join(unused_validators) - raise ConfigError( - f"Validators defined with incorrect fields: {fn} " # noqa: Q000 - f"(use check_fields=False if you're inheriting from the model and intended this)" - ) - - -def extract_validators(namespace: Dict[str, Any]) -> Dict[str, List[Validator]]: - validators: Dict[str, List[Validator]] = {} - for var_name, value in namespace.items(): - validator_config = getattr(value, VALIDATOR_CONFIG_KEY, None) - if validator_config: - fields, v = validator_config - for field in fields: - if field in validators: - validators[field].append(v) - else: - validators[field] = [v] - return validators - - -def extract_root_validators(namespace: Dict[str, Any]) -> Tuple[List[AnyCallable], List[Tuple[bool, AnyCallable]]]: - from inspect import signature - - pre_validators: List[AnyCallable] = [] - post_validators: List[Tuple[bool, AnyCallable]] = [] - for name, value in namespace.items(): - validator_config: Optional[Validator] = getattr(value, ROOT_VALIDATOR_CONFIG_KEY, None) - if validator_config: - sig = signature(validator_config.func) - args = list(sig.parameters.keys()) - if args[0] == 'self': - raise ConfigError( - f'Invalid signature for root validator {name}: {sig}, "self" not permitted as first argument, ' - f'should be: (cls, values).' - ) - if len(args) != 2: - raise ConfigError(f'Invalid signature for root validator {name}: {sig}, should be: (cls, values).') - # check function signature - if validator_config.pre: - pre_validators.append(validator_config.func) - else: - post_validators.append((validator_config.skip_on_failure, validator_config.func)) - return pre_validators, post_validators - - -def inherit_validators(base_validators: 'ValidatorListDict', validators: 'ValidatorListDict') -> 'ValidatorListDict': - for field, field_validators in base_validators.items(): - if field not in validators: - validators[field] = [] - validators[field] += field_validators - return validators - - -def make_generic_validator(validator: AnyCallable) -> 'ValidatorCallable': - """ - Make a generic function which calls a validator with the right arguments. - - Unfortunately other approaches (eg. return a partial of a function that builds the arguments) is slow, - hence this laborious way of doing things. - - It's done like this so validators don't all need **kwargs in their signature, eg. any combination of - the arguments "values", "fields" and/or "config" are permitted. - """ - from inspect import signature - - if not isinstance(validator, (partial, partialmethod)): - # This should be the default case, so overhead is reduced - sig = signature(validator) - args = list(sig.parameters.keys()) - else: - # Fix the generated argument lists of partial methods - sig = signature(validator.func) - args = [ - k - for k in signature(validator.func).parameters.keys() - if k not in validator.args | validator.keywords.keys() - ] - - first_arg = args.pop(0) - if first_arg == 'self': - raise ConfigError( - f'Invalid signature for validator {validator}: {sig}, "self" not permitted as first argument, ' - f'should be: (cls, value, values, config, field), "values", "config" and "field" are all optional.' - ) - elif first_arg == 'cls': - # assume the second argument is value - return wraps(validator)(_generic_validator_cls(validator, sig, set(args[1:]))) - else: - # assume the first argument was value which has already been removed - return wraps(validator)(_generic_validator_basic(validator, sig, set(args))) - - -def prep_validators(v_funcs: Iterable[AnyCallable]) -> 'ValidatorsList': - return [make_generic_validator(f) for f in v_funcs if f] - - -all_kwargs = {'values', 'field', 'config'} - - -def _generic_validator_cls(validator: AnyCallable, sig: 'Signature', args: Set[str]) -> 'ValidatorCallable': - # assume the first argument is value - has_kwargs = False - if 'kwargs' in args: - has_kwargs = True - args -= {'kwargs'} - - if not args.issubset(all_kwargs): - raise ConfigError( - f'Invalid signature for validator {validator}: {sig}, should be: ' - f'(cls, value, values, config, field), "values", "config" and "field" are all optional.' - ) - - if has_kwargs: - return lambda cls, v, values, field, config: validator(cls, v, values=values, field=field, config=config) - elif args == set(): - return lambda cls, v, values, field, config: validator(cls, v) - elif args == {'values'}: - return lambda cls, v, values, field, config: validator(cls, v, values=values) - elif args == {'field'}: - return lambda cls, v, values, field, config: validator(cls, v, field=field) - elif args == {'config'}: - return lambda cls, v, values, field, config: validator(cls, v, config=config) - elif args == {'values', 'field'}: - return lambda cls, v, values, field, config: validator(cls, v, values=values, field=field) - elif args == {'values', 'config'}: - return lambda cls, v, values, field, config: validator(cls, v, values=values, config=config) - elif args == {'field', 'config'}: - return lambda cls, v, values, field, config: validator(cls, v, field=field, config=config) - else: - # args == {'values', 'field', 'config'} - return lambda cls, v, values, field, config: validator(cls, v, values=values, field=field, config=config) - - -def _generic_validator_basic(validator: AnyCallable, sig: 'Signature', args: Set[str]) -> 'ValidatorCallable': - has_kwargs = False - if 'kwargs' in args: - has_kwargs = True - args -= {'kwargs'} - - if not args.issubset(all_kwargs): - raise ConfigError( - f'Invalid signature for validator {validator}: {sig}, should be: ' - f'(value, values, config, field), "values", "config" and "field" are all optional.' - ) - - if has_kwargs: - return lambda cls, v, values, field, config: validator(v, values=values, field=field, config=config) - elif args == set(): - return lambda cls, v, values, field, config: validator(v) - elif args == {'values'}: - return lambda cls, v, values, field, config: validator(v, values=values) - elif args == {'field'}: - return lambda cls, v, values, field, config: validator(v, field=field) - elif args == {'config'}: - return lambda cls, v, values, field, config: validator(v, config=config) - elif args == {'values', 'field'}: - return lambda cls, v, values, field, config: validator(v, values=values, field=field) - elif args == {'values', 'config'}: - return lambda cls, v, values, field, config: validator(v, values=values, config=config) - elif args == {'field', 'config'}: - return lambda cls, v, values, field, config: validator(v, field=field, config=config) - else: - # args == {'values', 'field', 'config'} - return lambda cls, v, values, field, config: validator(v, values=values, field=field, config=config) - - -def gather_all_validators(type_: 'ModelOrDc') -> Dict[str, 'AnyClassMethod']: - all_attributes = ChainMap(*[cls.__dict__ for cls in type_.__mro__]) # type: ignore[arg-type,var-annotated] - return { - k: v - for k, v in all_attributes.items() - if hasattr(v, VALIDATOR_CONFIG_KEY) or hasattr(v, ROOT_VALIDATOR_CONFIG_KEY) - } diff --git a/functions/python/data-processor/pydantic/v1/color.py b/functions/python/data-processor/pydantic/v1/color.py deleted file mode 100644 index b0bbf78f4..000000000 --- a/functions/python/data-processor/pydantic/v1/color.py +++ /dev/null @@ -1,494 +0,0 @@ -""" -Color definitions are used as per CSS3 specification: -http://www.w3.org/TR/css3-color/#svg-color - -A few colors have multiple names referring to the sames colors, eg. `grey` and `gray` or `aqua` and `cyan`. - -In these cases the LAST color when sorted alphabetically takes preferences, -eg. Color((0, 255, 255)).as_named() == 'cyan' because "cyan" comes after "aqua". -""" -import math -import re -from colorsys import hls_to_rgb, rgb_to_hls -from typing import TYPE_CHECKING, Any, Dict, Optional, Tuple, Union, cast - -from pydantic.v1.errors import ColorError -from pydantic.v1.utils import Representation, almost_equal_floats - -if TYPE_CHECKING: - from pydantic.v1.typing import CallableGenerator, ReprArgs - -ColorTuple = Union[Tuple[int, int, int], Tuple[int, int, int, float]] -ColorType = Union[ColorTuple, str] -HslColorTuple = Union[Tuple[float, float, float], Tuple[float, float, float, float]] - - -class RGBA: - """ - Internal use only as a representation of a color. - """ - - __slots__ = 'r', 'g', 'b', 'alpha', '_tuple' - - def __init__(self, r: float, g: float, b: float, alpha: Optional[float]): - self.r = r - self.g = g - self.b = b - self.alpha = alpha - - self._tuple: Tuple[float, float, float, Optional[float]] = (r, g, b, alpha) - - def __getitem__(self, item: Any) -> Any: - return self._tuple[item] - - -# these are not compiled here to avoid import slowdown, they'll be compiled the first time they're used, then cached -r_hex_short = r'\s*(?:#|0x)?([0-9a-f])([0-9a-f])([0-9a-f])([0-9a-f])?\s*' -r_hex_long = r'\s*(?:#|0x)?([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})([0-9a-f]{2})?\s*' -_r_255 = r'(\d{1,3}(?:\.\d+)?)' -_r_comma = r'\s*,\s*' -r_rgb = fr'\s*rgb\(\s*{_r_255}{_r_comma}{_r_255}{_r_comma}{_r_255}\)\s*' -_r_alpha = r'(\d(?:\.\d+)?|\.\d+|\d{1,2}%)' -r_rgba = fr'\s*rgba\(\s*{_r_255}{_r_comma}{_r_255}{_r_comma}{_r_255}{_r_comma}{_r_alpha}\s*\)\s*' -_r_h = r'(-?\d+(?:\.\d+)?|-?\.\d+)(deg|rad|turn)?' -_r_sl = r'(\d{1,3}(?:\.\d+)?)%' -r_hsl = fr'\s*hsl\(\s*{_r_h}{_r_comma}{_r_sl}{_r_comma}{_r_sl}\s*\)\s*' -r_hsla = fr'\s*hsl\(\s*{_r_h}{_r_comma}{_r_sl}{_r_comma}{_r_sl}{_r_comma}{_r_alpha}\s*\)\s*' - -# colors where the two hex characters are the same, if all colors match this the short version of hex colors can be used -repeat_colors = {int(c * 2, 16) for c in '0123456789abcdef'} -rads = 2 * math.pi - - -class Color(Representation): - __slots__ = '_original', '_rgba' - - def __init__(self, value: ColorType) -> None: - self._rgba: RGBA - self._original: ColorType - if isinstance(value, (tuple, list)): - self._rgba = parse_tuple(value) - elif isinstance(value, str): - self._rgba = parse_str(value) - elif isinstance(value, Color): - self._rgba = value._rgba - value = value._original - else: - raise ColorError(reason='value must be a tuple, list or string') - - # if we've got here value must be a valid color - self._original = value - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='color') - - def original(self) -> ColorType: - """ - Original value passed to Color - """ - return self._original - - def as_named(self, *, fallback: bool = False) -> str: - if self._rgba.alpha is None: - rgb = cast(Tuple[int, int, int], self.as_rgb_tuple()) - try: - return COLORS_BY_VALUE[rgb] - except KeyError as e: - if fallback: - return self.as_hex() - else: - raise ValueError('no named color found, use fallback=True, as_hex() or as_rgb()') from e - else: - return self.as_hex() - - def as_hex(self) -> str: - """ - Hex string representing the color can be 3, 4, 6 or 8 characters depending on whether the string - a "short" representation of the color is possible and whether there's an alpha channel. - """ - values = [float_to_255(c) for c in self._rgba[:3]] - if self._rgba.alpha is not None: - values.append(float_to_255(self._rgba.alpha)) - - as_hex = ''.join(f'{v:02x}' for v in values) - if all(c in repeat_colors for c in values): - as_hex = ''.join(as_hex[c] for c in range(0, len(as_hex), 2)) - return '#' + as_hex - - def as_rgb(self) -> str: - """ - Color as an rgb(, , ) or rgba(, , , ) string. - """ - if self._rgba.alpha is None: - return f'rgb({float_to_255(self._rgba.r)}, {float_to_255(self._rgba.g)}, {float_to_255(self._rgba.b)})' - else: - return ( - f'rgba({float_to_255(self._rgba.r)}, {float_to_255(self._rgba.g)}, {float_to_255(self._rgba.b)}, ' - f'{round(self._alpha_float(), 2)})' - ) - - def as_rgb_tuple(self, *, alpha: Optional[bool] = None) -> ColorTuple: - """ - Color as an RGB or RGBA tuple; red, green and blue are in the range 0 to 255, alpha if included is - in the range 0 to 1. - - :param alpha: whether to include the alpha channel, options are - None - (default) include alpha only if it's set (e.g. not None) - True - always include alpha, - False - always omit alpha, - """ - r, g, b = (float_to_255(c) for c in self._rgba[:3]) - if alpha is None: - if self._rgba.alpha is None: - return r, g, b - else: - return r, g, b, self._alpha_float() - elif alpha: - return r, g, b, self._alpha_float() - else: - # alpha is False - return r, g, b - - def as_hsl(self) -> str: - """ - Color as an hsl(, , ) or hsl(, , , ) string. - """ - if self._rgba.alpha is None: - h, s, li = self.as_hsl_tuple(alpha=False) # type: ignore - return f'hsl({h * 360:0.0f}, {s:0.0%}, {li:0.0%})' - else: - h, s, li, a = self.as_hsl_tuple(alpha=True) # type: ignore - return f'hsl({h * 360:0.0f}, {s:0.0%}, {li:0.0%}, {round(a, 2)})' - - def as_hsl_tuple(self, *, alpha: Optional[bool] = None) -> HslColorTuple: - """ - Color as an HSL or HSLA tuple, e.g. hue, saturation, lightness and optionally alpha; all elements are in - the range 0 to 1. - - NOTE: this is HSL as used in HTML and most other places, not HLS as used in python's colorsys. - - :param alpha: whether to include the alpha channel, options are - None - (default) include alpha only if it's set (e.g. not None) - True - always include alpha, - False - always omit alpha, - """ - h, l, s = rgb_to_hls(self._rgba.r, self._rgba.g, self._rgba.b) - if alpha is None: - if self._rgba.alpha is None: - return h, s, l - else: - return h, s, l, self._alpha_float() - if alpha: - return h, s, l, self._alpha_float() - else: - # alpha is False - return h, s, l - - def _alpha_float(self) -> float: - return 1 if self._rgba.alpha is None else self._rgba.alpha - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls - - def __str__(self) -> str: - return self.as_named(fallback=True) - - def __repr_args__(self) -> 'ReprArgs': - return [(None, self.as_named(fallback=True))] + [('rgb', self.as_rgb_tuple())] # type: ignore - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Color) and self.as_rgb_tuple() == other.as_rgb_tuple() - - def __hash__(self) -> int: - return hash(self.as_rgb_tuple()) - - -def parse_tuple(value: Tuple[Any, ...]) -> RGBA: - """ - Parse a tuple or list as a color. - """ - if len(value) == 3: - r, g, b = (parse_color_value(v) for v in value) - return RGBA(r, g, b, None) - elif len(value) == 4: - r, g, b = (parse_color_value(v) for v in value[:3]) - return RGBA(r, g, b, parse_float_alpha(value[3])) - else: - raise ColorError(reason='tuples must have length 3 or 4') - - -def parse_str(value: str) -> RGBA: - """ - Parse a string to an RGBA tuple, trying the following formats (in this order): - * named color, see COLORS_BY_NAME below - * hex short eg. `fff` (prefix can be `#`, `0x` or nothing) - * hex long eg. `ffffff` (prefix can be `#`, `0x` or nothing) - * `rgb(, , ) ` - * `rgba(, , , )` - """ - value_lower = value.lower() - try: - r, g, b = COLORS_BY_NAME[value_lower] - except KeyError: - pass - else: - return ints_to_rgba(r, g, b, None) - - m = re.fullmatch(r_hex_short, value_lower) - if m: - *rgb, a = m.groups() - r, g, b = (int(v * 2, 16) for v in rgb) - if a: - alpha: Optional[float] = int(a * 2, 16) / 255 - else: - alpha = None - return ints_to_rgba(r, g, b, alpha) - - m = re.fullmatch(r_hex_long, value_lower) - if m: - *rgb, a = m.groups() - r, g, b = (int(v, 16) for v in rgb) - if a: - alpha = int(a, 16) / 255 - else: - alpha = None - return ints_to_rgba(r, g, b, alpha) - - m = re.fullmatch(r_rgb, value_lower) - if m: - return ints_to_rgba(*m.groups(), None) # type: ignore - - m = re.fullmatch(r_rgba, value_lower) - if m: - return ints_to_rgba(*m.groups()) # type: ignore - - m = re.fullmatch(r_hsl, value_lower) - if m: - h, h_units, s, l_ = m.groups() - return parse_hsl(h, h_units, s, l_) - - m = re.fullmatch(r_hsla, value_lower) - if m: - h, h_units, s, l_, a = m.groups() - return parse_hsl(h, h_units, s, l_, parse_float_alpha(a)) - - raise ColorError(reason='string not recognised as a valid color') - - -def ints_to_rgba(r: Union[int, str], g: Union[int, str], b: Union[int, str], alpha: Optional[float]) -> RGBA: - return RGBA(parse_color_value(r), parse_color_value(g), parse_color_value(b), parse_float_alpha(alpha)) - - -def parse_color_value(value: Union[int, str], max_val: int = 255) -> float: - """ - Parse a value checking it's a valid int in the range 0 to max_val and divide by max_val to give a number - in the range 0 to 1 - """ - try: - color = float(value) - except ValueError: - raise ColorError(reason='color values must be a valid number') - if 0 <= color <= max_val: - return color / max_val - else: - raise ColorError(reason=f'color values must be in the range 0 to {max_val}') - - -def parse_float_alpha(value: Union[None, str, float, int]) -> Optional[float]: - """ - Parse a value checking it's a valid float in the range 0 to 1 - """ - if value is None: - return None - try: - if isinstance(value, str) and value.endswith('%'): - alpha = float(value[:-1]) / 100 - else: - alpha = float(value) - except ValueError: - raise ColorError(reason='alpha values must be a valid float') - - if almost_equal_floats(alpha, 1): - return None - elif 0 <= alpha <= 1: - return alpha - else: - raise ColorError(reason='alpha values must be in the range 0 to 1') - - -def parse_hsl(h: str, h_units: str, sat: str, light: str, alpha: Optional[float] = None) -> RGBA: - """ - Parse raw hue, saturation, lightness and alpha values and convert to RGBA. - """ - s_value, l_value = parse_color_value(sat, 100), parse_color_value(light, 100) - - h_value = float(h) - if h_units in {None, 'deg'}: - h_value = h_value % 360 / 360 - elif h_units == 'rad': - h_value = h_value % rads / rads - else: - # turns - h_value = h_value % 1 - - r, g, b = hls_to_rgb(h_value, l_value, s_value) - return RGBA(r, g, b, alpha) - - -def float_to_255(c: float) -> int: - return int(round(c * 255)) - - -COLORS_BY_NAME = { - 'aliceblue': (240, 248, 255), - 'antiquewhite': (250, 235, 215), - 'aqua': (0, 255, 255), - 'aquamarine': (127, 255, 212), - 'azure': (240, 255, 255), - 'beige': (245, 245, 220), - 'bisque': (255, 228, 196), - 'black': (0, 0, 0), - 'blanchedalmond': (255, 235, 205), - 'blue': (0, 0, 255), - 'blueviolet': (138, 43, 226), - 'brown': (165, 42, 42), - 'burlywood': (222, 184, 135), - 'cadetblue': (95, 158, 160), - 'chartreuse': (127, 255, 0), - 'chocolate': (210, 105, 30), - 'coral': (255, 127, 80), - 'cornflowerblue': (100, 149, 237), - 'cornsilk': (255, 248, 220), - 'crimson': (220, 20, 60), - 'cyan': (0, 255, 255), - 'darkblue': (0, 0, 139), - 'darkcyan': (0, 139, 139), - 'darkgoldenrod': (184, 134, 11), - 'darkgray': (169, 169, 169), - 'darkgreen': (0, 100, 0), - 'darkgrey': (169, 169, 169), - 'darkkhaki': (189, 183, 107), - 'darkmagenta': (139, 0, 139), - 'darkolivegreen': (85, 107, 47), - 'darkorange': (255, 140, 0), - 'darkorchid': (153, 50, 204), - 'darkred': (139, 0, 0), - 'darksalmon': (233, 150, 122), - 'darkseagreen': (143, 188, 143), - 'darkslateblue': (72, 61, 139), - 'darkslategray': (47, 79, 79), - 'darkslategrey': (47, 79, 79), - 'darkturquoise': (0, 206, 209), - 'darkviolet': (148, 0, 211), - 'deeppink': (255, 20, 147), - 'deepskyblue': (0, 191, 255), - 'dimgray': (105, 105, 105), - 'dimgrey': (105, 105, 105), - 'dodgerblue': (30, 144, 255), - 'firebrick': (178, 34, 34), - 'floralwhite': (255, 250, 240), - 'forestgreen': (34, 139, 34), - 'fuchsia': (255, 0, 255), - 'gainsboro': (220, 220, 220), - 'ghostwhite': (248, 248, 255), - 'gold': (255, 215, 0), - 'goldenrod': (218, 165, 32), - 'gray': (128, 128, 128), - 'green': (0, 128, 0), - 'greenyellow': (173, 255, 47), - 'grey': (128, 128, 128), - 'honeydew': (240, 255, 240), - 'hotpink': (255, 105, 180), - 'indianred': (205, 92, 92), - 'indigo': (75, 0, 130), - 'ivory': (255, 255, 240), - 'khaki': (240, 230, 140), - 'lavender': (230, 230, 250), - 'lavenderblush': (255, 240, 245), - 'lawngreen': (124, 252, 0), - 'lemonchiffon': (255, 250, 205), - 'lightblue': (173, 216, 230), - 'lightcoral': (240, 128, 128), - 'lightcyan': (224, 255, 255), - 'lightgoldenrodyellow': (250, 250, 210), - 'lightgray': (211, 211, 211), - 'lightgreen': (144, 238, 144), - 'lightgrey': (211, 211, 211), - 'lightpink': (255, 182, 193), - 'lightsalmon': (255, 160, 122), - 'lightseagreen': (32, 178, 170), - 'lightskyblue': (135, 206, 250), - 'lightslategray': (119, 136, 153), - 'lightslategrey': (119, 136, 153), - 'lightsteelblue': (176, 196, 222), - 'lightyellow': (255, 255, 224), - 'lime': (0, 255, 0), - 'limegreen': (50, 205, 50), - 'linen': (250, 240, 230), - 'magenta': (255, 0, 255), - 'maroon': (128, 0, 0), - 'mediumaquamarine': (102, 205, 170), - 'mediumblue': (0, 0, 205), - 'mediumorchid': (186, 85, 211), - 'mediumpurple': (147, 112, 219), - 'mediumseagreen': (60, 179, 113), - 'mediumslateblue': (123, 104, 238), - 'mediumspringgreen': (0, 250, 154), - 'mediumturquoise': (72, 209, 204), - 'mediumvioletred': (199, 21, 133), - 'midnightblue': (25, 25, 112), - 'mintcream': (245, 255, 250), - 'mistyrose': (255, 228, 225), - 'moccasin': (255, 228, 181), - 'navajowhite': (255, 222, 173), - 'navy': (0, 0, 128), - 'oldlace': (253, 245, 230), - 'olive': (128, 128, 0), - 'olivedrab': (107, 142, 35), - 'orange': (255, 165, 0), - 'orangered': (255, 69, 0), - 'orchid': (218, 112, 214), - 'palegoldenrod': (238, 232, 170), - 'palegreen': (152, 251, 152), - 'paleturquoise': (175, 238, 238), - 'palevioletred': (219, 112, 147), - 'papayawhip': (255, 239, 213), - 'peachpuff': (255, 218, 185), - 'peru': (205, 133, 63), - 'pink': (255, 192, 203), - 'plum': (221, 160, 221), - 'powderblue': (176, 224, 230), - 'purple': (128, 0, 128), - 'red': (255, 0, 0), - 'rosybrown': (188, 143, 143), - 'royalblue': (65, 105, 225), - 'saddlebrown': (139, 69, 19), - 'salmon': (250, 128, 114), - 'sandybrown': (244, 164, 96), - 'seagreen': (46, 139, 87), - 'seashell': (255, 245, 238), - 'sienna': (160, 82, 45), - 'silver': (192, 192, 192), - 'skyblue': (135, 206, 235), - 'slateblue': (106, 90, 205), - 'slategray': (112, 128, 144), - 'slategrey': (112, 128, 144), - 'snow': (255, 250, 250), - 'springgreen': (0, 255, 127), - 'steelblue': (70, 130, 180), - 'tan': (210, 180, 140), - 'teal': (0, 128, 128), - 'thistle': (216, 191, 216), - 'tomato': (255, 99, 71), - 'turquoise': (64, 224, 208), - 'violet': (238, 130, 238), - 'wheat': (245, 222, 179), - 'white': (255, 255, 255), - 'whitesmoke': (245, 245, 245), - 'yellow': (255, 255, 0), - 'yellowgreen': (154, 205, 50), -} - -COLORS_BY_VALUE = {v: k for k, v in COLORS_BY_NAME.items()} diff --git a/functions/python/data-processor/pydantic/v1/config.py b/functions/python/data-processor/pydantic/v1/config.py deleted file mode 100644 index 18f7c999b..000000000 --- a/functions/python/data-processor/pydantic/v1/config.py +++ /dev/null @@ -1,191 +0,0 @@ -import json -from enum import Enum -from typing import TYPE_CHECKING, Any, Callable, Dict, ForwardRef, Optional, Tuple, Type, Union - -from typing_extensions import Literal, Protocol - -from pydantic.v1.typing import AnyArgTCallable, AnyCallable -from pydantic.v1.utils import GetterDict -from pydantic.v1.version import compiled - -if TYPE_CHECKING: - from typing import overload - - from pydantic.v1.fields import ModelField - from pydantic.v1.main import BaseModel - - ConfigType = Type['BaseConfig'] - - class SchemaExtraCallable(Protocol): - @overload - def __call__(self, schema: Dict[str, Any]) -> None: - pass - - @overload - def __call__(self, schema: Dict[str, Any], model_class: Type[BaseModel]) -> None: - pass - -else: - SchemaExtraCallable = Callable[..., None] - -__all__ = 'BaseConfig', 'ConfigDict', 'get_config', 'Extra', 'inherit_config', 'prepare_config' - - -class Extra(str, Enum): - allow = 'allow' - ignore = 'ignore' - forbid = 'forbid' - - -# https://github.com/cython/cython/issues/4003 -# Fixed in Cython 3 and Pydantic v1 won't support Cython 3. -# Pydantic v2 doesn't depend on Cython at all. -if not compiled: - from typing_extensions import TypedDict - - class ConfigDict(TypedDict, total=False): - title: Optional[str] - anystr_lower: bool - anystr_strip_whitespace: bool - min_anystr_length: int - max_anystr_length: Optional[int] - validate_all: bool - extra: Extra - allow_mutation: bool - frozen: bool - allow_population_by_field_name: bool - use_enum_values: bool - fields: Dict[str, Union[str, Dict[str, str]]] - validate_assignment: bool - error_msg_templates: Dict[str, str] - arbitrary_types_allowed: bool - orm_mode: bool - getter_dict: Type[GetterDict] - alias_generator: Optional[Callable[[str], str]] - keep_untouched: Tuple[type, ...] - schema_extra: Union[Dict[str, object], 'SchemaExtraCallable'] - json_loads: Callable[[str], object] - json_dumps: AnyArgTCallable[str] - json_encoders: Dict[Type[object], AnyCallable] - underscore_attrs_are_private: bool - allow_inf_nan: bool - copy_on_model_validation: Literal['none', 'deep', 'shallow'] - # whether dataclass `__post_init__` should be run after validation - post_init_call: Literal['before_validation', 'after_validation'] - -else: - ConfigDict = dict # type: ignore - - -class BaseConfig: - title: Optional[str] = None - anystr_lower: bool = False - anystr_upper: bool = False - anystr_strip_whitespace: bool = False - min_anystr_length: int = 0 - max_anystr_length: Optional[int] = None - validate_all: bool = False - extra: Extra = Extra.ignore - allow_mutation: bool = True - frozen: bool = False - allow_population_by_field_name: bool = False - use_enum_values: bool = False - fields: Dict[str, Union[str, Dict[str, str]]] = {} - validate_assignment: bool = False - error_msg_templates: Dict[str, str] = {} - arbitrary_types_allowed: bool = False - orm_mode: bool = False - getter_dict: Type[GetterDict] = GetterDict - alias_generator: Optional[Callable[[str], str]] = None - keep_untouched: Tuple[type, ...] = () - schema_extra: Union[Dict[str, Any], 'SchemaExtraCallable'] = {} - json_loads: Callable[[str], Any] = json.loads - json_dumps: Callable[..., str] = json.dumps - json_encoders: Dict[Union[Type[Any], str, ForwardRef], AnyCallable] = {} - underscore_attrs_are_private: bool = False - allow_inf_nan: bool = True - - # whether inherited models as fields should be reconstructed as base model, - # and whether such a copy should be shallow or deep - copy_on_model_validation: Literal['none', 'deep', 'shallow'] = 'shallow' - - # whether `Union` should check all allowed types before even trying to coerce - smart_union: bool = False - # whether dataclass `__post_init__` should be run before or after validation - post_init_call: Literal['before_validation', 'after_validation'] = 'before_validation' - - @classmethod - def get_field_info(cls, name: str) -> Dict[str, Any]: - """ - Get properties of FieldInfo from the `fields` property of the config class. - """ - - fields_value = cls.fields.get(name) - - if isinstance(fields_value, str): - field_info: Dict[str, Any] = {'alias': fields_value} - elif isinstance(fields_value, dict): - field_info = fields_value - else: - field_info = {} - - if 'alias' in field_info: - field_info.setdefault('alias_priority', 2) - - if field_info.get('alias_priority', 0) <= 1 and cls.alias_generator: - alias = cls.alias_generator(name) - if not isinstance(alias, str): - raise TypeError(f'Config.alias_generator must return str, not {alias.__class__}') - field_info.update(alias=alias, alias_priority=1) - return field_info - - @classmethod - def prepare_field(cls, field: 'ModelField') -> None: - """ - Optional hook to check or modify fields during model creation. - """ - pass - - -def get_config(config: Union[ConfigDict, Type[object], None]) -> Type[BaseConfig]: - if config is None: - return BaseConfig - - else: - config_dict = ( - config - if isinstance(config, dict) - else {k: getattr(config, k) for k in dir(config) if not k.startswith('__')} - ) - - class Config(BaseConfig): - ... - - for k, v in config_dict.items(): - setattr(Config, k, v) - return Config - - -def inherit_config(self_config: 'ConfigType', parent_config: 'ConfigType', **namespace: Any) -> 'ConfigType': - if not self_config: - base_classes: Tuple['ConfigType', ...] = (parent_config,) - elif self_config == parent_config: - base_classes = (self_config,) - else: - base_classes = self_config, parent_config - - namespace['json_encoders'] = { - **getattr(parent_config, 'json_encoders', {}), - **getattr(self_config, 'json_encoders', {}), - **namespace.get('json_encoders', {}), - } - - return type('Config', base_classes, namespace) - - -def prepare_config(config: Type[BaseConfig], cls_name: str) -> None: - if not isinstance(config.extra, Extra): - try: - config.extra = Extra(config.extra) - except ValueError: - raise ValueError(f'"{cls_name}": {config.extra} is not a valid value for "extra"') diff --git a/functions/python/data-processor/pydantic/v1/dataclasses.py b/functions/python/data-processor/pydantic/v1/dataclasses.py deleted file mode 100644 index bd1670291..000000000 --- a/functions/python/data-processor/pydantic/v1/dataclasses.py +++ /dev/null @@ -1,500 +0,0 @@ -""" -The main purpose is to enhance stdlib dataclasses by adding validation -A pydantic dataclass can be generated from scratch or from a stdlib one. - -Behind the scene, a pydantic dataclass is just like a regular one on which we attach -a `BaseModel` and magic methods to trigger the validation of the data. -`__init__` and `__post_init__` are hence overridden and have extra logic to be -able to validate input data. - -When a pydantic dataclass is generated from scratch, it's just a plain dataclass -with validation triggered at initialization - -The tricky part if for stdlib dataclasses that are converted after into pydantic ones e.g. - -```py -@dataclasses.dataclass -class M: - x: int - -ValidatedM = pydantic.dataclasses.dataclass(M) -``` - -We indeed still want to support equality, hashing, repr, ... as if it was the stdlib one! - -```py -assert isinstance(ValidatedM(x=1), M) -assert ValidatedM(x=1) == M(x=1) -``` - -This means we **don't want to create a new dataclass that inherits from it** -The trick is to create a wrapper around `M` that will act as a proxy to trigger -validation without altering default `M` behaviour. -""" -import copy -import dataclasses -import sys -from contextlib import contextmanager -from functools import wraps - -try: - from functools import cached_property -except ImportError: - # cached_property available only for python3.8+ - pass - -from typing import TYPE_CHECKING, Any, Callable, ClassVar, Dict, Generator, Optional, Type, TypeVar, Union, overload - -from typing_extensions import dataclass_transform - -from pydantic.v1.class_validators import gather_all_validators -from pydantic.v1.config import BaseConfig, ConfigDict, Extra, get_config -from pydantic.v1.error_wrappers import ValidationError -from pydantic.v1.errors import DataclassTypeError -from pydantic.v1.fields import Field, FieldInfo, Required, Undefined -from pydantic.v1.main import create_model, validate_model -from pydantic.v1.utils import ClassAttribute - -if TYPE_CHECKING: - from pydantic.v1.main import BaseModel - from pydantic.v1.typing import CallableGenerator, NoArgAnyCallable - - DataclassT = TypeVar('DataclassT', bound='Dataclass') - - DataclassClassOrWrapper = Union[Type['Dataclass'], 'DataclassProxy'] - - class Dataclass: - # stdlib attributes - __dataclass_fields__: ClassVar[Dict[str, Any]] - __dataclass_params__: ClassVar[Any] # in reality `dataclasses._DataclassParams` - __post_init__: ClassVar[Callable[..., None]] - - # Added by pydantic - __pydantic_run_validation__: ClassVar[bool] - __post_init_post_parse__: ClassVar[Callable[..., None]] - __pydantic_initialised__: ClassVar[bool] - __pydantic_model__: ClassVar[Type[BaseModel]] - __pydantic_validate_values__: ClassVar[Callable[['Dataclass'], None]] - __pydantic_has_field_info_default__: ClassVar[bool] # whether a `pydantic.Field` is used as default value - - def __init__(self, *args: object, **kwargs: object) -> None: - pass - - @classmethod - def __get_validators__(cls: Type['Dataclass']) -> 'CallableGenerator': - pass - - @classmethod - def __validate__(cls: Type['DataclassT'], v: Any) -> 'DataclassT': - pass - - -__all__ = [ - 'dataclass', - 'set_validation', - 'create_pydantic_model_from_dataclass', - 'is_builtin_dataclass', - 'make_dataclass_validator', -] - -_T = TypeVar('_T') - -if sys.version_info >= (3, 10): - - @dataclass_transform(field_specifiers=(dataclasses.field, Field)) - @overload - def dataclass( - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Union[ConfigDict, Type[object], None] = None, - validate_on_init: Optional[bool] = None, - use_proxy: Optional[bool] = None, - kw_only: bool = ..., - ) -> Callable[[Type[_T]], 'DataclassClassOrWrapper']: - ... - - @dataclass_transform(field_specifiers=(dataclasses.field, Field)) - @overload - def dataclass( - _cls: Type[_T], - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Union[ConfigDict, Type[object], None] = None, - validate_on_init: Optional[bool] = None, - use_proxy: Optional[bool] = None, - kw_only: bool = ..., - ) -> 'DataclassClassOrWrapper': - ... - -else: - - @dataclass_transform(field_specifiers=(dataclasses.field, Field)) - @overload - def dataclass( - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Union[ConfigDict, Type[object], None] = None, - validate_on_init: Optional[bool] = None, - use_proxy: Optional[bool] = None, - ) -> Callable[[Type[_T]], 'DataclassClassOrWrapper']: - ... - - @dataclass_transform(field_specifiers=(dataclasses.field, Field)) - @overload - def dataclass( - _cls: Type[_T], - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Union[ConfigDict, Type[object], None] = None, - validate_on_init: Optional[bool] = None, - use_proxy: Optional[bool] = None, - ) -> 'DataclassClassOrWrapper': - ... - - -@dataclass_transform(field_specifiers=(dataclasses.field, Field)) -def dataclass( - _cls: Optional[Type[_T]] = None, - *, - init: bool = True, - repr: bool = True, - eq: bool = True, - order: bool = False, - unsafe_hash: bool = False, - frozen: bool = False, - config: Union[ConfigDict, Type[object], None] = None, - validate_on_init: Optional[bool] = None, - use_proxy: Optional[bool] = None, - kw_only: bool = False, -) -> Union[Callable[[Type[_T]], 'DataclassClassOrWrapper'], 'DataclassClassOrWrapper']: - """ - Like the python standard lib dataclasses but with type validation. - The result is either a pydantic dataclass that will validate input data - or a wrapper that will trigger validation around a stdlib dataclass - to avoid modifying it directly - """ - the_config = get_config(config) - - def wrap(cls: Type[Any]) -> 'DataclassClassOrWrapper': - should_use_proxy = ( - use_proxy - if use_proxy is not None - else ( - is_builtin_dataclass(cls) - and (cls.__bases__[0] is object or set(dir(cls)) == set(dir(cls.__bases__[0]))) - ) - ) - if should_use_proxy: - dc_cls_doc = '' - dc_cls = DataclassProxy(cls) - default_validate_on_init = False - else: - dc_cls_doc = cls.__doc__ or '' # needs to be done before generating dataclass - if sys.version_info >= (3, 10): - dc_cls = dataclasses.dataclass( - cls, - init=init, - repr=repr, - eq=eq, - order=order, - unsafe_hash=unsafe_hash, - frozen=frozen, - kw_only=kw_only, - ) - else: - dc_cls = dataclasses.dataclass( # type: ignore - cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen - ) - default_validate_on_init = True - - should_validate_on_init = default_validate_on_init if validate_on_init is None else validate_on_init - _add_pydantic_validation_attributes(cls, the_config, should_validate_on_init, dc_cls_doc) - dc_cls.__pydantic_model__.__try_update_forward_refs__(**{cls.__name__: cls}) - return dc_cls - - if _cls is None: - return wrap - - return wrap(_cls) - - -@contextmanager -def set_validation(cls: Type['DataclassT'], value: bool) -> Generator[Type['DataclassT'], None, None]: - original_run_validation = cls.__pydantic_run_validation__ - try: - cls.__pydantic_run_validation__ = value - yield cls - finally: - cls.__pydantic_run_validation__ = original_run_validation - - -class DataclassProxy: - __slots__ = '__dataclass__' - - def __init__(self, dc_cls: Type['Dataclass']) -> None: - object.__setattr__(self, '__dataclass__', dc_cls) - - def __call__(self, *args: Any, **kwargs: Any) -> Any: - with set_validation(self.__dataclass__, True): - return self.__dataclass__(*args, **kwargs) - - def __getattr__(self, name: str) -> Any: - return getattr(self.__dataclass__, name) - - def __setattr__(self, __name: str, __value: Any) -> None: - return setattr(self.__dataclass__, __name, __value) - - def __instancecheck__(self, instance: Any) -> bool: - return isinstance(instance, self.__dataclass__) - - def __copy__(self) -> 'DataclassProxy': - return DataclassProxy(copy.copy(self.__dataclass__)) - - def __deepcopy__(self, memo: Any) -> 'DataclassProxy': - return DataclassProxy(copy.deepcopy(self.__dataclass__, memo)) - - -def _add_pydantic_validation_attributes( # noqa: C901 (ignore complexity) - dc_cls: Type['Dataclass'], - config: Type[BaseConfig], - validate_on_init: bool, - dc_cls_doc: str, -) -> None: - """ - We need to replace the right method. If no `__post_init__` has been set in the stdlib dataclass - it won't even exist (code is generated on the fly by `dataclasses`) - By default, we run validation after `__init__` or `__post_init__` if defined - """ - init = dc_cls.__init__ - - @wraps(init) - def handle_extra_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - if config.extra == Extra.ignore: - init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) - - elif config.extra == Extra.allow: - for k, v in kwargs.items(): - self.__dict__.setdefault(k, v) - init(self, *args, **{k: v for k, v in kwargs.items() if k in self.__dataclass_fields__}) - - else: - init(self, *args, **kwargs) - - if hasattr(dc_cls, '__post_init__'): - try: - post_init = dc_cls.__post_init__.__wrapped__ # type: ignore[attr-defined] - except AttributeError: - post_init = dc_cls.__post_init__ - - @wraps(post_init) - def new_post_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - if config.post_init_call == 'before_validation': - post_init(self, *args, **kwargs) - - if self.__class__.__pydantic_run_validation__: - self.__pydantic_validate_values__() - if hasattr(self, '__post_init_post_parse__'): - self.__post_init_post_parse__(*args, **kwargs) - - if config.post_init_call == 'after_validation': - post_init(self, *args, **kwargs) - - setattr(dc_cls, '__init__', handle_extra_init) - setattr(dc_cls, '__post_init__', new_post_init) - - else: - - @wraps(init) - def new_init(self: 'Dataclass', *args: Any, **kwargs: Any) -> None: - handle_extra_init(self, *args, **kwargs) - - if self.__class__.__pydantic_run_validation__: - self.__pydantic_validate_values__() - - if hasattr(self, '__post_init_post_parse__'): - # We need to find again the initvars. To do that we use `__dataclass_fields__` instead of - # public method `dataclasses.fields` - - # get all initvars and their default values - initvars_and_values: Dict[str, Any] = {} - for i, f in enumerate(self.__class__.__dataclass_fields__.values()): - if f._field_type is dataclasses._FIELD_INITVAR: # type: ignore[attr-defined] - try: - # set arg value by default - initvars_and_values[f.name] = args[i] - except IndexError: - initvars_and_values[f.name] = kwargs.get(f.name, f.default) - - self.__post_init_post_parse__(**initvars_and_values) - - setattr(dc_cls, '__init__', new_init) - - setattr(dc_cls, '__pydantic_run_validation__', ClassAttribute('__pydantic_run_validation__', validate_on_init)) - setattr(dc_cls, '__pydantic_initialised__', False) - setattr(dc_cls, '__pydantic_model__', create_pydantic_model_from_dataclass(dc_cls, config, dc_cls_doc)) - setattr(dc_cls, '__pydantic_validate_values__', _dataclass_validate_values) - setattr(dc_cls, '__validate__', classmethod(_validate_dataclass)) - setattr(dc_cls, '__get_validators__', classmethod(_get_validators)) - - if dc_cls.__pydantic_model__.__config__.validate_assignment and not dc_cls.__dataclass_params__.frozen: - setattr(dc_cls, '__setattr__', _dataclass_validate_assignment_setattr) - - -def _get_validators(cls: 'DataclassClassOrWrapper') -> 'CallableGenerator': - yield cls.__validate__ - - -def _validate_dataclass(cls: Type['DataclassT'], v: Any) -> 'DataclassT': - with set_validation(cls, True): - if isinstance(v, cls): - v.__pydantic_validate_values__() - return v - elif isinstance(v, (list, tuple)): - return cls(*v) - elif isinstance(v, dict): - return cls(**v) - else: - raise DataclassTypeError(class_name=cls.__name__) - - -def create_pydantic_model_from_dataclass( - dc_cls: Type['Dataclass'], - config: Type[Any] = BaseConfig, - dc_cls_doc: Optional[str] = None, -) -> Type['BaseModel']: - field_definitions: Dict[str, Any] = {} - for field in dataclasses.fields(dc_cls): - default: Any = Undefined - default_factory: Optional['NoArgAnyCallable'] = None - field_info: FieldInfo - - if field.default is not dataclasses.MISSING: - default = field.default - elif field.default_factory is not dataclasses.MISSING: - default_factory = field.default_factory - else: - default = Required - - if isinstance(default, FieldInfo): - field_info = default - dc_cls.__pydantic_has_field_info_default__ = True - else: - field_info = Field(default=default, default_factory=default_factory, **field.metadata) - - field_definitions[field.name] = (field.type, field_info) - - validators = gather_all_validators(dc_cls) - model: Type['BaseModel'] = create_model( - dc_cls.__name__, - __config__=config, - __module__=dc_cls.__module__, - __validators__=validators, - __cls_kwargs__={'__resolve_forward_refs__': False}, - **field_definitions, - ) - model.__doc__ = dc_cls_doc if dc_cls_doc is not None else dc_cls.__doc__ or '' - return model - - -if sys.version_info >= (3, 8): - - def _is_field_cached_property(obj: 'Dataclass', k: str) -> bool: - return isinstance(getattr(type(obj), k, None), cached_property) - -else: - - def _is_field_cached_property(obj: 'Dataclass', k: str) -> bool: - return False - - -def _dataclass_validate_values(self: 'Dataclass') -> None: - # validation errors can occur if this function is called twice on an already initialised dataclass. - # for example if Extra.forbid is enabled, it would consider __pydantic_initialised__ an invalid extra property - if getattr(self, '__pydantic_initialised__'): - return - if getattr(self, '__pydantic_has_field_info_default__', False): - # We need to remove `FieldInfo` values since they are not valid as input - # It's ok to do that because they are obviously the default values! - input_data = { - k: v - for k, v in self.__dict__.items() - if not (isinstance(v, FieldInfo) or _is_field_cached_property(self, k)) - } - else: - input_data = {k: v for k, v in self.__dict__.items() if not _is_field_cached_property(self, k)} - d, _, validation_error = validate_model(self.__pydantic_model__, input_data, cls=self.__class__) - if validation_error: - raise validation_error - self.__dict__.update(d) - object.__setattr__(self, '__pydantic_initialised__', True) - - -def _dataclass_validate_assignment_setattr(self: 'Dataclass', name: str, value: Any) -> None: - if self.__pydantic_initialised__: - d = dict(self.__dict__) - d.pop(name, None) - known_field = self.__pydantic_model__.__fields__.get(name, None) - if known_field: - value, error_ = known_field.validate(value, d, loc=name, cls=self.__class__) - if error_: - raise ValidationError([error_], self.__class__) - - object.__setattr__(self, name, value) - - -def is_builtin_dataclass(_cls: Type[Any]) -> bool: - """ - Whether a class is a stdlib dataclass - (useful to discriminated a pydantic dataclass that is actually a wrapper around a stdlib dataclass) - - we check that - - `_cls` is a dataclass - - `_cls` is not a processed pydantic dataclass (with a basemodel attached) - - `_cls` is not a pydantic dataclass inheriting directly from a stdlib dataclass - e.g. - ``` - @dataclasses.dataclass - class A: - x: int - - @pydantic.dataclasses.dataclass - class B(A): - y: int - ``` - In this case, when we first check `B`, we make an extra check and look at the annotations ('y'), - which won't be a superset of all the dataclass fields (only the stdlib fields i.e. 'x') - """ - return ( - dataclasses.is_dataclass(_cls) - and not hasattr(_cls, '__pydantic_model__') - and set(_cls.__dataclass_fields__).issuperset(set(getattr(_cls, '__annotations__', {}))) - ) - - -def make_dataclass_validator(dc_cls: Type['Dataclass'], config: Type[BaseConfig]) -> 'CallableGenerator': - """ - Create a pydantic.dataclass from a builtin dataclass to add type validation - and yield the validators - It retrieves the parameters of the dataclass and forwards them to the newly created dataclass - """ - yield from _get_validators(dataclass(dc_cls, config=config, use_proxy=True)) diff --git a/functions/python/data-processor/pydantic/v1/datetime_parse.py b/functions/python/data-processor/pydantic/v1/datetime_parse.py deleted file mode 100644 index a7598fc6c..000000000 --- a/functions/python/data-processor/pydantic/v1/datetime_parse.py +++ /dev/null @@ -1,248 +0,0 @@ -""" -Functions to parse datetime objects. - -We're using regular expressions rather than time.strptime because: -- They provide both validation and parsing. -- They're more flexible for datetimes. -- The date/datetime/time constructors produce friendlier error messages. - -Stolen from https://raw.githubusercontent.com/django/django/main/django/utils/dateparse.py at -9718fa2e8abe430c3526a9278dd976443d4ae3c6 - -Changed to: -* use standard python datetime types not django.utils.timezone -* raise ValueError when regex doesn't match rather than returning None -* support parsing unix timestamps for dates and datetimes -""" -import re -from datetime import date, datetime, time, timedelta, timezone -from typing import Dict, Optional, Type, Union - -from pydantic.v1 import errors - -date_expr = r'(?P\d{4})-(?P\d{1,2})-(?P\d{1,2})' -time_expr = ( - r'(?P\d{1,2}):(?P\d{1,2})' - r'(?::(?P\d{1,2})(?:\.(?P\d{1,6})\d{0,6})?)?' - r'(?PZ|[+-]\d{2}(?::?\d{2})?)?$' -) - -date_re = re.compile(f'{date_expr}$') -time_re = re.compile(time_expr) -datetime_re = re.compile(f'{date_expr}[T ]{time_expr}') - -standard_duration_re = re.compile( - r'^' - r'(?:(?P-?\d+) (days?, )?)?' - r'((?:(?P-?\d+):)(?=\d+:\d+))?' - r'(?:(?P-?\d+):)?' - r'(?P-?\d+)' - r'(?:\.(?P\d{1,6})\d{0,6})?' - r'$' -) - -# Support the sections of ISO 8601 date representation that are accepted by timedelta -iso8601_duration_re = re.compile( - r'^(?P[-+]?)' - r'P' - r'(?:(?P\d+(.\d+)?)D)?' - r'(?:T' - r'(?:(?P\d+(.\d+)?)H)?' - r'(?:(?P\d+(.\d+)?)M)?' - r'(?:(?P\d+(.\d+)?)S)?' - r')?' - r'$' -) - -EPOCH = datetime(1970, 1, 1) -# if greater than this, the number is in ms, if less than or equal it's in seconds -# (in seconds this is 11th October 2603, in ms it's 20th August 1970) -MS_WATERSHED = int(2e10) -# slightly more than datetime.max in ns - (datetime.max - EPOCH).total_seconds() * 1e9 -MAX_NUMBER = int(3e20) -StrBytesIntFloat = Union[str, bytes, int, float] - - -def get_numeric(value: StrBytesIntFloat, native_expected_type: str) -> Union[None, int, float]: - if isinstance(value, (int, float)): - return value - try: - return float(value) - except ValueError: - return None - except TypeError: - raise TypeError(f'invalid type; expected {native_expected_type}, string, bytes, int or float') - - -def from_unix_seconds(seconds: Union[int, float]) -> datetime: - if seconds > MAX_NUMBER: - return datetime.max - elif seconds < -MAX_NUMBER: - return datetime.min - - while abs(seconds) > MS_WATERSHED: - seconds /= 1000 - dt = EPOCH + timedelta(seconds=seconds) - return dt.replace(tzinfo=timezone.utc) - - -def _parse_timezone(value: Optional[str], error: Type[Exception]) -> Union[None, int, timezone]: - if value == 'Z': - return timezone.utc - elif value is not None: - offset_mins = int(value[-2:]) if len(value) > 3 else 0 - offset = 60 * int(value[1:3]) + offset_mins - if value[0] == '-': - offset = -offset - try: - return timezone(timedelta(minutes=offset)) - except ValueError: - raise error() - else: - return None - - -def parse_date(value: Union[date, StrBytesIntFloat]) -> date: - """ - Parse a date/int/float/string and return a datetime.date. - - Raise ValueError if the input is well formatted but not a valid date. - Raise ValueError if the input isn't well formatted. - """ - if isinstance(value, date): - if isinstance(value, datetime): - return value.date() - else: - return value - - number = get_numeric(value, 'date') - if number is not None: - return from_unix_seconds(number).date() - - if isinstance(value, bytes): - value = value.decode() - - match = date_re.match(value) # type: ignore - if match is None: - raise errors.DateError() - - kw = {k: int(v) for k, v in match.groupdict().items()} - - try: - return date(**kw) - except ValueError: - raise errors.DateError() - - -def parse_time(value: Union[time, StrBytesIntFloat]) -> time: - """ - Parse a time/string and return a datetime.time. - - Raise ValueError if the input is well formatted but not a valid time. - Raise ValueError if the input isn't well formatted, in particular if it contains an offset. - """ - if isinstance(value, time): - return value - - number = get_numeric(value, 'time') - if number is not None: - if number >= 86400: - # doesn't make sense since the time time loop back around to 0 - raise errors.TimeError() - return (datetime.min + timedelta(seconds=number)).time() - - if isinstance(value, bytes): - value = value.decode() - - match = time_re.match(value) # type: ignore - if match is None: - raise errors.TimeError() - - kw = match.groupdict() - if kw['microsecond']: - kw['microsecond'] = kw['microsecond'].ljust(6, '0') - - tzinfo = _parse_timezone(kw.pop('tzinfo'), errors.TimeError) - kw_: Dict[str, Union[None, int, timezone]] = {k: int(v) for k, v in kw.items() if v is not None} - kw_['tzinfo'] = tzinfo - - try: - return time(**kw_) # type: ignore - except ValueError: - raise errors.TimeError() - - -def parse_datetime(value: Union[datetime, StrBytesIntFloat]) -> datetime: - """ - Parse a datetime/int/float/string and return a datetime.datetime. - - This function supports time zone offsets. When the input contains one, - the output uses a timezone with a fixed offset from UTC. - - Raise ValueError if the input is well formatted but not a valid datetime. - Raise ValueError if the input isn't well formatted. - """ - if isinstance(value, datetime): - return value - - number = get_numeric(value, 'datetime') - if number is not None: - return from_unix_seconds(number) - - if isinstance(value, bytes): - value = value.decode() - - match = datetime_re.match(value) # type: ignore - if match is None: - raise errors.DateTimeError() - - kw = match.groupdict() - if kw['microsecond']: - kw['microsecond'] = kw['microsecond'].ljust(6, '0') - - tzinfo = _parse_timezone(kw.pop('tzinfo'), errors.DateTimeError) - kw_: Dict[str, Union[None, int, timezone]] = {k: int(v) for k, v in kw.items() if v is not None} - kw_['tzinfo'] = tzinfo - - try: - return datetime(**kw_) # type: ignore - except ValueError: - raise errors.DateTimeError() - - -def parse_duration(value: StrBytesIntFloat) -> timedelta: - """ - Parse a duration int/float/string and return a datetime.timedelta. - - The preferred format for durations in Django is '%d %H:%M:%S.%f'. - - Also supports ISO 8601 representation. - """ - if isinstance(value, timedelta): - return value - - if isinstance(value, (int, float)): - # below code requires a string - value = f'{value:f}' - elif isinstance(value, bytes): - value = value.decode() - - try: - match = standard_duration_re.match(value) or iso8601_duration_re.match(value) - except TypeError: - raise TypeError('invalid type; expected timedelta, string, bytes, int or float') - - if not match: - raise errors.DurationError() - - kw = match.groupdict() - sign = -1 if kw.pop('sign', '+') == '-' else 1 - if kw.get('microseconds'): - kw['microseconds'] = kw['microseconds'].ljust(6, '0') - - if kw.get('seconds') and kw.get('microseconds') and kw['seconds'].startswith('-'): - kw['microseconds'] = '-' + kw['microseconds'] - - kw_ = {k: float(v) for k, v in kw.items() if v is not None} - - return sign * timedelta(**kw_) diff --git a/functions/python/data-processor/pydantic/v1/decorator.py b/functions/python/data-processor/pydantic/v1/decorator.py deleted file mode 100644 index 2c7c2c2ff..000000000 --- a/functions/python/data-processor/pydantic/v1/decorator.py +++ /dev/null @@ -1,264 +0,0 @@ -from functools import wraps -from typing import TYPE_CHECKING, Any, Callable, Dict, List, Mapping, Optional, Tuple, Type, TypeVar, Union, overload - -from pydantic.v1 import validator -from pydantic.v1.config import Extra -from pydantic.v1.errors import ConfigError -from pydantic.v1.main import BaseModel, create_model -from pydantic.v1.typing import get_all_type_hints -from pydantic.v1.utils import to_camel - -__all__ = ('validate_arguments',) - -if TYPE_CHECKING: - from pydantic.v1.typing import AnyCallable - - AnyCallableT = TypeVar('AnyCallableT', bound=AnyCallable) - ConfigType = Union[None, Type[Any], Dict[str, Any]] - - -@overload -def validate_arguments(func: None = None, *, config: 'ConfigType' = None) -> Callable[['AnyCallableT'], 'AnyCallableT']: - ... - - -@overload -def validate_arguments(func: 'AnyCallableT') -> 'AnyCallableT': - ... - - -def validate_arguments(func: Optional['AnyCallableT'] = None, *, config: 'ConfigType' = None) -> Any: - """ - Decorator to validate the arguments passed to a function. - """ - - def validate(_func: 'AnyCallable') -> 'AnyCallable': - vd = ValidatedFunction(_func, config) - - @wraps(_func) - def wrapper_function(*args: Any, **kwargs: Any) -> Any: - return vd.call(*args, **kwargs) - - wrapper_function.vd = vd # type: ignore - wrapper_function.validate = vd.init_model_instance # type: ignore - wrapper_function.raw_function = vd.raw_function # type: ignore - wrapper_function.model = vd.model # type: ignore - return wrapper_function - - if func: - return validate(func) - else: - return validate - - -ALT_V_ARGS = 'v__args' -ALT_V_KWARGS = 'v__kwargs' -V_POSITIONAL_ONLY_NAME = 'v__positional_only' -V_DUPLICATE_KWARGS = 'v__duplicate_kwargs' - - -class ValidatedFunction: - def __init__(self, function: 'AnyCallableT', config: 'ConfigType'): # noqa C901 - from inspect import Parameter, signature - - parameters: Mapping[str, Parameter] = signature(function).parameters - - if parameters.keys() & {ALT_V_ARGS, ALT_V_KWARGS, V_POSITIONAL_ONLY_NAME, V_DUPLICATE_KWARGS}: - raise ConfigError( - f'"{ALT_V_ARGS}", "{ALT_V_KWARGS}", "{V_POSITIONAL_ONLY_NAME}" and "{V_DUPLICATE_KWARGS}" ' - f'are not permitted as argument names when using the "{validate_arguments.__name__}" decorator' - ) - - self.raw_function = function - self.arg_mapping: Dict[int, str] = {} - self.positional_only_args = set() - self.v_args_name = 'args' - self.v_kwargs_name = 'kwargs' - - type_hints = get_all_type_hints(function) - takes_args = False - takes_kwargs = False - fields: Dict[str, Tuple[Any, Any]] = {} - for i, (name, p) in enumerate(parameters.items()): - if p.annotation is p.empty: - annotation = Any - else: - annotation = type_hints[name] - - default = ... if p.default is p.empty else p.default - if p.kind == Parameter.POSITIONAL_ONLY: - self.arg_mapping[i] = name - fields[name] = annotation, default - fields[V_POSITIONAL_ONLY_NAME] = List[str], None - self.positional_only_args.add(name) - elif p.kind == Parameter.POSITIONAL_OR_KEYWORD: - self.arg_mapping[i] = name - fields[name] = annotation, default - fields[V_DUPLICATE_KWARGS] = List[str], None - elif p.kind == Parameter.KEYWORD_ONLY: - fields[name] = annotation, default - elif p.kind == Parameter.VAR_POSITIONAL: - self.v_args_name = name - fields[name] = Tuple[annotation, ...], None - takes_args = True - else: - assert p.kind == Parameter.VAR_KEYWORD, p.kind - self.v_kwargs_name = name - fields[name] = Dict[str, annotation], None # type: ignore - takes_kwargs = True - - # these checks avoid a clash between "args" and a field with that name - if not takes_args and self.v_args_name in fields: - self.v_args_name = ALT_V_ARGS - - # same with "kwargs" - if not takes_kwargs and self.v_kwargs_name in fields: - self.v_kwargs_name = ALT_V_KWARGS - - if not takes_args: - # we add the field so validation below can raise the correct exception - fields[self.v_args_name] = List[Any], None - - if not takes_kwargs: - # same with kwargs - fields[self.v_kwargs_name] = Dict[Any, Any], None - - self.create_model(fields, takes_args, takes_kwargs, config) - - def init_model_instance(self, *args: Any, **kwargs: Any) -> BaseModel: - values = self.build_values(args, kwargs) - return self.model(**values) - - def call(self, *args: Any, **kwargs: Any) -> Any: - m = self.init_model_instance(*args, **kwargs) - return self.execute(m) - - def build_values(self, args: Tuple[Any, ...], kwargs: Dict[str, Any]) -> Dict[str, Any]: - values: Dict[str, Any] = {} - if args: - arg_iter = enumerate(args) - while True: - try: - i, a = next(arg_iter) - except StopIteration: - break - arg_name = self.arg_mapping.get(i) - if arg_name is not None: - values[arg_name] = a - else: - values[self.v_args_name] = [a] + [a for _, a in arg_iter] - break - - var_kwargs: Dict[str, Any] = {} - wrong_positional_args = [] - duplicate_kwargs = [] - fields_alias = [ - field.alias - for name, field in self.model.__fields__.items() - if name not in (self.v_args_name, self.v_kwargs_name) - ] - non_var_fields = set(self.model.__fields__) - {self.v_args_name, self.v_kwargs_name} - for k, v in kwargs.items(): - if k in non_var_fields or k in fields_alias: - if k in self.positional_only_args: - wrong_positional_args.append(k) - if k in values: - duplicate_kwargs.append(k) - values[k] = v - else: - var_kwargs[k] = v - - if var_kwargs: - values[self.v_kwargs_name] = var_kwargs - if wrong_positional_args: - values[V_POSITIONAL_ONLY_NAME] = wrong_positional_args - if duplicate_kwargs: - values[V_DUPLICATE_KWARGS] = duplicate_kwargs - return values - - def execute(self, m: BaseModel) -> Any: - d = {k: v for k, v in m._iter() if k in m.__fields_set__ or m.__fields__[k].default_factory} - var_kwargs = d.pop(self.v_kwargs_name, {}) - - if self.v_args_name in d: - args_: List[Any] = [] - in_kwargs = False - kwargs = {} - for name, value in d.items(): - if in_kwargs: - kwargs[name] = value - elif name == self.v_args_name: - args_ += value - in_kwargs = True - else: - args_.append(value) - return self.raw_function(*args_, **kwargs, **var_kwargs) - elif self.positional_only_args: - args_ = [] - kwargs = {} - for name, value in d.items(): - if name in self.positional_only_args: - args_.append(value) - else: - kwargs[name] = value - return self.raw_function(*args_, **kwargs, **var_kwargs) - else: - return self.raw_function(**d, **var_kwargs) - - def create_model(self, fields: Dict[str, Any], takes_args: bool, takes_kwargs: bool, config: 'ConfigType') -> None: - pos_args = len(self.arg_mapping) - - class CustomConfig: - pass - - if not TYPE_CHECKING: # pragma: no branch - if isinstance(config, dict): - CustomConfig = type('Config', (), config) # noqa: F811 - elif config is not None: - CustomConfig = config # noqa: F811 - - if hasattr(CustomConfig, 'fields') or hasattr(CustomConfig, 'alias_generator'): - raise ConfigError( - 'Setting the "fields" and "alias_generator" property on custom Config for ' - '@validate_arguments is not yet supported, please remove.' - ) - - class DecoratorBaseModel(BaseModel): - @validator(self.v_args_name, check_fields=False, allow_reuse=True) - def check_args(cls, v: Optional[List[Any]]) -> Optional[List[Any]]: - if takes_args or v is None: - return v - - raise TypeError(f'{pos_args} positional arguments expected but {pos_args + len(v)} given') - - @validator(self.v_kwargs_name, check_fields=False, allow_reuse=True) - def check_kwargs(cls, v: Optional[Dict[str, Any]]) -> Optional[Dict[str, Any]]: - if takes_kwargs or v is None: - return v - - plural = '' if len(v) == 1 else 's' - keys = ', '.join(map(repr, v.keys())) - raise TypeError(f'unexpected keyword argument{plural}: {keys}') - - @validator(V_POSITIONAL_ONLY_NAME, check_fields=False, allow_reuse=True) - def check_positional_only(cls, v: Optional[List[str]]) -> None: - if v is None: - return - - plural = '' if len(v) == 1 else 's' - keys = ', '.join(map(repr, v)) - raise TypeError(f'positional-only argument{plural} passed as keyword argument{plural}: {keys}') - - @validator(V_DUPLICATE_KWARGS, check_fields=False, allow_reuse=True) - def check_duplicate_kwargs(cls, v: Optional[List[str]]) -> None: - if v is None: - return - - plural = '' if len(v) == 1 else 's' - keys = ', '.join(map(repr, v)) - raise TypeError(f'multiple values for argument{plural}: {keys}') - - class Config(CustomConfig): - extra = getattr(CustomConfig, 'extra', Extra.forbid) - - self.model = create_model(to_camel(self.raw_function.__name__), __base__=DecoratorBaseModel, **fields) diff --git a/functions/python/data-processor/pydantic/v1/env_settings.py b/functions/python/data-processor/pydantic/v1/env_settings.py deleted file mode 100644 index 5f6f21750..000000000 --- a/functions/python/data-processor/pydantic/v1/env_settings.py +++ /dev/null @@ -1,350 +0,0 @@ -import os -import warnings -from pathlib import Path -from typing import AbstractSet, Any, Callable, ClassVar, Dict, List, Mapping, Optional, Tuple, Type, Union - -from pydantic.v1.config import BaseConfig, Extra -from pydantic.v1.fields import ModelField -from pydantic.v1.main import BaseModel -from pydantic.v1.types import JsonWrapper -from pydantic.v1.typing import StrPath, display_as_type, get_origin, is_union -from pydantic.v1.utils import deep_update, lenient_issubclass, path_type, sequence_like - -env_file_sentinel = str(object()) - -SettingsSourceCallable = Callable[['BaseSettings'], Dict[str, Any]] -DotenvType = Union[StrPath, List[StrPath], Tuple[StrPath, ...]] - - -class SettingsError(ValueError): - pass - - -class BaseSettings(BaseModel): - """ - Base class for settings, allowing values to be overridden by environment variables. - - This is useful in production for secrets you do not wish to save in code, it plays nicely with docker(-compose), - Heroku and any 12 factor app design. - """ - - def __init__( - __pydantic_self__, - _env_file: Optional[DotenvType] = env_file_sentinel, - _env_file_encoding: Optional[str] = None, - _env_nested_delimiter: Optional[str] = None, - _secrets_dir: Optional[StrPath] = None, - **values: Any, - ) -> None: - # Uses something other than `self` the first arg to allow "self" as a settable attribute - super().__init__( - **__pydantic_self__._build_values( - values, - _env_file=_env_file, - _env_file_encoding=_env_file_encoding, - _env_nested_delimiter=_env_nested_delimiter, - _secrets_dir=_secrets_dir, - ) - ) - - def _build_values( - self, - init_kwargs: Dict[str, Any], - _env_file: Optional[DotenvType] = None, - _env_file_encoding: Optional[str] = None, - _env_nested_delimiter: Optional[str] = None, - _secrets_dir: Optional[StrPath] = None, - ) -> Dict[str, Any]: - # Configure built-in sources - init_settings = InitSettingsSource(init_kwargs=init_kwargs) - env_settings = EnvSettingsSource( - env_file=(_env_file if _env_file != env_file_sentinel else self.__config__.env_file), - env_file_encoding=( - _env_file_encoding if _env_file_encoding is not None else self.__config__.env_file_encoding - ), - env_nested_delimiter=( - _env_nested_delimiter if _env_nested_delimiter is not None else self.__config__.env_nested_delimiter - ), - env_prefix_len=len(self.__config__.env_prefix), - ) - file_secret_settings = SecretsSettingsSource(secrets_dir=_secrets_dir or self.__config__.secrets_dir) - # Provide a hook to set built-in sources priority and add / remove sources - sources = self.__config__.customise_sources( - init_settings=init_settings, env_settings=env_settings, file_secret_settings=file_secret_settings - ) - if sources: - return deep_update(*reversed([source(self) for source in sources])) - else: - # no one should mean to do this, but I think returning an empty dict is marginally preferable - # to an informative error and much better than a confusing error - return {} - - class Config(BaseConfig): - env_prefix: str = '' - env_file: Optional[DotenvType] = None - env_file_encoding: Optional[str] = None - env_nested_delimiter: Optional[str] = None - secrets_dir: Optional[StrPath] = None - validate_all: bool = True - extra: Extra = Extra.forbid - arbitrary_types_allowed: bool = True - case_sensitive: bool = False - - @classmethod - def prepare_field(cls, field: ModelField) -> None: - env_names: Union[List[str], AbstractSet[str]] - field_info_from_config = cls.get_field_info(field.name) - - env = field_info_from_config.get('env') or field.field_info.extra.get('env') - if env is None: - if field.has_alias: - warnings.warn( - 'aliases are no longer used by BaseSettings to define which environment variables to read. ' - 'Instead use the "env" field setting. ' - 'See https://pydantic-docs.helpmanual.io/usage/settings/#environment-variable-names', - FutureWarning, - ) - env_names = {cls.env_prefix + field.name} - elif isinstance(env, str): - env_names = {env} - elif isinstance(env, (set, frozenset)): - env_names = env - elif sequence_like(env): - env_names = list(env) - else: - raise TypeError(f'invalid field env: {env!r} ({display_as_type(env)}); should be string, list or set') - - if not cls.case_sensitive: - env_names = env_names.__class__(n.lower() for n in env_names) - field.field_info.extra['env_names'] = env_names - - @classmethod - def customise_sources( - cls, - init_settings: SettingsSourceCallable, - env_settings: SettingsSourceCallable, - file_secret_settings: SettingsSourceCallable, - ) -> Tuple[SettingsSourceCallable, ...]: - return init_settings, env_settings, file_secret_settings - - @classmethod - def parse_env_var(cls, field_name: str, raw_val: str) -> Any: - return cls.json_loads(raw_val) - - # populated by the metaclass using the Config class defined above, annotated here to help IDEs only - __config__: ClassVar[Type[Config]] - - -class InitSettingsSource: - __slots__ = ('init_kwargs',) - - def __init__(self, init_kwargs: Dict[str, Any]): - self.init_kwargs = init_kwargs - - def __call__(self, settings: BaseSettings) -> Dict[str, Any]: - return self.init_kwargs - - def __repr__(self) -> str: - return f'InitSettingsSource(init_kwargs={self.init_kwargs!r})' - - -class EnvSettingsSource: - __slots__ = ('env_file', 'env_file_encoding', 'env_nested_delimiter', 'env_prefix_len') - - def __init__( - self, - env_file: Optional[DotenvType], - env_file_encoding: Optional[str], - env_nested_delimiter: Optional[str] = None, - env_prefix_len: int = 0, - ): - self.env_file: Optional[DotenvType] = env_file - self.env_file_encoding: Optional[str] = env_file_encoding - self.env_nested_delimiter: Optional[str] = env_nested_delimiter - self.env_prefix_len: int = env_prefix_len - - def __call__(self, settings: BaseSettings) -> Dict[str, Any]: # noqa C901 - """ - Build environment variables suitable for passing to the Model. - """ - d: Dict[str, Any] = {} - - if settings.__config__.case_sensitive: - env_vars: Mapping[str, Optional[str]] = os.environ - else: - env_vars = {k.lower(): v for k, v in os.environ.items()} - - dotenv_vars = self._read_env_files(settings.__config__.case_sensitive) - if dotenv_vars: - env_vars = {**dotenv_vars, **env_vars} - - for field in settings.__fields__.values(): - env_val: Optional[str] = None - for env_name in field.field_info.extra['env_names']: - env_val = env_vars.get(env_name) - if env_val is not None: - break - - is_complex, allow_parse_failure = self.field_is_complex(field) - if is_complex: - if env_val is None: - # field is complex but no value found so far, try explode_env_vars - env_val_built = self.explode_env_vars(field, env_vars) - if env_val_built: - d[field.alias] = env_val_built - else: - # field is complex and there's a value, decode that as JSON, then add explode_env_vars - try: - env_val = settings.__config__.parse_env_var(field.name, env_val) - except ValueError as e: - if not allow_parse_failure: - raise SettingsError(f'error parsing env var "{env_name}"') from e - - if isinstance(env_val, dict): - d[field.alias] = deep_update(env_val, self.explode_env_vars(field, env_vars)) - else: - d[field.alias] = env_val - elif env_val is not None: - # simplest case, field is not complex, we only need to add the value if it was found - d[field.alias] = env_val - - return d - - def _read_env_files(self, case_sensitive: bool) -> Dict[str, Optional[str]]: - env_files = self.env_file - if env_files is None: - return {} - - if isinstance(env_files, (str, os.PathLike)): - env_files = [env_files] - - dotenv_vars = {} - for env_file in env_files: - env_path = Path(env_file).expanduser() - if env_path.is_file(): - dotenv_vars.update( - read_env_file(env_path, encoding=self.env_file_encoding, case_sensitive=case_sensitive) - ) - - return dotenv_vars - - def field_is_complex(self, field: ModelField) -> Tuple[bool, bool]: - """ - Find out if a field is complex, and if so whether JSON errors should be ignored - """ - if lenient_issubclass(field.annotation, JsonWrapper): - return False, False - - if field.is_complex(): - allow_parse_failure = False - elif is_union(get_origin(field.type_)) and field.sub_fields and any(f.is_complex() for f in field.sub_fields): - allow_parse_failure = True - else: - return False, False - - return True, allow_parse_failure - - def explode_env_vars(self, field: ModelField, env_vars: Mapping[str, Optional[str]]) -> Dict[str, Any]: - """ - Process env_vars and extract the values of keys containing env_nested_delimiter into nested dictionaries. - - This is applied to a single field, hence filtering by env_var prefix. - """ - prefixes = [f'{env_name}{self.env_nested_delimiter}' for env_name in field.field_info.extra['env_names']] - result: Dict[str, Any] = {} - for env_name, env_val in env_vars.items(): - if not any(env_name.startswith(prefix) for prefix in prefixes): - continue - # we remove the prefix before splitting in case the prefix has characters in common with the delimiter - env_name_without_prefix = env_name[self.env_prefix_len :] - _, *keys, last_key = env_name_without_prefix.split(self.env_nested_delimiter) - env_var = result - for key in keys: - env_var = env_var.setdefault(key, {}) - env_var[last_key] = env_val - - return result - - def __repr__(self) -> str: - return ( - f'EnvSettingsSource(env_file={self.env_file!r}, env_file_encoding={self.env_file_encoding!r}, ' - f'env_nested_delimiter={self.env_nested_delimiter!r})' - ) - - -class SecretsSettingsSource: - __slots__ = ('secrets_dir',) - - def __init__(self, secrets_dir: Optional[StrPath]): - self.secrets_dir: Optional[StrPath] = secrets_dir - - def __call__(self, settings: BaseSettings) -> Dict[str, Any]: - """ - Build fields from "secrets" files. - """ - secrets: Dict[str, Optional[str]] = {} - - if self.secrets_dir is None: - return secrets - - secrets_path = Path(self.secrets_dir).expanduser() - - if not secrets_path.exists(): - warnings.warn(f'directory "{secrets_path}" does not exist') - return secrets - - if not secrets_path.is_dir(): - raise SettingsError(f'secrets_dir must reference a directory, not a {path_type(secrets_path)}') - - for field in settings.__fields__.values(): - for env_name in field.field_info.extra['env_names']: - path = find_case_path(secrets_path, env_name, settings.__config__.case_sensitive) - if not path: - # path does not exist, we currently don't return a warning for this - continue - - if path.is_file(): - secret_value = path.read_text().strip() - if field.is_complex(): - try: - secret_value = settings.__config__.parse_env_var(field.name, secret_value) - except ValueError as e: - raise SettingsError(f'error parsing env var "{env_name}"') from e - - secrets[field.alias] = secret_value - else: - warnings.warn( - f'attempted to load secret file "{path}" but found a {path_type(path)} instead.', - stacklevel=4, - ) - return secrets - - def __repr__(self) -> str: - return f'SecretsSettingsSource(secrets_dir={self.secrets_dir!r})' - - -def read_env_file( - file_path: StrPath, *, encoding: str = None, case_sensitive: bool = False -) -> Dict[str, Optional[str]]: - try: - from dotenv import dotenv_values - except ImportError as e: - raise ImportError('python-dotenv is not installed, run `pip install pydantic[dotenv]`') from e - - file_vars: Dict[str, Optional[str]] = dotenv_values(file_path, encoding=encoding or 'utf8') - if not case_sensitive: - return {k.lower(): v for k, v in file_vars.items()} - else: - return file_vars - - -def find_case_path(dir_path: Path, file_name: str, case_sensitive: bool) -> Optional[Path]: - """ - Find a file within path's directory matching filename, optionally ignoring case. - """ - for f in dir_path.iterdir(): - if f.name == file_name: - return f - elif not case_sensitive and f.name.lower() == file_name.lower(): - return f - return None diff --git a/functions/python/data-processor/pydantic/v1/error_wrappers.py b/functions/python/data-processor/pydantic/v1/error_wrappers.py deleted file mode 100644 index bc7f26314..000000000 --- a/functions/python/data-processor/pydantic/v1/error_wrappers.py +++ /dev/null @@ -1,161 +0,0 @@ -import json -from typing import TYPE_CHECKING, Any, Dict, Generator, List, Optional, Sequence, Tuple, Type, Union - -from pydantic.v1.json import pydantic_encoder -from pydantic.v1.utils import Representation - -if TYPE_CHECKING: - from typing_extensions import TypedDict - - from pydantic.v1.config import BaseConfig - from pydantic.v1.types import ModelOrDc - from pydantic.v1.typing import ReprArgs - - Loc = Tuple[Union[int, str], ...] - - class _ErrorDictRequired(TypedDict): - loc: Loc - msg: str - type: str - - class ErrorDict(_ErrorDictRequired, total=False): - ctx: Dict[str, Any] - - -__all__ = 'ErrorWrapper', 'ValidationError' - - -class ErrorWrapper(Representation): - __slots__ = 'exc', '_loc' - - def __init__(self, exc: Exception, loc: Union[str, 'Loc']) -> None: - self.exc = exc - self._loc = loc - - def loc_tuple(self) -> 'Loc': - if isinstance(self._loc, tuple): - return self._loc - else: - return (self._loc,) - - def __repr_args__(self) -> 'ReprArgs': - return [('exc', self.exc), ('loc', self.loc_tuple())] - - -# ErrorList is something like Union[List[Union[List[ErrorWrapper], ErrorWrapper]], ErrorWrapper] -# but recursive, therefore just use: -ErrorList = Union[Sequence[Any], ErrorWrapper] - - -class ValidationError(Representation, ValueError): - __slots__ = 'raw_errors', 'model', '_error_cache' - - def __init__(self, errors: Sequence[ErrorList], model: 'ModelOrDc') -> None: - self.raw_errors = errors - self.model = model - self._error_cache: Optional[List['ErrorDict']] = None - - def errors(self) -> List['ErrorDict']: - if self._error_cache is None: - try: - config = self.model.__config__ # type: ignore - except AttributeError: - config = self.model.__pydantic_model__.__config__ # type: ignore - self._error_cache = list(flatten_errors(self.raw_errors, config)) - return self._error_cache - - def json(self, *, indent: Union[None, int, str] = 2) -> str: - return json.dumps(self.errors(), indent=indent, default=pydantic_encoder) - - def __str__(self) -> str: - errors = self.errors() - no_errors = len(errors) - return ( - f'{no_errors} validation error{"" if no_errors == 1 else "s"} for {self.model.__name__}\n' - f'{display_errors(errors)}' - ) - - def __repr_args__(self) -> 'ReprArgs': - return [('model', self.model.__name__), ('errors', self.errors())] - - -def display_errors(errors: List['ErrorDict']) -> str: - return '\n'.join(f'{_display_error_loc(e)}\n {e["msg"]} ({_display_error_type_and_ctx(e)})' for e in errors) - - -def _display_error_loc(error: 'ErrorDict') -> str: - return ' -> '.join(str(e) for e in error['loc']) - - -def _display_error_type_and_ctx(error: 'ErrorDict') -> str: - t = 'type=' + error['type'] - ctx = error.get('ctx') - if ctx: - return t + ''.join(f'; {k}={v}' for k, v in ctx.items()) - else: - return t - - -def flatten_errors( - errors: Sequence[Any], config: Type['BaseConfig'], loc: Optional['Loc'] = None -) -> Generator['ErrorDict', None, None]: - for error in errors: - if isinstance(error, ErrorWrapper): - if loc: - error_loc = loc + error.loc_tuple() - else: - error_loc = error.loc_tuple() - - if isinstance(error.exc, ValidationError): - yield from flatten_errors(error.exc.raw_errors, config, error_loc) - else: - yield error_dict(error.exc, config, error_loc) - elif isinstance(error, list): - yield from flatten_errors(error, config, loc=loc) - else: - raise RuntimeError(f'Unknown error object: {error}') - - -def error_dict(exc: Exception, config: Type['BaseConfig'], loc: 'Loc') -> 'ErrorDict': - type_ = get_exc_type(exc.__class__) - msg_template = config.error_msg_templates.get(type_) or getattr(exc, 'msg_template', None) - ctx = exc.__dict__ - if msg_template: - msg = msg_template.format(**ctx) - else: - msg = str(exc) - - d: 'ErrorDict' = {'loc': loc, 'msg': msg, 'type': type_} - - if ctx: - d['ctx'] = ctx - - return d - - -_EXC_TYPE_CACHE: Dict[Type[Exception], str] = {} - - -def get_exc_type(cls: Type[Exception]) -> str: - # slightly more efficient than using lru_cache since we don't need to worry about the cache filling up - try: - return _EXC_TYPE_CACHE[cls] - except KeyError: - r = _get_exc_type(cls) - _EXC_TYPE_CACHE[cls] = r - return r - - -def _get_exc_type(cls: Type[Exception]) -> str: - if issubclass(cls, AssertionError): - return 'assertion_error' - - base_name = 'type_error' if issubclass(cls, TypeError) else 'value_error' - if cls in (TypeError, ValueError): - # just TypeError or ValueError, no extra code - return base_name - - # if it's not a TypeError or ValueError, we just take the lowercase of the exception name - # no chaining or snake case logic, use "code" for more complex error types. - code = getattr(cls, 'code', None) or cls.__name__.replace('Error', '').lower() - return base_name + '.' + code diff --git a/functions/python/data-processor/pydantic/v1/errors.py b/functions/python/data-processor/pydantic/v1/errors.py deleted file mode 100644 index 6e8644258..000000000 --- a/functions/python/data-processor/pydantic/v1/errors.py +++ /dev/null @@ -1,646 +0,0 @@ -from decimal import Decimal -from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Sequence, Set, Tuple, Type, Union - -from pydantic.v1.typing import display_as_type - -if TYPE_CHECKING: - from pydantic.v1.typing import DictStrAny - -# explicitly state exports to avoid "from pydantic.v1.errors import *" also importing Decimal, Path etc. -__all__ = ( - 'PydanticTypeError', - 'PydanticValueError', - 'ConfigError', - 'MissingError', - 'ExtraError', - 'NoneIsNotAllowedError', - 'NoneIsAllowedError', - 'WrongConstantError', - 'NotNoneError', - 'BoolError', - 'BytesError', - 'DictError', - 'EmailError', - 'UrlError', - 'UrlSchemeError', - 'UrlSchemePermittedError', - 'UrlUserInfoError', - 'UrlHostError', - 'UrlHostTldError', - 'UrlPortError', - 'UrlExtraError', - 'EnumError', - 'IntEnumError', - 'EnumMemberError', - 'IntegerError', - 'FloatError', - 'PathError', - 'PathNotExistsError', - 'PathNotAFileError', - 'PathNotADirectoryError', - 'PyObjectError', - 'SequenceError', - 'ListError', - 'SetError', - 'FrozenSetError', - 'TupleError', - 'TupleLengthError', - 'ListMinLengthError', - 'ListMaxLengthError', - 'ListUniqueItemsError', - 'SetMinLengthError', - 'SetMaxLengthError', - 'FrozenSetMinLengthError', - 'FrozenSetMaxLengthError', - 'AnyStrMinLengthError', - 'AnyStrMaxLengthError', - 'StrError', - 'StrRegexError', - 'NumberNotGtError', - 'NumberNotGeError', - 'NumberNotLtError', - 'NumberNotLeError', - 'NumberNotMultipleError', - 'DecimalError', - 'DecimalIsNotFiniteError', - 'DecimalMaxDigitsError', - 'DecimalMaxPlacesError', - 'DecimalWholeDigitsError', - 'DateTimeError', - 'DateError', - 'DateNotInThePastError', - 'DateNotInTheFutureError', - 'TimeError', - 'DurationError', - 'HashableError', - 'UUIDError', - 'UUIDVersionError', - 'ArbitraryTypeError', - 'ClassError', - 'SubclassError', - 'JsonError', - 'JsonTypeError', - 'PatternError', - 'DataclassTypeError', - 'CallableError', - 'IPvAnyAddressError', - 'IPvAnyInterfaceError', - 'IPvAnyNetworkError', - 'IPv4AddressError', - 'IPv6AddressError', - 'IPv4NetworkError', - 'IPv6NetworkError', - 'IPv4InterfaceError', - 'IPv6InterfaceError', - 'ColorError', - 'StrictBoolError', - 'NotDigitError', - 'LuhnValidationError', - 'InvalidLengthForBrand', - 'InvalidByteSize', - 'InvalidByteSizeUnit', - 'MissingDiscriminator', - 'InvalidDiscriminator', -) - - -def cls_kwargs(cls: Type['PydanticErrorMixin'], ctx: 'DictStrAny') -> 'PydanticErrorMixin': - """ - For built-in exceptions like ValueError or TypeError, we need to implement - __reduce__ to override the default behaviour (instead of __getstate__/__setstate__) - By default pickle protocol 2 calls `cls.__new__(cls, *args)`. - Since we only use kwargs, we need a little constructor to change that. - Note: the callable can't be a lambda as pickle looks in the namespace to find it - """ - return cls(**ctx) - - -class PydanticErrorMixin: - code: str - msg_template: str - - def __init__(self, **ctx: Any) -> None: - self.__dict__ = ctx - - def __str__(self) -> str: - return self.msg_template.format(**self.__dict__) - - def __reduce__(self) -> Tuple[Callable[..., 'PydanticErrorMixin'], Tuple[Type['PydanticErrorMixin'], 'DictStrAny']]: - return cls_kwargs, (self.__class__, self.__dict__) - - -class PydanticTypeError(PydanticErrorMixin, TypeError): - pass - - -class PydanticValueError(PydanticErrorMixin, ValueError): - pass - - -class ConfigError(RuntimeError): - pass - - -class MissingError(PydanticValueError): - msg_template = 'field required' - - -class ExtraError(PydanticValueError): - msg_template = 'extra fields not permitted' - - -class NoneIsNotAllowedError(PydanticTypeError): - code = 'none.not_allowed' - msg_template = 'none is not an allowed value' - - -class NoneIsAllowedError(PydanticTypeError): - code = 'none.allowed' - msg_template = 'value is not none' - - -class WrongConstantError(PydanticValueError): - code = 'const' - - def __str__(self) -> str: - permitted = ', '.join(repr(v) for v in self.permitted) # type: ignore - return f'unexpected value; permitted: {permitted}' - - -class NotNoneError(PydanticTypeError): - code = 'not_none' - msg_template = 'value is not None' - - -class BoolError(PydanticTypeError): - msg_template = 'value could not be parsed to a boolean' - - -class BytesError(PydanticTypeError): - msg_template = 'byte type expected' - - -class DictError(PydanticTypeError): - msg_template = 'value is not a valid dict' - - -class EmailError(PydanticValueError): - msg_template = 'value is not a valid email address' - - -class UrlError(PydanticValueError): - code = 'url' - - -class UrlSchemeError(UrlError): - code = 'url.scheme' - msg_template = 'invalid or missing URL scheme' - - -class UrlSchemePermittedError(UrlError): - code = 'url.scheme' - msg_template = 'URL scheme not permitted' - - def __init__(self, allowed_schemes: Set[str]): - super().__init__(allowed_schemes=allowed_schemes) - - -class UrlUserInfoError(UrlError): - code = 'url.userinfo' - msg_template = 'userinfo required in URL but missing' - - -class UrlHostError(UrlError): - code = 'url.host' - msg_template = 'URL host invalid' - - -class UrlHostTldError(UrlError): - code = 'url.host' - msg_template = 'URL host invalid, top level domain required' - - -class UrlPortError(UrlError): - code = 'url.port' - msg_template = 'URL port invalid, port cannot exceed 65535' - - -class UrlExtraError(UrlError): - code = 'url.extra' - msg_template = 'URL invalid, extra characters found after valid URL: {extra!r}' - - -class EnumMemberError(PydanticTypeError): - code = 'enum' - - def __str__(self) -> str: - permitted = ', '.join(repr(v.value) for v in self.enum_values) # type: ignore - return f'value is not a valid enumeration member; permitted: {permitted}' - - -class IntegerError(PydanticTypeError): - msg_template = 'value is not a valid integer' - - -class FloatError(PydanticTypeError): - msg_template = 'value is not a valid float' - - -class PathError(PydanticTypeError): - msg_template = 'value is not a valid path' - - -class _PathValueError(PydanticValueError): - def __init__(self, *, path: Path) -> None: - super().__init__(path=str(path)) - - -class PathNotExistsError(_PathValueError): - code = 'path.not_exists' - msg_template = 'file or directory at path "{path}" does not exist' - - -class PathNotAFileError(_PathValueError): - code = 'path.not_a_file' - msg_template = 'path "{path}" does not point to a file' - - -class PathNotADirectoryError(_PathValueError): - code = 'path.not_a_directory' - msg_template = 'path "{path}" does not point to a directory' - - -class PyObjectError(PydanticTypeError): - msg_template = 'ensure this value contains valid import path or valid callable: {error_message}' - - -class SequenceError(PydanticTypeError): - msg_template = 'value is not a valid sequence' - - -class IterableError(PydanticTypeError): - msg_template = 'value is not a valid iterable' - - -class ListError(PydanticTypeError): - msg_template = 'value is not a valid list' - - -class SetError(PydanticTypeError): - msg_template = 'value is not a valid set' - - -class FrozenSetError(PydanticTypeError): - msg_template = 'value is not a valid frozenset' - - -class DequeError(PydanticTypeError): - msg_template = 'value is not a valid deque' - - -class TupleError(PydanticTypeError): - msg_template = 'value is not a valid tuple' - - -class TupleLengthError(PydanticValueError): - code = 'tuple.length' - msg_template = 'wrong tuple length {actual_length}, expected {expected_length}' - - def __init__(self, *, actual_length: int, expected_length: int) -> None: - super().__init__(actual_length=actual_length, expected_length=expected_length) - - -class ListMinLengthError(PydanticValueError): - code = 'list.min_items' - msg_template = 'ensure this value has at least {limit_value} items' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class ListMaxLengthError(PydanticValueError): - code = 'list.max_items' - msg_template = 'ensure this value has at most {limit_value} items' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class ListUniqueItemsError(PydanticValueError): - code = 'list.unique_items' - msg_template = 'the list has duplicated items' - - -class SetMinLengthError(PydanticValueError): - code = 'set.min_items' - msg_template = 'ensure this value has at least {limit_value} items' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class SetMaxLengthError(PydanticValueError): - code = 'set.max_items' - msg_template = 'ensure this value has at most {limit_value} items' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class FrozenSetMinLengthError(PydanticValueError): - code = 'frozenset.min_items' - msg_template = 'ensure this value has at least {limit_value} items' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class FrozenSetMaxLengthError(PydanticValueError): - code = 'frozenset.max_items' - msg_template = 'ensure this value has at most {limit_value} items' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class AnyStrMinLengthError(PydanticValueError): - code = 'any_str.min_length' - msg_template = 'ensure this value has at least {limit_value} characters' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class AnyStrMaxLengthError(PydanticValueError): - code = 'any_str.max_length' - msg_template = 'ensure this value has at most {limit_value} characters' - - def __init__(self, *, limit_value: int) -> None: - super().__init__(limit_value=limit_value) - - -class StrError(PydanticTypeError): - msg_template = 'str type expected' - - -class StrRegexError(PydanticValueError): - code = 'str.regex' - msg_template = 'string does not match regex "{pattern}"' - - def __init__(self, *, pattern: str) -> None: - super().__init__(pattern=pattern) - - -class _NumberBoundError(PydanticValueError): - def __init__(self, *, limit_value: Union[int, float, Decimal]) -> None: - super().__init__(limit_value=limit_value) - - -class NumberNotGtError(_NumberBoundError): - code = 'number.not_gt' - msg_template = 'ensure this value is greater than {limit_value}' - - -class NumberNotGeError(_NumberBoundError): - code = 'number.not_ge' - msg_template = 'ensure this value is greater than or equal to {limit_value}' - - -class NumberNotLtError(_NumberBoundError): - code = 'number.not_lt' - msg_template = 'ensure this value is less than {limit_value}' - - -class NumberNotLeError(_NumberBoundError): - code = 'number.not_le' - msg_template = 'ensure this value is less than or equal to {limit_value}' - - -class NumberNotFiniteError(PydanticValueError): - code = 'number.not_finite_number' - msg_template = 'ensure this value is a finite number' - - -class NumberNotMultipleError(PydanticValueError): - code = 'number.not_multiple' - msg_template = 'ensure this value is a multiple of {multiple_of}' - - def __init__(self, *, multiple_of: Union[int, float, Decimal]) -> None: - super().__init__(multiple_of=multiple_of) - - -class DecimalError(PydanticTypeError): - msg_template = 'value is not a valid decimal' - - -class DecimalIsNotFiniteError(PydanticValueError): - code = 'decimal.not_finite' - msg_template = 'value is not a valid decimal' - - -class DecimalMaxDigitsError(PydanticValueError): - code = 'decimal.max_digits' - msg_template = 'ensure that there are no more than {max_digits} digits in total' - - def __init__(self, *, max_digits: int) -> None: - super().__init__(max_digits=max_digits) - - -class DecimalMaxPlacesError(PydanticValueError): - code = 'decimal.max_places' - msg_template = 'ensure that there are no more than {decimal_places} decimal places' - - def __init__(self, *, decimal_places: int) -> None: - super().__init__(decimal_places=decimal_places) - - -class DecimalWholeDigitsError(PydanticValueError): - code = 'decimal.whole_digits' - msg_template = 'ensure that there are no more than {whole_digits} digits before the decimal point' - - def __init__(self, *, whole_digits: int) -> None: - super().__init__(whole_digits=whole_digits) - - -class DateTimeError(PydanticValueError): - msg_template = 'invalid datetime format' - - -class DateError(PydanticValueError): - msg_template = 'invalid date format' - - -class DateNotInThePastError(PydanticValueError): - code = 'date.not_in_the_past' - msg_template = 'date is not in the past' - - -class DateNotInTheFutureError(PydanticValueError): - code = 'date.not_in_the_future' - msg_template = 'date is not in the future' - - -class TimeError(PydanticValueError): - msg_template = 'invalid time format' - - -class DurationError(PydanticValueError): - msg_template = 'invalid duration format' - - -class HashableError(PydanticTypeError): - msg_template = 'value is not a valid hashable' - - -class UUIDError(PydanticTypeError): - msg_template = 'value is not a valid uuid' - - -class UUIDVersionError(PydanticValueError): - code = 'uuid.version' - msg_template = 'uuid version {required_version} expected' - - def __init__(self, *, required_version: int) -> None: - super().__init__(required_version=required_version) - - -class ArbitraryTypeError(PydanticTypeError): - code = 'arbitrary_type' - msg_template = 'instance of {expected_arbitrary_type} expected' - - def __init__(self, *, expected_arbitrary_type: Type[Any]) -> None: - super().__init__(expected_arbitrary_type=display_as_type(expected_arbitrary_type)) - - -class ClassError(PydanticTypeError): - code = 'class' - msg_template = 'a class is expected' - - -class SubclassError(PydanticTypeError): - code = 'subclass' - msg_template = 'subclass of {expected_class} expected' - - def __init__(self, *, expected_class: Type[Any]) -> None: - super().__init__(expected_class=display_as_type(expected_class)) - - -class JsonError(PydanticValueError): - msg_template = 'Invalid JSON' - - -class JsonTypeError(PydanticTypeError): - code = 'json' - msg_template = 'JSON object must be str, bytes or bytearray' - - -class PatternError(PydanticValueError): - code = 'regex_pattern' - msg_template = 'Invalid regular expression' - - -class DataclassTypeError(PydanticTypeError): - code = 'dataclass' - msg_template = 'instance of {class_name}, tuple or dict expected' - - -class CallableError(PydanticTypeError): - msg_template = '{value} is not callable' - - -class EnumError(PydanticTypeError): - code = 'enum_instance' - msg_template = '{value} is not a valid Enum instance' - - -class IntEnumError(PydanticTypeError): - code = 'int_enum_instance' - msg_template = '{value} is not a valid IntEnum instance' - - -class IPvAnyAddressError(PydanticValueError): - msg_template = 'value is not a valid IPv4 or IPv6 address' - - -class IPvAnyInterfaceError(PydanticValueError): - msg_template = 'value is not a valid IPv4 or IPv6 interface' - - -class IPvAnyNetworkError(PydanticValueError): - msg_template = 'value is not a valid IPv4 or IPv6 network' - - -class IPv4AddressError(PydanticValueError): - msg_template = 'value is not a valid IPv4 address' - - -class IPv6AddressError(PydanticValueError): - msg_template = 'value is not a valid IPv6 address' - - -class IPv4NetworkError(PydanticValueError): - msg_template = 'value is not a valid IPv4 network' - - -class IPv6NetworkError(PydanticValueError): - msg_template = 'value is not a valid IPv6 network' - - -class IPv4InterfaceError(PydanticValueError): - msg_template = 'value is not a valid IPv4 interface' - - -class IPv6InterfaceError(PydanticValueError): - msg_template = 'value is not a valid IPv6 interface' - - -class ColorError(PydanticValueError): - msg_template = 'value is not a valid color: {reason}' - - -class StrictBoolError(PydanticValueError): - msg_template = 'value is not a valid boolean' - - -class NotDigitError(PydanticValueError): - code = 'payment_card_number.digits' - msg_template = 'card number is not all digits' - - -class LuhnValidationError(PydanticValueError): - code = 'payment_card_number.luhn_check' - msg_template = 'card number is not luhn valid' - - -class InvalidLengthForBrand(PydanticValueError): - code = 'payment_card_number.invalid_length_for_brand' - msg_template = 'Length for a {brand} card must be {required_length}' - - -class InvalidByteSize(PydanticValueError): - msg_template = 'could not parse value and unit from byte string' - - -class InvalidByteSizeUnit(PydanticValueError): - msg_template = 'could not interpret byte unit: {unit}' - - -class MissingDiscriminator(PydanticValueError): - code = 'discriminated_union.missing_discriminator' - msg_template = 'Discriminator {discriminator_key!r} is missing in value' - - -class InvalidDiscriminator(PydanticValueError): - code = 'discriminated_union.invalid_discriminator' - msg_template = ( - 'No match for discriminator {discriminator_key!r} and value {discriminator_value!r} ' - '(allowed values: {allowed_values})' - ) - - def __init__(self, *, discriminator_key: str, discriminator_value: Any, allowed_values: Sequence[Any]) -> None: - super().__init__( - discriminator_key=discriminator_key, - discriminator_value=discriminator_value, - allowed_values=', '.join(map(repr, allowed_values)), - ) diff --git a/functions/python/data-processor/pydantic/v1/fields.py b/functions/python/data-processor/pydantic/v1/fields.py deleted file mode 100644 index 002b60cde..000000000 --- a/functions/python/data-processor/pydantic/v1/fields.py +++ /dev/null @@ -1,1253 +0,0 @@ -import copy -import re -from collections import Counter as CollectionCounter, defaultdict, deque -from collections.abc import Callable, Hashable as CollectionsHashable, Iterable as CollectionsIterable -from typing import ( - TYPE_CHECKING, - Any, - Counter, - DefaultDict, - Deque, - Dict, - ForwardRef, - FrozenSet, - Generator, - Iterable, - Iterator, - List, - Mapping, - Optional, - Pattern, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, -) - -from typing_extensions import Annotated, Final - -from pydantic.v1 import errors as errors_ -from pydantic.v1.class_validators import Validator, make_generic_validator, prep_validators -from pydantic.v1.error_wrappers import ErrorWrapper -from pydantic.v1.errors import ConfigError, InvalidDiscriminator, MissingDiscriminator, NoneIsNotAllowedError -from pydantic.v1.types import Json, JsonWrapper -from pydantic.v1.typing import ( - NoArgAnyCallable, - convert_generics, - display_as_type, - get_args, - get_origin, - is_finalvar, - is_literal_type, - is_new_type, - is_none_type, - is_typeddict, - is_typeddict_special, - is_union, - new_type_supertype, -) -from pydantic.v1.utils import ( - PyObjectStr, - Representation, - ValueItems, - get_discriminator_alias_and_values, - get_unique_discriminator_alias, - lenient_isinstance, - lenient_issubclass, - sequence_like, - smart_deepcopy, -) -from pydantic.v1.validators import constant_validator, dict_validator, find_validators, validate_json - -Required: Any = Ellipsis - -T = TypeVar('T') - - -class UndefinedType: - def __repr__(self) -> str: - return 'PydanticUndefined' - - def __copy__(self: T) -> T: - return self - - def __reduce__(self) -> str: - return 'Undefined' - - def __deepcopy__(self: T, _: Any) -> T: - return self - - -Undefined = UndefinedType() - -if TYPE_CHECKING: - from pydantic.v1.class_validators import ValidatorsList - from pydantic.v1.config import BaseConfig - from pydantic.v1.error_wrappers import ErrorList - from pydantic.v1.types import ModelOrDc - from pydantic.v1.typing import AbstractSetIntStr, MappingIntStrAny, ReprArgs - - ValidateReturn = Tuple[Optional[Any], Optional[ErrorList]] - LocStr = Union[Tuple[Union[int, str], ...], str] - BoolUndefined = Union[bool, UndefinedType] - - -class FieldInfo(Representation): - """ - Captures extra information about a field. - """ - - __slots__ = ( - 'default', - 'default_factory', - 'alias', - 'alias_priority', - 'title', - 'description', - 'exclude', - 'include', - 'const', - 'gt', - 'ge', - 'lt', - 'le', - 'multiple_of', - 'allow_inf_nan', - 'max_digits', - 'decimal_places', - 'min_items', - 'max_items', - 'unique_items', - 'min_length', - 'max_length', - 'allow_mutation', - 'repr', - 'regex', - 'discriminator', - 'extra', - ) - - # field constraints with the default value, it's also used in update_from_config below - __field_constraints__ = { - 'min_length': None, - 'max_length': None, - 'regex': None, - 'gt': None, - 'lt': None, - 'ge': None, - 'le': None, - 'multiple_of': None, - 'allow_inf_nan': None, - 'max_digits': None, - 'decimal_places': None, - 'min_items': None, - 'max_items': None, - 'unique_items': None, - 'allow_mutation': True, - } - - def __init__(self, default: Any = Undefined, **kwargs: Any) -> None: - self.default = default - self.default_factory = kwargs.pop('default_factory', None) - self.alias = kwargs.pop('alias', None) - self.alias_priority = kwargs.pop('alias_priority', 2 if self.alias is not None else None) - self.title = kwargs.pop('title', None) - self.description = kwargs.pop('description', None) - self.exclude = kwargs.pop('exclude', None) - self.include = kwargs.pop('include', None) - self.const = kwargs.pop('const', None) - self.gt = kwargs.pop('gt', None) - self.ge = kwargs.pop('ge', None) - self.lt = kwargs.pop('lt', None) - self.le = kwargs.pop('le', None) - self.multiple_of = kwargs.pop('multiple_of', None) - self.allow_inf_nan = kwargs.pop('allow_inf_nan', None) - self.max_digits = kwargs.pop('max_digits', None) - self.decimal_places = kwargs.pop('decimal_places', None) - self.min_items = kwargs.pop('min_items', None) - self.max_items = kwargs.pop('max_items', None) - self.unique_items = kwargs.pop('unique_items', None) - self.min_length = kwargs.pop('min_length', None) - self.max_length = kwargs.pop('max_length', None) - self.allow_mutation = kwargs.pop('allow_mutation', True) - self.regex = kwargs.pop('regex', None) - self.discriminator = kwargs.pop('discriminator', None) - self.repr = kwargs.pop('repr', True) - self.extra = kwargs - - def __repr_args__(self) -> 'ReprArgs': - field_defaults_to_hide: Dict[str, Any] = { - 'repr': True, - **self.__field_constraints__, - } - - attrs = ((s, getattr(self, s)) for s in self.__slots__) - return [(a, v) for a, v in attrs if v != field_defaults_to_hide.get(a, None)] - - def get_constraints(self) -> Set[str]: - """ - Gets the constraints set on the field by comparing the constraint value with its default value - - :return: the constraints set on field_info - """ - return {attr for attr, default in self.__field_constraints__.items() if getattr(self, attr) != default} - - def update_from_config(self, from_config: Dict[str, Any]) -> None: - """ - Update this FieldInfo based on a dict from get_field_info, only fields which have not been set are dated. - """ - for attr_name, value in from_config.items(): - try: - current_value = getattr(self, attr_name) - except AttributeError: - # attr_name is not an attribute of FieldInfo, it should therefore be added to extra - # (except if extra already has this value!) - self.extra.setdefault(attr_name, value) - else: - if current_value is self.__field_constraints__.get(attr_name, None): - setattr(self, attr_name, value) - elif attr_name == 'exclude': - self.exclude = ValueItems.merge(value, current_value) - elif attr_name == 'include': - self.include = ValueItems.merge(value, current_value, intersect=True) - - def _validate(self) -> None: - if self.default is not Undefined and self.default_factory is not None: - raise ValueError('cannot specify both default and default_factory') - - -def Field( - default: Any = Undefined, - *, - default_factory: Optional[NoArgAnyCallable] = None, - alias: Optional[str] = None, - title: Optional[str] = None, - description: Optional[str] = None, - exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny', Any]] = None, - include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny', Any]] = None, - const: Optional[bool] = None, - gt: Optional[float] = None, - ge: Optional[float] = None, - lt: Optional[float] = None, - le: Optional[float] = None, - multiple_of: Optional[float] = None, - allow_inf_nan: Optional[bool] = None, - max_digits: Optional[int] = None, - decimal_places: Optional[int] = None, - min_items: Optional[int] = None, - max_items: Optional[int] = None, - unique_items: Optional[bool] = None, - min_length: Optional[int] = None, - max_length: Optional[int] = None, - allow_mutation: bool = True, - regex: Optional[str] = None, - discriminator: Optional[str] = None, - repr: bool = True, - **extra: Any, -) -> Any: - """ - Used to provide extra information about a field, either for the model schema or complex validation. Some arguments - apply only to number fields (``int``, ``float``, ``Decimal``) and some apply only to ``str``. - - :param default: since this is replacing the field’s default, its first argument is used - to set the default, use ellipsis (``...``) to indicate the field is required - :param default_factory: callable that will be called when a default value is needed for this field - If both `default` and `default_factory` are set, an error is raised. - :param alias: the public name of the field - :param title: can be any string, used in the schema - :param description: can be any string, used in the schema - :param exclude: exclude this field while dumping. - Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method. - :param include: include this field while dumping. - Takes same values as the ``include`` and ``exclude`` arguments on the ``.dict`` method. - :param const: this field is required and *must* take it's default value - :param gt: only applies to numbers, requires the field to be "greater than". The schema - will have an ``exclusiveMinimum`` validation keyword - :param ge: only applies to numbers, requires the field to be "greater than or equal to". The - schema will have a ``minimum`` validation keyword - :param lt: only applies to numbers, requires the field to be "less than". The schema - will have an ``exclusiveMaximum`` validation keyword - :param le: only applies to numbers, requires the field to be "less than or equal to". The - schema will have a ``maximum`` validation keyword - :param multiple_of: only applies to numbers, requires the field to be "a multiple of". The - schema will have a ``multipleOf`` validation keyword - :param allow_inf_nan: only applies to numbers, allows the field to be NaN or infinity (+inf or -inf), - which is a valid Python float. Default True, set to False for compatibility with JSON. - :param max_digits: only applies to Decimals, requires the field to have a maximum number - of digits within the decimal. It does not include a zero before the decimal point or trailing decimal zeroes. - :param decimal_places: only applies to Decimals, requires the field to have at most a number of decimal places - allowed. It does not include trailing decimal zeroes. - :param min_items: only applies to lists, requires the field to have a minimum number of - elements. The schema will have a ``minItems`` validation keyword - :param max_items: only applies to lists, requires the field to have a maximum number of - elements. The schema will have a ``maxItems`` validation keyword - :param unique_items: only applies to lists, requires the field not to have duplicated - elements. The schema will have a ``uniqueItems`` validation keyword - :param min_length: only applies to strings, requires the field to have a minimum length. The - schema will have a ``minLength`` validation keyword - :param max_length: only applies to strings, requires the field to have a maximum length. The - schema will have a ``maxLength`` validation keyword - :param allow_mutation: a boolean which defaults to True. When False, the field raises a TypeError if the field is - assigned on an instance. The BaseModel Config must set validate_assignment to True - :param regex: only applies to strings, requires the field match against a regular expression - pattern string. The schema will have a ``pattern`` validation keyword - :param discriminator: only useful with a (discriminated a.k.a. tagged) `Union` of sub models with a common field. - The `discriminator` is the name of this common field to shorten validation and improve generated schema - :param repr: show this field in the representation - :param **extra: any additional keyword arguments will be added as is to the schema - """ - field_info = FieldInfo( - default, - default_factory=default_factory, - alias=alias, - title=title, - description=description, - exclude=exclude, - include=include, - const=const, - gt=gt, - ge=ge, - lt=lt, - le=le, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - max_digits=max_digits, - decimal_places=decimal_places, - min_items=min_items, - max_items=max_items, - unique_items=unique_items, - min_length=min_length, - max_length=max_length, - allow_mutation=allow_mutation, - regex=regex, - discriminator=discriminator, - repr=repr, - **extra, - ) - field_info._validate() - return field_info - - -# used to be an enum but changed to int's for small performance improvement as less access overhead -SHAPE_SINGLETON = 1 -SHAPE_LIST = 2 -SHAPE_SET = 3 -SHAPE_MAPPING = 4 -SHAPE_TUPLE = 5 -SHAPE_TUPLE_ELLIPSIS = 6 -SHAPE_SEQUENCE = 7 -SHAPE_FROZENSET = 8 -SHAPE_ITERABLE = 9 -SHAPE_GENERIC = 10 -SHAPE_DEQUE = 11 -SHAPE_DICT = 12 -SHAPE_DEFAULTDICT = 13 -SHAPE_COUNTER = 14 -SHAPE_NAME_LOOKUP = { - SHAPE_LIST: 'List[{}]', - SHAPE_SET: 'Set[{}]', - SHAPE_TUPLE_ELLIPSIS: 'Tuple[{}, ...]', - SHAPE_SEQUENCE: 'Sequence[{}]', - SHAPE_FROZENSET: 'FrozenSet[{}]', - SHAPE_ITERABLE: 'Iterable[{}]', - SHAPE_DEQUE: 'Deque[{}]', - SHAPE_DICT: 'Dict[{}]', - SHAPE_DEFAULTDICT: 'DefaultDict[{}]', - SHAPE_COUNTER: 'Counter[{}]', -} - -MAPPING_LIKE_SHAPES: Set[int] = {SHAPE_DEFAULTDICT, SHAPE_DICT, SHAPE_MAPPING, SHAPE_COUNTER} - - -class ModelField(Representation): - __slots__ = ( - 'type_', - 'outer_type_', - 'annotation', - 'sub_fields', - 'sub_fields_mapping', - 'key_field', - 'validators', - 'pre_validators', - 'post_validators', - 'default', - 'default_factory', - 'required', - 'final', - 'model_config', - 'name', - 'alias', - 'has_alias', - 'field_info', - 'discriminator_key', - 'discriminator_alias', - 'validate_always', - 'allow_none', - 'shape', - 'class_validators', - 'parse_json', - ) - - def __init__( - self, - *, - name: str, - type_: Type[Any], - class_validators: Optional[Dict[str, Validator]], - model_config: Type['BaseConfig'], - default: Any = None, - default_factory: Optional[NoArgAnyCallable] = None, - required: 'BoolUndefined' = Undefined, - final: bool = False, - alias: Optional[str] = None, - field_info: Optional[FieldInfo] = None, - ) -> None: - self.name: str = name - self.has_alias: bool = alias is not None - self.alias: str = alias if alias is not None else name - self.annotation = type_ - self.type_: Any = convert_generics(type_) - self.outer_type_: Any = type_ - self.class_validators = class_validators or {} - self.default: Any = default - self.default_factory: Optional[NoArgAnyCallable] = default_factory - self.required: 'BoolUndefined' = required - self.final: bool = final - self.model_config = model_config - self.field_info: FieldInfo = field_info or FieldInfo(default) - self.discriminator_key: Optional[str] = self.field_info.discriminator - self.discriminator_alias: Optional[str] = self.discriminator_key - - self.allow_none: bool = False - self.validate_always: bool = False - self.sub_fields: Optional[List[ModelField]] = None - self.sub_fields_mapping: Optional[Dict[str, 'ModelField']] = None # used for discriminated union - self.key_field: Optional[ModelField] = None - self.validators: 'ValidatorsList' = [] - self.pre_validators: Optional['ValidatorsList'] = None - self.post_validators: Optional['ValidatorsList'] = None - self.parse_json: bool = False - self.shape: int = SHAPE_SINGLETON - self.model_config.prepare_field(self) - self.prepare() - - def get_default(self) -> Any: - return smart_deepcopy(self.default) if self.default_factory is None else self.default_factory() - - @staticmethod - def _get_field_info( - field_name: str, annotation: Any, value: Any, config: Type['BaseConfig'] - ) -> Tuple[FieldInfo, Any]: - """ - Get a FieldInfo from a root typing.Annotated annotation, value, or config default. - - The FieldInfo may be set in typing.Annotated or the value, but not both. If neither contain - a FieldInfo, a new one will be created using the config. - - :param field_name: name of the field for use in error messages - :param annotation: a type hint such as `str` or `Annotated[str, Field(..., min_length=5)]` - :param value: the field's assigned value - :param config: the model's config object - :return: the FieldInfo contained in the `annotation`, the value, or a new one from the config. - """ - field_info_from_config = config.get_field_info(field_name) - - field_info = None - if get_origin(annotation) is Annotated: - field_infos = [arg for arg in get_args(annotation)[1:] if isinstance(arg, FieldInfo)] - if len(field_infos) > 1: - raise ValueError(f'cannot specify multiple `Annotated` `Field`s for {field_name!r}') - field_info = next(iter(field_infos), None) - if field_info is not None: - field_info = copy.copy(field_info) - field_info.update_from_config(field_info_from_config) - if field_info.default not in (Undefined, Required): - raise ValueError(f'`Field` default cannot be set in `Annotated` for {field_name!r}') - if value is not Undefined and value is not Required: - # check also `Required` because of `validate_arguments` that sets `...` as default value - field_info.default = value - - if isinstance(value, FieldInfo): - if field_info is not None: - raise ValueError(f'cannot specify `Annotated` and value `Field`s together for {field_name!r}') - field_info = value - field_info.update_from_config(field_info_from_config) - elif field_info is None: - field_info = FieldInfo(value, **field_info_from_config) - value = None if field_info.default_factory is not None else field_info.default - field_info._validate() - return field_info, value - - @classmethod - def infer( - cls, - *, - name: str, - value: Any, - annotation: Any, - class_validators: Optional[Dict[str, Validator]], - config: Type['BaseConfig'], - ) -> 'ModelField': - from pydantic.v1.schema import get_annotation_from_field_info - - field_info, value = cls._get_field_info(name, annotation, value, config) - required: 'BoolUndefined' = Undefined - if value is Required: - required = True - value = None - elif value is not Undefined: - required = False - annotation = get_annotation_from_field_info(annotation, field_info, name, config.validate_assignment) - - return cls( - name=name, - type_=annotation, - alias=field_info.alias, - class_validators=class_validators, - default=value, - default_factory=field_info.default_factory, - required=required, - model_config=config, - field_info=field_info, - ) - - def set_config(self, config: Type['BaseConfig']) -> None: - self.model_config = config - info_from_config = config.get_field_info(self.name) - config.prepare_field(self) - new_alias = info_from_config.get('alias') - new_alias_priority = info_from_config.get('alias_priority') or 0 - if new_alias and new_alias_priority >= (self.field_info.alias_priority or 0): - self.field_info.alias = new_alias - self.field_info.alias_priority = new_alias_priority - self.alias = new_alias - new_exclude = info_from_config.get('exclude') - if new_exclude is not None: - self.field_info.exclude = ValueItems.merge(self.field_info.exclude, new_exclude) - new_include = info_from_config.get('include') - if new_include is not None: - self.field_info.include = ValueItems.merge(self.field_info.include, new_include, intersect=True) - - @property - def alt_alias(self) -> bool: - return self.name != self.alias - - def prepare(self) -> None: - """ - Prepare the field but inspecting self.default, self.type_ etc. - - Note: this method is **not** idempotent (because _type_analysis is not idempotent), - e.g. calling it it multiple times may modify the field and configure it incorrectly. - """ - self._set_default_and_type() - if self.type_.__class__ is ForwardRef or self.type_.__class__ is DeferredType: - # self.type_ is currently a ForwardRef and there's nothing we can do now, - # user will need to call model.update_forward_refs() - return - - self._type_analysis() - if self.required is Undefined: - self.required = True - if self.default is Undefined and self.default_factory is None: - self.default = None - self.populate_validators() - - def _set_default_and_type(self) -> None: - """ - Set the default value, infer the type if needed and check if `None` value is valid. - """ - if self.default_factory is not None: - if self.type_ is Undefined: - raise errors_.ConfigError( - f'you need to set the type of field {self.name!r} when using `default_factory`' - ) - return - - default_value = self.get_default() - - if default_value is not None and self.type_ is Undefined: - self.type_ = default_value.__class__ - self.outer_type_ = self.type_ - self.annotation = self.type_ - - if self.type_ is Undefined: - raise errors_.ConfigError(f'unable to infer type for attribute "{self.name}"') - - if self.required is False and default_value is None: - self.allow_none = True - - def _type_analysis(self) -> None: # noqa: C901 (ignore complexity) - # typing interface is horrible, we have to do some ugly checks - if lenient_issubclass(self.type_, JsonWrapper): - self.type_ = self.type_.inner_type - self.parse_json = True - elif lenient_issubclass(self.type_, Json): - self.type_ = Any - self.parse_json = True - elif isinstance(self.type_, TypeVar): - if self.type_.__bound__: - self.type_ = self.type_.__bound__ - elif self.type_.__constraints__: - self.type_ = Union[self.type_.__constraints__] - else: - self.type_ = Any - elif is_new_type(self.type_): - self.type_ = new_type_supertype(self.type_) - - if self.type_ is Any or self.type_ is object: - if self.required is Undefined: - self.required = False - self.allow_none = True - return - elif self.type_ is Pattern or self.type_ is re.Pattern: - # python 3.7 only, Pattern is a typing object but without sub fields - return - elif is_literal_type(self.type_): - return - elif is_typeddict(self.type_): - return - - if is_finalvar(self.type_): - self.final = True - - if self.type_ is Final: - self.type_ = Any - else: - self.type_ = get_args(self.type_)[0] - - self._type_analysis() - return - - origin = get_origin(self.type_) - - if origin is Annotated or is_typeddict_special(origin): - self.type_ = get_args(self.type_)[0] - self._type_analysis() - return - - if self.discriminator_key is not None and not is_union(origin): - raise TypeError('`discriminator` can only be used with `Union` type with more than one variant') - - # add extra check for `collections.abc.Hashable` for python 3.10+ where origin is not `None` - if origin is None or origin is CollectionsHashable: - # field is not "typing" object eg. Union, Dict, List etc. - # allow None for virtual superclasses of NoneType, e.g. Hashable - if isinstance(self.type_, type) and isinstance(None, self.type_): - self.allow_none = True - return - elif origin is Callable: - return - elif is_union(origin): - types_ = [] - for type_ in get_args(self.type_): - if is_none_type(type_) or type_ is Any or type_ is object: - if self.required is Undefined: - self.required = False - self.allow_none = True - if is_none_type(type_): - continue - types_.append(type_) - - if len(types_) == 1: - # Optional[] - self.type_ = types_[0] - # this is the one case where the "outer type" isn't just the original type - self.outer_type_ = self.type_ - # re-run to correctly interpret the new self.type_ - self._type_analysis() - else: - self.sub_fields = [self._create_sub_type(t, f'{self.name}_{display_as_type(t)}') for t in types_] - - if self.discriminator_key is not None: - self.prepare_discriminated_union_sub_fields() - return - elif issubclass(origin, Tuple): # type: ignore - # origin == Tuple without item type - args = get_args(self.type_) - if not args: # plain tuple - self.type_ = Any - self.shape = SHAPE_TUPLE_ELLIPSIS - elif len(args) == 2 and args[1] is Ellipsis: # e.g. Tuple[int, ...] - self.type_ = args[0] - self.shape = SHAPE_TUPLE_ELLIPSIS - self.sub_fields = [self._create_sub_type(args[0], f'{self.name}_0')] - elif args == ((),): # Tuple[()] means empty tuple - self.shape = SHAPE_TUPLE - self.type_ = Any - self.sub_fields = [] - else: - self.shape = SHAPE_TUPLE - self.sub_fields = [self._create_sub_type(t, f'{self.name}_{i}') for i, t in enumerate(args)] - return - elif issubclass(origin, List): - # Create self validators - get_validators = getattr(self.type_, '__get_validators__', None) - if get_validators: - self.class_validators.update( - {f'list_{i}': Validator(validator, pre=True) for i, validator in enumerate(get_validators())} - ) - - self.type_ = get_args(self.type_)[0] - self.shape = SHAPE_LIST - elif issubclass(origin, Set): - # Create self validators - get_validators = getattr(self.type_, '__get_validators__', None) - if get_validators: - self.class_validators.update( - {f'set_{i}': Validator(validator, pre=True) for i, validator in enumerate(get_validators())} - ) - - self.type_ = get_args(self.type_)[0] - self.shape = SHAPE_SET - elif issubclass(origin, FrozenSet): - # Create self validators - get_validators = getattr(self.type_, '__get_validators__', None) - if get_validators: - self.class_validators.update( - {f'frozenset_{i}': Validator(validator, pre=True) for i, validator in enumerate(get_validators())} - ) - - self.type_ = get_args(self.type_)[0] - self.shape = SHAPE_FROZENSET - elif issubclass(origin, Deque): - self.type_ = get_args(self.type_)[0] - self.shape = SHAPE_DEQUE - elif issubclass(origin, Sequence): - self.type_ = get_args(self.type_)[0] - self.shape = SHAPE_SEQUENCE - # priority to most common mapping: dict - elif origin is dict or origin is Dict: - self.key_field = self._create_sub_type(get_args(self.type_)[0], 'key_' + self.name, for_keys=True) - self.type_ = get_args(self.type_)[1] - self.shape = SHAPE_DICT - elif issubclass(origin, DefaultDict): - self.key_field = self._create_sub_type(get_args(self.type_)[0], 'key_' + self.name, for_keys=True) - self.type_ = get_args(self.type_)[1] - self.shape = SHAPE_DEFAULTDICT - elif issubclass(origin, Counter): - self.key_field = self._create_sub_type(get_args(self.type_)[0], 'key_' + self.name, for_keys=True) - self.type_ = int - self.shape = SHAPE_COUNTER - elif issubclass(origin, Mapping): - self.key_field = self._create_sub_type(get_args(self.type_)[0], 'key_' + self.name, for_keys=True) - self.type_ = get_args(self.type_)[1] - self.shape = SHAPE_MAPPING - # Equality check as almost everything inherits form Iterable, including str - # check for Iterable and CollectionsIterable, as it could receive one even when declared with the other - elif origin in {Iterable, CollectionsIterable}: - self.type_ = get_args(self.type_)[0] - self.shape = SHAPE_ITERABLE - self.sub_fields = [self._create_sub_type(self.type_, f'{self.name}_type')] - elif issubclass(origin, Type): # type: ignore - return - elif hasattr(origin, '__get_validators__') or self.model_config.arbitrary_types_allowed: - # Is a Pydantic-compatible generic that handles itself - # or we have arbitrary_types_allowed = True - self.shape = SHAPE_GENERIC - self.sub_fields = [self._create_sub_type(t, f'{self.name}_{i}') for i, t in enumerate(get_args(self.type_))] - self.type_ = origin - return - else: - raise TypeError(f'Fields of type "{origin}" are not supported.') - - # type_ has been refined eg. as the type of a List and sub_fields needs to be populated - self.sub_fields = [self._create_sub_type(self.type_, '_' + self.name)] - - def prepare_discriminated_union_sub_fields(self) -> None: - """ - Prepare the mapping -> and update `sub_fields` - Note that this process can be aborted if a `ForwardRef` is encountered - """ - assert self.discriminator_key is not None - - if self.type_.__class__ is DeferredType: - return - - assert self.sub_fields is not None - sub_fields_mapping: Dict[str, 'ModelField'] = {} - all_aliases: Set[str] = set() - - for sub_field in self.sub_fields: - t = sub_field.type_ - if t.__class__ is ForwardRef: - # Stopping everything...will need to call `update_forward_refs` - return - - alias, discriminator_values = get_discriminator_alias_and_values(t, self.discriminator_key) - all_aliases.add(alias) - for discriminator_value in discriminator_values: - sub_fields_mapping[discriminator_value] = sub_field - - self.sub_fields_mapping = sub_fields_mapping - self.discriminator_alias = get_unique_discriminator_alias(all_aliases, self.discriminator_key) - - def _create_sub_type(self, type_: Type[Any], name: str, *, for_keys: bool = False) -> 'ModelField': - if for_keys: - class_validators = None - else: - # validators for sub items should not have `each_item` as we want to check only the first sublevel - class_validators = { - k: Validator( - func=v.func, - pre=v.pre, - each_item=False, - always=v.always, - check_fields=v.check_fields, - skip_on_failure=v.skip_on_failure, - ) - for k, v in self.class_validators.items() - if v.each_item - } - - field_info, _ = self._get_field_info(name, type_, None, self.model_config) - - return self.__class__( - type_=type_, - name=name, - class_validators=class_validators, - model_config=self.model_config, - field_info=field_info, - ) - - def populate_validators(self) -> None: - """ - Prepare self.pre_validators, self.validators, and self.post_validators based on self.type_'s __get_validators__ - and class validators. This method should be idempotent, e.g. it should be safe to call multiple times - without mis-configuring the field. - """ - self.validate_always = getattr(self.type_, 'validate_always', False) or any( - v.always for v in self.class_validators.values() - ) - - class_validators_ = self.class_validators.values() - if not self.sub_fields or self.shape == SHAPE_GENERIC: - get_validators = getattr(self.type_, '__get_validators__', None) - v_funcs = ( - *[v.func for v in class_validators_ if v.each_item and v.pre], - *(get_validators() if get_validators else list(find_validators(self.type_, self.model_config))), - *[v.func for v in class_validators_ if v.each_item and not v.pre], - ) - self.validators = prep_validators(v_funcs) - - self.pre_validators = [] - self.post_validators = [] - - if self.field_info and self.field_info.const: - self.post_validators.append(make_generic_validator(constant_validator)) - - if class_validators_: - self.pre_validators += prep_validators(v.func for v in class_validators_ if not v.each_item and v.pre) - self.post_validators += prep_validators(v.func for v in class_validators_ if not v.each_item and not v.pre) - - if self.parse_json: - self.pre_validators.append(make_generic_validator(validate_json)) - - self.pre_validators = self.pre_validators or None - self.post_validators = self.post_validators or None - - def validate( - self, v: Any, values: Dict[str, Any], *, loc: 'LocStr', cls: Optional['ModelOrDc'] = None - ) -> 'ValidateReturn': - assert self.type_.__class__ is not DeferredType - - if self.type_.__class__ is ForwardRef: - assert cls is not None - raise ConfigError( - f'field "{self.name}" not yet prepared so type is still a ForwardRef, ' - f'you might need to call {cls.__name__}.update_forward_refs().' - ) - - errors: Optional['ErrorList'] - if self.pre_validators: - v, errors = self._apply_validators(v, values, loc, cls, self.pre_validators) - if errors: - return v, errors - - if v is None: - if is_none_type(self.type_): - # keep validating - pass - elif self.allow_none: - if self.post_validators: - return self._apply_validators(v, values, loc, cls, self.post_validators) - else: - return None, None - else: - return v, ErrorWrapper(NoneIsNotAllowedError(), loc) - - if self.shape == SHAPE_SINGLETON: - v, errors = self._validate_singleton(v, values, loc, cls) - elif self.shape in MAPPING_LIKE_SHAPES: - v, errors = self._validate_mapping_like(v, values, loc, cls) - elif self.shape == SHAPE_TUPLE: - v, errors = self._validate_tuple(v, values, loc, cls) - elif self.shape == SHAPE_ITERABLE: - v, errors = self._validate_iterable(v, values, loc, cls) - elif self.shape == SHAPE_GENERIC: - v, errors = self._apply_validators(v, values, loc, cls, self.validators) - else: - # sequence, list, set, generator, tuple with ellipsis, frozen set - v, errors = self._validate_sequence_like(v, values, loc, cls) - - if not errors and self.post_validators: - v, errors = self._apply_validators(v, values, loc, cls, self.post_validators) - return v, errors - - def _validate_sequence_like( # noqa: C901 (ignore complexity) - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] - ) -> 'ValidateReturn': - """ - Validate sequence-like containers: lists, tuples, sets and generators - Note that large if-else blocks are necessary to enable Cython - optimization, which is why we disable the complexity check above. - """ - if not sequence_like(v): - e: errors_.PydanticTypeError - if self.shape == SHAPE_LIST: - e = errors_.ListError() - elif self.shape in (SHAPE_TUPLE, SHAPE_TUPLE_ELLIPSIS): - e = errors_.TupleError() - elif self.shape == SHAPE_SET: - e = errors_.SetError() - elif self.shape == SHAPE_FROZENSET: - e = errors_.FrozenSetError() - else: - e = errors_.SequenceError() - return v, ErrorWrapper(e, loc) - - loc = loc if isinstance(loc, tuple) else (loc,) - result = [] - errors: List[ErrorList] = [] - for i, v_ in enumerate(v): - v_loc = *loc, i - r, ee = self._validate_singleton(v_, values, v_loc, cls) - if ee: - errors.append(ee) - else: - result.append(r) - - if errors: - return v, errors - - converted: Union[List[Any], Set[Any], FrozenSet[Any], Tuple[Any, ...], Iterator[Any], Deque[Any]] = result - - if self.shape == SHAPE_SET: - converted = set(result) - elif self.shape == SHAPE_FROZENSET: - converted = frozenset(result) - elif self.shape == SHAPE_TUPLE_ELLIPSIS: - converted = tuple(result) - elif self.shape == SHAPE_DEQUE: - converted = deque(result, maxlen=getattr(v, 'maxlen', None)) - elif self.shape == SHAPE_SEQUENCE: - if isinstance(v, tuple): - converted = tuple(result) - elif isinstance(v, set): - converted = set(result) - elif isinstance(v, Generator): - converted = iter(result) - elif isinstance(v, deque): - converted = deque(result, maxlen=getattr(v, 'maxlen', None)) - return converted, None - - def _validate_iterable( - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] - ) -> 'ValidateReturn': - """ - Validate Iterables. - - This intentionally doesn't validate values to allow infinite generators. - """ - - try: - iterable = iter(v) - except TypeError: - return v, ErrorWrapper(errors_.IterableError(), loc) - return iterable, None - - def _validate_tuple( - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] - ) -> 'ValidateReturn': - e: Optional[Exception] = None - if not sequence_like(v): - e = errors_.TupleError() - else: - actual_length, expected_length = len(v), len(self.sub_fields) # type: ignore - if actual_length != expected_length: - e = errors_.TupleLengthError(actual_length=actual_length, expected_length=expected_length) - - if e: - return v, ErrorWrapper(e, loc) - - loc = loc if isinstance(loc, tuple) else (loc,) - result = [] - errors: List[ErrorList] = [] - for i, (v_, field) in enumerate(zip(v, self.sub_fields)): # type: ignore - v_loc = *loc, i - r, ee = field.validate(v_, values, loc=v_loc, cls=cls) - if ee: - errors.append(ee) - else: - result.append(r) - - if errors: - return v, errors - else: - return tuple(result), None - - def _validate_mapping_like( - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] - ) -> 'ValidateReturn': - try: - v_iter = dict_validator(v) - except TypeError as exc: - return v, ErrorWrapper(exc, loc) - - loc = loc if isinstance(loc, tuple) else (loc,) - result, errors = {}, [] - for k, v_ in v_iter.items(): - v_loc = *loc, '__key__' - key_result, key_errors = self.key_field.validate(k, values, loc=v_loc, cls=cls) # type: ignore - if key_errors: - errors.append(key_errors) - continue - - v_loc = *loc, k - value_result, value_errors = self._validate_singleton(v_, values, v_loc, cls) - if value_errors: - errors.append(value_errors) - continue - - result[key_result] = value_result - if errors: - return v, errors - elif self.shape == SHAPE_DICT: - return result, None - elif self.shape == SHAPE_DEFAULTDICT: - return defaultdict(self.type_, result), None - elif self.shape == SHAPE_COUNTER: - return CollectionCounter(result), None - else: - return self._get_mapping_value(v, result), None - - def _get_mapping_value(self, original: T, converted: Dict[Any, Any]) -> Union[T, Dict[Any, Any]]: - """ - When type is `Mapping[KT, KV]` (or another unsupported mapping), we try to avoid - coercing to `dict` unwillingly. - """ - original_cls = original.__class__ - - if original_cls == dict or original_cls == Dict: - return converted - elif original_cls in {defaultdict, DefaultDict}: - return defaultdict(self.type_, converted) - else: - try: - # Counter, OrderedDict, UserDict, ... - return original_cls(converted) # type: ignore - except TypeError: - raise RuntimeError(f'Could not convert dictionary to {original_cls.__name__!r}') from None - - def _validate_singleton( - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] - ) -> 'ValidateReturn': - if self.sub_fields: - if self.discriminator_key is not None: - return self._validate_discriminated_union(v, values, loc, cls) - - errors = [] - - if self.model_config.smart_union and is_union(get_origin(self.type_)): - # 1st pass: check if the value is an exact instance of one of the Union types - # (e.g. to avoid coercing a bool into an int) - for field in self.sub_fields: - if v.__class__ is field.outer_type_: - return v, None - - # 2nd pass: check if the value is an instance of any subclass of the Union types - for field in self.sub_fields: - # This whole logic will be improved later on to support more complex `isinstance` checks - # It will probably be done once a strict mode is added and be something like: - # ``` - # value, error = field.validate(v, values, strict=True) - # if error is None: - # return value, None - # ``` - try: - if isinstance(v, field.outer_type_): - return v, None - except TypeError: - # compound type - if lenient_isinstance(v, get_origin(field.outer_type_)): - value, error = field.validate(v, values, loc=loc, cls=cls) - if not error: - return value, None - - # 1st pass by default or 3rd pass with `smart_union` enabled: - # check if the value can be coerced into one of the Union types - for field in self.sub_fields: - value, error = field.validate(v, values, loc=loc, cls=cls) - if error: - errors.append(error) - else: - return value, None - return v, errors - else: - return self._apply_validators(v, values, loc, cls, self.validators) - - def _validate_discriminated_union( - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'] - ) -> 'ValidateReturn': - assert self.discriminator_key is not None - assert self.discriminator_alias is not None - - try: - try: - discriminator_value = v[self.discriminator_alias] - except KeyError: - if self.model_config.allow_population_by_field_name: - discriminator_value = v[self.discriminator_key] - else: - raise - except KeyError: - return v, ErrorWrapper(MissingDiscriminator(discriminator_key=self.discriminator_key), loc) - except TypeError: - try: - # BaseModel or dataclass - discriminator_value = getattr(v, self.discriminator_key) - except (AttributeError, TypeError): - return v, ErrorWrapper(MissingDiscriminator(discriminator_key=self.discriminator_key), loc) - - if self.sub_fields_mapping is None: - assert cls is not None - raise ConfigError( - f'field "{self.name}" not yet prepared so type is still a ForwardRef, ' - f'you might need to call {cls.__name__}.update_forward_refs().' - ) - - try: - sub_field = self.sub_fields_mapping[discriminator_value] - except (KeyError, TypeError): - # KeyError: `discriminator_value` is not in the dictionary. - # TypeError: `discriminator_value` is unhashable. - assert self.sub_fields_mapping is not None - return v, ErrorWrapper( - InvalidDiscriminator( - discriminator_key=self.discriminator_key, - discriminator_value=discriminator_value, - allowed_values=list(self.sub_fields_mapping), - ), - loc, - ) - else: - if not isinstance(loc, tuple): - loc = (loc,) - return sub_field.validate(v, values, loc=(*loc, display_as_type(sub_field.type_)), cls=cls) - - def _apply_validators( - self, v: Any, values: Dict[str, Any], loc: 'LocStr', cls: Optional['ModelOrDc'], validators: 'ValidatorsList' - ) -> 'ValidateReturn': - for validator in validators: - try: - v = validator(cls, v, values, self, self.model_config) - except (ValueError, TypeError, AssertionError) as exc: - return v, ErrorWrapper(exc, loc) - return v, None - - def is_complex(self) -> bool: - """ - Whether the field is "complex" eg. env variables should be parsed as JSON. - """ - from pydantic.v1.main import BaseModel - - return ( - self.shape != SHAPE_SINGLETON - or hasattr(self.type_, '__pydantic_model__') - or lenient_issubclass(self.type_, (BaseModel, list, set, frozenset, dict)) - ) - - def _type_display(self) -> PyObjectStr: - t = display_as_type(self.type_) - - if self.shape in MAPPING_LIKE_SHAPES: - t = f'Mapping[{display_as_type(self.key_field.type_)}, {t}]' # type: ignore - elif self.shape == SHAPE_TUPLE: - t = 'Tuple[{}]'.format(', '.join(display_as_type(f.type_) for f in self.sub_fields)) # type: ignore - elif self.shape == SHAPE_GENERIC: - assert self.sub_fields - t = '{}[{}]'.format( - display_as_type(self.type_), ', '.join(display_as_type(f.type_) for f in self.sub_fields) - ) - elif self.shape != SHAPE_SINGLETON: - t = SHAPE_NAME_LOOKUP[self.shape].format(t) - - if self.allow_none and (self.shape != SHAPE_SINGLETON or not self.sub_fields): - t = f'Optional[{t}]' - return PyObjectStr(t) - - def __repr_args__(self) -> 'ReprArgs': - args = [('name', self.name), ('type', self._type_display()), ('required', self.required)] - - if not self.required: - if self.default_factory is not None: - args.append(('default_factory', f'')) - else: - args.append(('default', self.default)) - - if self.alt_alias: - args.append(('alias', self.alias)) - return args - - -class ModelPrivateAttr(Representation): - __slots__ = ('default', 'default_factory') - - def __init__(self, default: Any = Undefined, *, default_factory: Optional[NoArgAnyCallable] = None) -> None: - self.default = default - self.default_factory = default_factory - - def get_default(self) -> Any: - return smart_deepcopy(self.default) if self.default_factory is None else self.default_factory() - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and (self.default, self.default_factory) == ( - other.default, - other.default_factory, - ) - - -def PrivateAttr( - default: Any = Undefined, - *, - default_factory: Optional[NoArgAnyCallable] = None, -) -> Any: - """ - Indicates that attribute is only used internally and never mixed with regular fields. - - Types or values of private attrs are not checked by pydantic and it's up to you to keep them relevant. - - Private attrs are stored in model __slots__. - - :param default: the attribute’s default value - :param default_factory: callable that will be called when a default value is needed for this attribute - If both `default` and `default_factory` are set, an error is raised. - """ - if default is not Undefined and default_factory is not None: - raise ValueError('cannot specify both default and default_factory') - - return ModelPrivateAttr( - default, - default_factory=default_factory, - ) - - -class DeferredType: - """ - Used to postpone field preparation, while creating recursive generic models. - """ - - -def is_finalvar_with_default_val(type_: Type[Any], val: Any) -> bool: - return is_finalvar(type_) and val is not Undefined and not isinstance(val, FieldInfo) diff --git a/functions/python/data-processor/pydantic/v1/generics.py b/functions/python/data-processor/pydantic/v1/generics.py deleted file mode 100644 index fa1dcec42..000000000 --- a/functions/python/data-processor/pydantic/v1/generics.py +++ /dev/null @@ -1,400 +0,0 @@ -import functools -import operator -import sys -import types -import typing -from typing import ( - TYPE_CHECKING, - Any, - ClassVar, - Dict, - ForwardRef, - Generic, - Iterator, - List, - Mapping, - Optional, - Tuple, - Type, - TypeVar, - Union, - cast, -) -from weakref import WeakKeyDictionary, WeakValueDictionary - -from typing_extensions import Annotated, Literal as ExtLiteral - -from pydantic.v1.class_validators import gather_all_validators -from pydantic.v1.fields import DeferredType -from pydantic.v1.main import BaseModel, create_model -from pydantic.v1.types import JsonWrapper -from pydantic.v1.typing import display_as_type, get_all_type_hints, get_args, get_origin, typing_base -from pydantic.v1.utils import all_identical, lenient_issubclass - -if sys.version_info >= (3, 8): - from typing import Literal - -GenericModelT = TypeVar('GenericModelT', bound='GenericModel') -TypeVarType = Any # since mypy doesn't allow the use of TypeVar as a type - -CacheKey = Tuple[Type[Any], Any, Tuple[Any, ...]] -Parametrization = Mapping[TypeVarType, Type[Any]] - -# weak dictionaries allow the dynamically created parametrized versions of generic models to get collected -# once they are no longer referenced by the caller. -if sys.version_info >= (3, 9): # Typing for weak dictionaries available at 3.9 - GenericTypesCache = WeakValueDictionary[CacheKey, Type[BaseModel]] - AssignedParameters = WeakKeyDictionary[Type[BaseModel], Parametrization] -else: - GenericTypesCache = WeakValueDictionary - AssignedParameters = WeakKeyDictionary - -# _generic_types_cache is a Mapping from __class_getitem__ arguments to the parametrized version of generic models. -# This ensures multiple calls of e.g. A[B] return always the same class. -_generic_types_cache = GenericTypesCache() - -# _assigned_parameters is a Mapping from parametrized version of generic models to assigned types of parametrizations -# as captured during construction of the class (not instances). -# E.g., for generic model `Model[A, B]`, when parametrized model `Model[int, str]` is created, -# `Model[int, str]`: {A: int, B: str}` will be stored in `_assigned_parameters`. -# (This information is only otherwise available after creation from the class name string). -_assigned_parameters = AssignedParameters() - - -class GenericModel(BaseModel): - __slots__ = () - __concrete__: ClassVar[bool] = False - - if TYPE_CHECKING: - # Putting this in a TYPE_CHECKING block allows us to replace `if Generic not in cls.__bases__` with - # `not hasattr(cls, "__parameters__")`. This means we don't need to force non-concrete subclasses of - # `GenericModel` to also inherit from `Generic`, which would require changes to the use of `create_model` below. - __parameters__: ClassVar[Tuple[TypeVarType, ...]] - - # Setting the return type as Type[Any] instead of Type[BaseModel] prevents PyCharm warnings - def __class_getitem__(cls: Type[GenericModelT], params: Union[Type[Any], Tuple[Type[Any], ...]]) -> Type[Any]: - """Instantiates a new class from a generic class `cls` and type variables `params`. - - :param params: Tuple of types the class . Given a generic class - `Model` with 2 type variables and a concrete model `Model[str, int]`, - the value `(str, int)` would be passed to `params`. - :return: New model class inheriting from `cls` with instantiated - types described by `params`. If no parameters are given, `cls` is - returned as is. - - """ - - def _cache_key(_params: Any) -> CacheKey: - args = get_args(_params) - # python returns a list for Callables, which is not hashable - if len(args) == 2 and isinstance(args[0], list): - args = (tuple(args[0]), args[1]) - return cls, _params, args - - cached = _generic_types_cache.get(_cache_key(params)) - if cached is not None: - return cached - if cls.__concrete__ and Generic not in cls.__bases__: - raise TypeError('Cannot parameterize a concrete instantiation of a generic model') - if not isinstance(params, tuple): - params = (params,) - if cls is GenericModel and any(isinstance(param, TypeVar) for param in params): - raise TypeError('Type parameters should be placed on typing.Generic, not GenericModel') - if not hasattr(cls, '__parameters__'): - raise TypeError(f'Type {cls.__name__} must inherit from typing.Generic before being parameterized') - - check_parameters_count(cls, params) - # Build map from generic typevars to passed params - typevars_map: Dict[TypeVarType, Type[Any]] = dict(zip(cls.__parameters__, params)) - if all_identical(typevars_map.keys(), typevars_map.values()) and typevars_map: - return cls # if arguments are equal to parameters it's the same object - - # Create new model with original model as parent inserting fields with DeferredType. - model_name = cls.__concrete_name__(params) - validators = gather_all_validators(cls) - - type_hints = get_all_type_hints(cls).items() - instance_type_hints = {k: v for k, v in type_hints if get_origin(v) is not ClassVar} - - fields = {k: (DeferredType(), cls.__fields__[k].field_info) for k in instance_type_hints if k in cls.__fields__} - - model_module, called_globally = get_caller_frame_info() - created_model = cast( - Type[GenericModel], # casting ensures mypy is aware of the __concrete__ and __parameters__ attributes - create_model( - model_name, - __module__=model_module or cls.__module__, - __base__=(cls,) + tuple(cls.__parameterized_bases__(typevars_map)), - __config__=None, - __validators__=validators, - __cls_kwargs__=None, - **fields, - ), - ) - - _assigned_parameters[created_model] = typevars_map - - if called_globally: # create global reference and therefore allow pickling - object_by_reference = None - reference_name = model_name - reference_module_globals = sys.modules[created_model.__module__].__dict__ - while object_by_reference is not created_model: - object_by_reference = reference_module_globals.setdefault(reference_name, created_model) - reference_name += '_' - - created_model.Config = cls.Config - - # Find any typevars that are still present in the model. - # If none are left, the model is fully "concrete", otherwise the new - # class is a generic class as well taking the found typevars as - # parameters. - new_params = tuple( - {param: None for param in iter_contained_typevars(typevars_map.values())} - ) # use dict as ordered set - created_model.__concrete__ = not new_params - if new_params: - created_model.__parameters__ = new_params - - # Save created model in cache so we don't end up creating duplicate - # models that should be identical. - _generic_types_cache[_cache_key(params)] = created_model - if len(params) == 1: - _generic_types_cache[_cache_key(params[0])] = created_model - - # Recursively walk class type hints and replace generic typevars - # with concrete types that were passed. - _prepare_model_fields(created_model, fields, instance_type_hints, typevars_map) - - return created_model - - @classmethod - def __concrete_name__(cls: Type[Any], params: Tuple[Type[Any], ...]) -> str: - """Compute class name for child classes. - - :param params: Tuple of types the class . Given a generic class - `Model` with 2 type variables and a concrete model `Model[str, int]`, - the value `(str, int)` would be passed to `params`. - :return: String representing a the new class where `params` are - passed to `cls` as type variables. - - This method can be overridden to achieve a custom naming scheme for GenericModels. - """ - param_names = [display_as_type(param) for param in params] - params_component = ', '.join(param_names) - return f'{cls.__name__}[{params_component}]' - - @classmethod - def __parameterized_bases__(cls, typevars_map: Parametrization) -> Iterator[Type[Any]]: - """ - Returns unbound bases of cls parameterised to given type variables - - :param typevars_map: Dictionary of type applications for binding subclasses. - Given a generic class `Model` with 2 type variables [S, T] - and a concrete model `Model[str, int]`, - the value `{S: str, T: int}` would be passed to `typevars_map`. - :return: an iterator of generic sub classes, parameterised by `typevars_map` - and other assigned parameters of `cls` - - e.g.: - ``` - class A(GenericModel, Generic[T]): - ... - - class B(A[V], Generic[V]): - ... - - assert A[int] in B.__parameterized_bases__({V: int}) - ``` - """ - - def build_base_model( - base_model: Type[GenericModel], mapped_types: Parametrization - ) -> Iterator[Type[GenericModel]]: - base_parameters = tuple(mapped_types[param] for param in base_model.__parameters__) - parameterized_base = base_model.__class_getitem__(base_parameters) - if parameterized_base is base_model or parameterized_base is cls: - # Avoid duplication in MRO - return - yield parameterized_base - - for base_model in cls.__bases__: - if not issubclass(base_model, GenericModel): - # not a class that can be meaningfully parameterized - continue - elif not getattr(base_model, '__parameters__', None): - # base_model is "GenericModel" (and has no __parameters__) - # or - # base_model is already concrete, and will be included transitively via cls. - continue - elif cls in _assigned_parameters: - if base_model in _assigned_parameters: - # cls is partially parameterised but not from base_model - # e.g. cls = B[S], base_model = A[S] - # B[S][int] should subclass A[int], (and will be transitively via B[int]) - # but it's not viable to consistently subclass types with arbitrary construction - # So don't attempt to include A[S][int] - continue - else: # base_model not in _assigned_parameters: - # cls is partially parameterized, base_model is original generic - # e.g. cls = B[str, T], base_model = B[S, T] - # Need to determine the mapping for the base_model parameters - mapped_types: Parametrization = { - key: typevars_map.get(value, value) for key, value in _assigned_parameters[cls].items() - } - yield from build_base_model(base_model, mapped_types) - else: - # cls is base generic, so base_class has a distinct base - # can construct the Parameterised base model using typevars_map directly - yield from build_base_model(base_model, typevars_map) - - -def replace_types(type_: Any, type_map: Mapping[Any, Any]) -> Any: - """Return type with all occurrences of `type_map` keys recursively replaced with their values. - - :param type_: Any type, class or generic alias - :param type_map: Mapping from `TypeVar` instance to concrete types. - :return: New type representing the basic structure of `type_` with all - `typevar_map` keys recursively replaced. - - >>> replace_types(Tuple[str, Union[List[str], float]], {str: int}) - Tuple[int, Union[List[int], float]] - - """ - if not type_map: - return type_ - - type_args = get_args(type_) - origin_type = get_origin(type_) - - if origin_type is Annotated: - annotated_type, *annotations = type_args - return Annotated[replace_types(annotated_type, type_map), tuple(annotations)] - - if (origin_type is ExtLiteral) or (sys.version_info >= (3, 8) and origin_type is Literal): - return type_map.get(type_, type_) - # Having type args is a good indicator that this is a typing module - # class instantiation or a generic alias of some sort. - if type_args: - resolved_type_args = tuple(replace_types(arg, type_map) for arg in type_args) - if all_identical(type_args, resolved_type_args): - # If all arguments are the same, there is no need to modify the - # type or create a new object at all - return type_ - if ( - origin_type is not None - and isinstance(type_, typing_base) - and not isinstance(origin_type, typing_base) - and getattr(type_, '_name', None) is not None - ): - # In python < 3.9 generic aliases don't exist so any of these like `list`, - # `type` or `collections.abc.Callable` need to be translated. - # See: https://www.python.org/dev/peps/pep-0585 - origin_type = getattr(typing, type_._name) - assert origin_type is not None - # PEP-604 syntax (Ex.: list | str) is represented with a types.UnionType object that does not have __getitem__. - # We also cannot use isinstance() since we have to compare types. - if sys.version_info >= (3, 10) and origin_type is types.UnionType: # noqa: E721 - return functools.reduce(operator.or_, resolved_type_args) - return origin_type[resolved_type_args] - - # We handle pydantic generic models separately as they don't have the same - # semantics as "typing" classes or generic aliases - if not origin_type and lenient_issubclass(type_, GenericModel) and not type_.__concrete__: - type_args = type_.__parameters__ - resolved_type_args = tuple(replace_types(t, type_map) for t in type_args) - if all_identical(type_args, resolved_type_args): - return type_ - return type_[resolved_type_args] - - # Handle special case for typehints that can have lists as arguments. - # `typing.Callable[[int, str], int]` is an example for this. - if isinstance(type_, (List, list)): - resolved_list = list(replace_types(element, type_map) for element in type_) - if all_identical(type_, resolved_list): - return type_ - return resolved_list - - # For JsonWrapperValue, need to handle its inner type to allow correct parsing - # of generic Json arguments like Json[T] - if not origin_type and lenient_issubclass(type_, JsonWrapper): - type_.inner_type = replace_types(type_.inner_type, type_map) - return type_ - - # If all else fails, we try to resolve the type directly and otherwise just - # return the input with no modifications. - new_type = type_map.get(type_, type_) - # Convert string to ForwardRef - if isinstance(new_type, str): - return ForwardRef(new_type) - else: - return new_type - - -def check_parameters_count(cls: Type[GenericModel], parameters: Tuple[Any, ...]) -> None: - actual = len(parameters) - expected = len(cls.__parameters__) - if actual != expected: - description = 'many' if actual > expected else 'few' - raise TypeError(f'Too {description} parameters for {cls.__name__}; actual {actual}, expected {expected}') - - -DictValues: Type[Any] = {}.values().__class__ - - -def iter_contained_typevars(v: Any) -> Iterator[TypeVarType]: - """Recursively iterate through all subtypes and type args of `v` and yield any typevars that are found.""" - if isinstance(v, TypeVar): - yield v - elif hasattr(v, '__parameters__') and not get_origin(v) and lenient_issubclass(v, GenericModel): - yield from v.__parameters__ - elif isinstance(v, (DictValues, list)): - for var in v: - yield from iter_contained_typevars(var) - else: - args = get_args(v) - for arg in args: - yield from iter_contained_typevars(arg) - - -def get_caller_frame_info() -> Tuple[Optional[str], bool]: - """ - Used inside a function to check whether it was called globally - - Will only work against non-compiled code, therefore used only in pydantic.generics - - :returns Tuple[module_name, called_globally] - """ - try: - previous_caller_frame = sys._getframe(2) - except ValueError as e: - raise RuntimeError('This function must be used inside another function') from e - except AttributeError: # sys module does not have _getframe function, so there's nothing we can do about it - return None, False - frame_globals = previous_caller_frame.f_globals - return frame_globals.get('__name__'), previous_caller_frame.f_locals is frame_globals - - -def _prepare_model_fields( - created_model: Type[GenericModel], - fields: Mapping[str, Any], - instance_type_hints: Mapping[str, type], - typevars_map: Mapping[Any, type], -) -> None: - """ - Replace DeferredType fields with concrete type hints and prepare them. - """ - - for key, field in created_model.__fields__.items(): - if key not in fields: - assert field.type_.__class__ is not DeferredType - # https://github.com/nedbat/coveragepy/issues/198 - continue # pragma: no cover - - assert field.type_.__class__ is DeferredType, field.type_.__class__ - - field_type_hint = instance_type_hints[key] - concrete_type = replace_types(field_type_hint, typevars_map) - field.type_ = concrete_type - field.outer_type_ = concrete_type - field.prepare() - created_model.__annotations__[key] = concrete_type diff --git a/functions/python/data-processor/pydantic/v1/json.py b/functions/python/data-processor/pydantic/v1/json.py deleted file mode 100644 index 41d0d5fca..000000000 --- a/functions/python/data-processor/pydantic/v1/json.py +++ /dev/null @@ -1,112 +0,0 @@ -import datetime -from collections import deque -from decimal import Decimal -from enum import Enum -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from pathlib import Path -from re import Pattern -from types import GeneratorType -from typing import Any, Callable, Dict, Type, Union -from uuid import UUID - -from pydantic.v1.color import Color -from pydantic.v1.networks import NameEmail -from pydantic.v1.types import SecretBytes, SecretStr - -__all__ = 'pydantic_encoder', 'custom_pydantic_encoder', 'timedelta_isoformat' - - -def isoformat(o: Union[datetime.date, datetime.time]) -> str: - return o.isoformat() - - -def decimal_encoder(dec_value: Decimal) -> Union[int, float]: - """ - Encodes a Decimal as int of there's no exponent, otherwise float - - This is useful when we use ConstrainedDecimal to represent Numeric(x,0) - where a integer (but not int typed) is used. Encoding this as a float - results in failed round-tripping between encode and parse. - Our Id type is a prime example of this. - - >>> decimal_encoder(Decimal("1.0")) - 1.0 - - >>> decimal_encoder(Decimal("1")) - 1 - """ - if dec_value.as_tuple().exponent >= 0: - return int(dec_value) - else: - return float(dec_value) - - -ENCODERS_BY_TYPE: Dict[Type[Any], Callable[[Any], Any]] = { - bytes: lambda o: o.decode(), - Color: str, - datetime.date: isoformat, - datetime.datetime: isoformat, - datetime.time: isoformat, - datetime.timedelta: lambda td: td.total_seconds(), - Decimal: decimal_encoder, - Enum: lambda o: o.value, - frozenset: list, - deque: list, - GeneratorType: list, - IPv4Address: str, - IPv4Interface: str, - IPv4Network: str, - IPv6Address: str, - IPv6Interface: str, - IPv6Network: str, - NameEmail: str, - Path: str, - Pattern: lambda o: o.pattern, - SecretBytes: str, - SecretStr: str, - set: list, - UUID: str, -} - - -def pydantic_encoder(obj: Any) -> Any: - from dataclasses import asdict, is_dataclass - - from pydantic.v1.main import BaseModel - - if isinstance(obj, BaseModel): - return obj.dict() - elif is_dataclass(obj): - return asdict(obj) - - # Check the class type and its superclasses for a matching encoder - for base in obj.__class__.__mro__[:-1]: - try: - encoder = ENCODERS_BY_TYPE[base] - except KeyError: - continue - return encoder(obj) - else: # We have exited the for loop without finding a suitable encoder - raise TypeError(f"Object of type '{obj.__class__.__name__}' is not JSON serializable") - - -def custom_pydantic_encoder(type_encoders: Dict[Any, Callable[[Type[Any]], Any]], obj: Any) -> Any: - # Check the class type and its superclasses for a matching encoder - for base in obj.__class__.__mro__[:-1]: - try: - encoder = type_encoders[base] - except KeyError: - continue - - return encoder(obj) - else: # We have exited the for loop without finding a suitable encoder - return pydantic_encoder(obj) - - -def timedelta_isoformat(td: datetime.timedelta) -> str: - """ - ISO 8601 encoding for Python timedelta object. - """ - minutes, seconds = divmod(td.seconds, 60) - hours, minutes = divmod(minutes, 60) - return f'{"-" if td.days < 0 else ""}P{abs(td.days)}DT{hours:d}H{minutes:d}M{seconds:d}.{td.microseconds:06d}S' diff --git a/functions/python/data-processor/pydantic/v1/main.py b/functions/python/data-processor/pydantic/v1/main.py deleted file mode 100644 index 36e9c44c9..000000000 --- a/functions/python/data-processor/pydantic/v1/main.py +++ /dev/null @@ -1,1130 +0,0 @@ -import sys -import warnings -from abc import ABCMeta -from copy import deepcopy -from enum import Enum -from functools import partial -from pathlib import Path -from types import FunctionType, prepare_class, resolve_bases -from typing import ( - TYPE_CHECKING, - AbstractSet, - Any, - Callable, - ClassVar, - Dict, - List, - Mapping, - Optional, - Tuple, - Type, - TypeVar, - Union, - cast, - no_type_check, - overload, -) - -from typing_extensions import dataclass_transform - -from pydantic.v1.class_validators import ValidatorGroup, extract_root_validators, extract_validators, inherit_validators -from pydantic.v1.config import BaseConfig, Extra, inherit_config, prepare_config -from pydantic.v1.error_wrappers import ErrorWrapper, ValidationError -from pydantic.v1.errors import ConfigError, DictError, ExtraError, MissingError -from pydantic.v1.fields import ( - MAPPING_LIKE_SHAPES, - Field, - ModelField, - ModelPrivateAttr, - PrivateAttr, - Undefined, - is_finalvar_with_default_val, -) -from pydantic.v1.json import custom_pydantic_encoder, pydantic_encoder -from pydantic.v1.parse import Protocol, load_file, load_str_bytes -from pydantic.v1.schema import default_ref_template, model_schema -from pydantic.v1.types import PyObject, StrBytes -from pydantic.v1.typing import ( - AnyCallable, - get_args, - get_origin, - is_classvar, - is_namedtuple, - is_union, - resolve_annotations, - update_model_forward_refs, -) -from pydantic.v1.utils import ( - DUNDER_ATTRIBUTES, - ROOT_KEY, - ClassAttribute, - GetterDict, - Representation, - ValueItems, - generate_model_signature, - is_valid_field, - is_valid_private_name, - lenient_issubclass, - sequence_like, - smart_deepcopy, - unique_list, - validate_field_name, -) - -if TYPE_CHECKING: - from inspect import Signature - - from pydantic.v1.class_validators import ValidatorListDict - from pydantic.v1.types import ModelOrDc - from pydantic.v1.typing import ( - AbstractSetIntStr, - AnyClassMethod, - CallableGenerator, - DictAny, - DictStrAny, - MappingIntStrAny, - ReprArgs, - SetStr, - TupleGenerator, - ) - - Model = TypeVar('Model', bound='BaseModel') - -__all__ = 'BaseModel', 'create_model', 'validate_model' - -_T = TypeVar('_T') - - -def validate_custom_root_type(fields: Dict[str, ModelField]) -> None: - if len(fields) > 1: - raise ValueError(f'{ROOT_KEY} cannot be mixed with other fields') - - -def generate_hash_function(frozen: bool) -> Optional[Callable[[Any], int]]: - def hash_function(self_: Any) -> int: - return hash(self_.__class__) + hash(tuple(self_.__dict__.values())) - - return hash_function if frozen else None - - -# If a field is of type `Callable`, its default value should be a function and cannot to ignored. -ANNOTATED_FIELD_UNTOUCHED_TYPES: Tuple[Any, ...] = (property, type, classmethod, staticmethod) -# When creating a `BaseModel` instance, we bypass all the methods, properties... added to the model -UNTOUCHED_TYPES: Tuple[Any, ...] = (FunctionType,) + ANNOTATED_FIELD_UNTOUCHED_TYPES -# Note `ModelMetaclass` refers to `BaseModel`, but is also used to *create* `BaseModel`, so we need to add this extra -# (somewhat hacky) boolean to keep track of whether we've created the `BaseModel` class yet, and therefore whether it's -# safe to refer to it. If it *hasn't* been created, we assume that the `__new__` call we're in the middle of is for -# the `BaseModel` class, since that's defined immediately after the metaclass. -_is_base_model_class_defined = False - - -@dataclass_transform(kw_only_default=True, field_specifiers=(Field,)) -class ModelMetaclass(ABCMeta): - @no_type_check # noqa C901 - def __new__(mcs, name, bases, namespace, **kwargs): # noqa C901 - fields: Dict[str, ModelField] = {} - config = BaseConfig - validators: 'ValidatorListDict' = {} - - pre_root_validators, post_root_validators = [], [] - private_attributes: Dict[str, ModelPrivateAttr] = {} - base_private_attributes: Dict[str, ModelPrivateAttr] = {} - slots: SetStr = namespace.get('__slots__', ()) - slots = {slots} if isinstance(slots, str) else set(slots) - class_vars: SetStr = set() - hash_func: Optional[Callable[[Any], int]] = None - - for base in reversed(bases): - if _is_base_model_class_defined and issubclass(base, BaseModel) and base != BaseModel: - fields.update(smart_deepcopy(base.__fields__)) - config = inherit_config(base.__config__, config) - validators = inherit_validators(base.__validators__, validators) - pre_root_validators += base.__pre_root_validators__ - post_root_validators += base.__post_root_validators__ - base_private_attributes.update(base.__private_attributes__) - class_vars.update(base.__class_vars__) - hash_func = base.__hash__ - - resolve_forward_refs = kwargs.pop('__resolve_forward_refs__', True) - allowed_config_kwargs: SetStr = { - key - for key in dir(config) - if not (key.startswith('__') and key.endswith('__')) # skip dunder methods and attributes - } - config_kwargs = {key: kwargs.pop(key) for key in kwargs.keys() & allowed_config_kwargs} - config_from_namespace = namespace.get('Config') - if config_kwargs and config_from_namespace: - raise TypeError('Specifying config in two places is ambiguous, use either Config attribute or class kwargs') - config = inherit_config(config_from_namespace, config, **config_kwargs) - - validators = inherit_validators(extract_validators(namespace), validators) - vg = ValidatorGroup(validators) - - for f in fields.values(): - f.set_config(config) - extra_validators = vg.get_validators(f.name) - if extra_validators: - f.class_validators.update(extra_validators) - # re-run prepare to add extra validators - f.populate_validators() - - prepare_config(config, name) - - untouched_types = ANNOTATED_FIELD_UNTOUCHED_TYPES - - def is_untouched(v: Any) -> bool: - return isinstance(v, untouched_types) or v.__class__.__name__ == 'cython_function_or_method' - - if (namespace.get('__module__'), namespace.get('__qualname__')) != ('pydantic.main', 'BaseModel'): - if sys.version_info >= (3, 14): - if '__annotations__' in namespace: - # `from __future__ import annotations` was used in the model's module - raw_annotations = namespace['__annotations__'] - else: - # See https://docs.python.org/3/library/annotationlib.html#using-annotations-in-a-metaclass: - from annotationlib import Format, call_annotate_function, get_annotate_from_class_namespace - - annotate = get_annotate_from_class_namespace(namespace) - if annotate is not None: - raw_annotations = call_annotate_function(annotate, format=Format.FORWARDREF) - else: - raw_annotations = {} - else: - raw_annotations = namespace.get('__annotations__', {}) - - annotations = resolve_annotations(raw_annotations, namespace.get('__module__', None)) - # annotation only fields need to come first in fields - for ann_name, ann_type in annotations.items(): - if is_classvar(ann_type): - class_vars.add(ann_name) - elif is_finalvar_with_default_val(ann_type, namespace.get(ann_name, Undefined)): - class_vars.add(ann_name) - elif is_valid_field(ann_name): - validate_field_name(bases, ann_name) - value = namespace.get(ann_name, Undefined) - allowed_types = get_args(ann_type) if is_union(get_origin(ann_type)) else (ann_type,) - if ( - is_untouched(value) - and ann_type != PyObject - and not any( - lenient_issubclass(get_origin(allowed_type), Type) for allowed_type in allowed_types - ) - ): - continue - fields[ann_name] = ModelField.infer( - name=ann_name, - value=value, - annotation=ann_type, - class_validators=vg.get_validators(ann_name), - config=config, - ) - elif ann_name not in namespace and config.underscore_attrs_are_private: - private_attributes[ann_name] = PrivateAttr() - - untouched_types = UNTOUCHED_TYPES + config.keep_untouched - for var_name, value in namespace.items(): - can_be_changed = var_name not in class_vars and not is_untouched(value) - if isinstance(value, ModelPrivateAttr): - if not is_valid_private_name(var_name): - raise NameError( - f'Private attributes "{var_name}" must not be a valid field name; ' - f'Use sunder or dunder names, e. g. "_{var_name}" or "__{var_name}__"' - ) - private_attributes[var_name] = value - elif config.underscore_attrs_are_private and is_valid_private_name(var_name) and can_be_changed: - private_attributes[var_name] = PrivateAttr(default=value) - elif is_valid_field(var_name) and var_name not in annotations and can_be_changed: - validate_field_name(bases, var_name) - inferred = ModelField.infer( - name=var_name, - value=value, - annotation=annotations.get(var_name, Undefined), - class_validators=vg.get_validators(var_name), - config=config, - ) - if var_name in fields: - if lenient_issubclass(inferred.type_, fields[var_name].type_): - inferred.type_ = fields[var_name].type_ - else: - raise TypeError( - f'The type of {name}.{var_name} differs from the new default value; ' - f'if you wish to change the type of this field, please use a type annotation' - ) - fields[var_name] = inferred - - _custom_root_type = ROOT_KEY in fields - if _custom_root_type: - validate_custom_root_type(fields) - vg.check_for_unused() - if config.json_encoders: - json_encoder = partial(custom_pydantic_encoder, config.json_encoders) - else: - json_encoder = pydantic_encoder - pre_rv_new, post_rv_new = extract_root_validators(namespace) - - if hash_func is None: - hash_func = generate_hash_function(config.frozen) - - exclude_from_namespace = fields | private_attributes.keys() | {'__slots__'} - new_namespace = { - '__config__': config, - '__fields__': fields, - '__exclude_fields__': { - name: field.field_info.exclude for name, field in fields.items() if field.field_info.exclude is not None - } - or None, - '__include_fields__': { - name: field.field_info.include for name, field in fields.items() if field.field_info.include is not None - } - or None, - '__validators__': vg.validators, - '__pre_root_validators__': unique_list( - pre_root_validators + pre_rv_new, - name_factory=lambda v: v.__name__, - ), - '__post_root_validators__': unique_list( - post_root_validators + post_rv_new, - name_factory=lambda skip_on_failure_and_v: skip_on_failure_and_v[1].__name__, - ), - '__schema_cache__': {}, - '__json_encoder__': staticmethod(json_encoder), - '__custom_root_type__': _custom_root_type, - '__private_attributes__': {**base_private_attributes, **private_attributes}, - '__slots__': slots | private_attributes.keys(), - '__hash__': hash_func, - '__class_vars__': class_vars, - **{n: v for n, v in namespace.items() if n not in exclude_from_namespace}, - } - - cls = super().__new__(mcs, name, bases, new_namespace, **kwargs) - # set __signature__ attr only for model class, but not for its instances - cls.__signature__ = ClassAttribute('__signature__', generate_model_signature(cls.__init__, fields, config)) - - if not _is_base_model_class_defined: - # Cython does not understand the `if TYPE_CHECKING:` condition in the - # BaseModel's body (where annotations are set), so clear them manually: - getattr(cls, '__annotations__', {}).clear() - - if resolve_forward_refs: - cls.__try_update_forward_refs__() - - # preserve `__set_name__` protocol defined in https://peps.python.org/pep-0487 - # for attributes not in `new_namespace` (e.g. private attributes) - for name, obj in namespace.items(): - if name not in new_namespace: - set_name = getattr(obj, '__set_name__', None) - if callable(set_name): - set_name(cls, name) - - return cls - - def __instancecheck__(self, instance: Any) -> bool: - """ - Avoid calling ABC _abc_subclasscheck unless we're pretty sure. - - See #3829 and python/cpython#92810 - """ - return hasattr(instance, '__post_root_validators__') and super().__instancecheck__(instance) - - -object_setattr = object.__setattr__ - - -class BaseModel(Representation, metaclass=ModelMetaclass): - if TYPE_CHECKING: - # populated by the metaclass, defined here to help IDEs only - __fields__: ClassVar[Dict[str, ModelField]] = {} - __include_fields__: ClassVar[Optional[Mapping[str, Any]]] = None - __exclude_fields__: ClassVar[Optional[Mapping[str, Any]]] = None - __validators__: ClassVar[Dict[str, AnyCallable]] = {} - __pre_root_validators__: ClassVar[List[AnyCallable]] - __post_root_validators__: ClassVar[List[Tuple[bool, AnyCallable]]] - __config__: ClassVar[Type[BaseConfig]] = BaseConfig - __json_encoder__: ClassVar[Callable[[Any], Any]] = lambda x: x - __schema_cache__: ClassVar['DictAny'] = {} - __custom_root_type__: ClassVar[bool] = False - __signature__: ClassVar['Signature'] - __private_attributes__: ClassVar[Dict[str, ModelPrivateAttr]] - __class_vars__: ClassVar[SetStr] - __fields_set__: ClassVar[SetStr] = set() - - Config = BaseConfig - __slots__ = ('__dict__', '__fields_set__') - __doc__ = '' # Null out the Representation docstring - - def __init__(__pydantic_self__, **data: Any) -> None: - """ - Create a new model by parsing and validating input data from keyword arguments. - - Raises ValidationError if the input data cannot be parsed to form a valid model. - """ - # Uses something other than `self` the first arg to allow "self" as a settable attribute - values, fields_set, validation_error = validate_model(__pydantic_self__.__class__, data) - if validation_error: - raise validation_error - try: - object_setattr(__pydantic_self__, '__dict__', values) - except TypeError as e: - raise TypeError( - 'Model values must be a dict; you may not have returned a dictionary from a root validator' - ) from e - object_setattr(__pydantic_self__, '__fields_set__', fields_set) - __pydantic_self__._init_private_attributes() - - @no_type_check - def __setattr__(self, name, value): # noqa: C901 (ignore complexity) - if name in self.__private_attributes__ or name in DUNDER_ATTRIBUTES: - return object_setattr(self, name, value) - - if self.__config__.extra is not Extra.allow and name not in self.__fields__: - raise ValueError(f'"{self.__class__.__name__}" object has no field "{name}"') - elif not self.__config__.allow_mutation or self.__config__.frozen: - raise TypeError(f'"{self.__class__.__name__}" is immutable and does not support item assignment') - elif name in self.__fields__ and self.__fields__[name].final: - raise TypeError( - f'"{self.__class__.__name__}" object "{name}" field is final and does not support reassignment' - ) - elif self.__config__.validate_assignment: - new_values = {**self.__dict__, name: value} - - for validator in self.__pre_root_validators__: - try: - new_values = validator(self.__class__, new_values) - except (ValueError, TypeError, AssertionError) as exc: - raise ValidationError([ErrorWrapper(exc, loc=ROOT_KEY)], self.__class__) - - known_field = self.__fields__.get(name, None) - if known_field: - # We want to - # - make sure validators are called without the current value for this field inside `values` - # - keep other values (e.g. submodels) untouched (using `BaseModel.dict()` will change them into dicts) - # - keep the order of the fields - if not known_field.field_info.allow_mutation: - raise TypeError(f'"{known_field.name}" has allow_mutation set to False and cannot be assigned') - dict_without_original_value = {k: v for k, v in self.__dict__.items() if k != name} - value, error_ = known_field.validate(value, dict_without_original_value, loc=name, cls=self.__class__) - if error_: - raise ValidationError([error_], self.__class__) - else: - new_values[name] = value - - errors = [] - for skip_on_failure, validator in self.__post_root_validators__: - if skip_on_failure and errors: - continue - try: - new_values = validator(self.__class__, new_values) - except (ValueError, TypeError, AssertionError) as exc: - errors.append(ErrorWrapper(exc, loc=ROOT_KEY)) - if errors: - raise ValidationError(errors, self.__class__) - - # update the whole __dict__ as other values than just `value` - # may be changed (e.g. with `root_validator`) - object_setattr(self, '__dict__', new_values) - else: - self.__dict__[name] = value - - self.__fields_set__.add(name) - - def __getstate__(self) -> 'DictAny': - private_attrs = ((k, getattr(self, k, Undefined)) for k in self.__private_attributes__) - return { - '__dict__': self.__dict__, - '__fields_set__': self.__fields_set__, - '__private_attribute_values__': {k: v for k, v in private_attrs if v is not Undefined}, - } - - def __setstate__(self, state: 'DictAny') -> None: - object_setattr(self, '__dict__', state['__dict__']) - object_setattr(self, '__fields_set__', state['__fields_set__']) - for name, value in state.get('__private_attribute_values__', {}).items(): - object_setattr(self, name, value) - - def _init_private_attributes(self) -> None: - for name, private_attr in self.__private_attributes__.items(): - default = private_attr.get_default() - if default is not Undefined: - object_setattr(self, name, default) - - def dict( - self, - *, - include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - by_alias: bool = False, - skip_defaults: Optional[bool] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> 'DictStrAny': - """ - Generate a dictionary representation of the model, optionally specifying which fields to include or exclude. - - """ - if skip_defaults is not None: - warnings.warn( - f'{self.__class__.__name__}.dict(): "skip_defaults" is deprecated and replaced by "exclude_unset"', - DeprecationWarning, - ) - exclude_unset = skip_defaults - - return dict( - self._iter( - to_dict=True, - by_alias=by_alias, - include=include, - exclude=exclude, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - ) - - def json( - self, - *, - include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - by_alias: bool = False, - skip_defaults: Optional[bool] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - encoder: Optional[Callable[[Any], Any]] = None, - models_as_dict: bool = True, - **dumps_kwargs: Any, - ) -> str: - """ - Generate a JSON representation of the model, `include` and `exclude` arguments as per `dict()`. - - `encoder` is an optional function to supply as `default` to json.dumps(), other arguments as per `json.dumps()`. - """ - if skip_defaults is not None: - warnings.warn( - f'{self.__class__.__name__}.json(): "skip_defaults" is deprecated and replaced by "exclude_unset"', - DeprecationWarning, - ) - exclude_unset = skip_defaults - encoder = cast(Callable[[Any], Any], encoder or self.__json_encoder__) - - # We don't directly call `self.dict()`, which does exactly this with `to_dict=True` - # because we want to be able to keep raw `BaseModel` instances and not as `dict`. - # This allows users to write custom JSON encoders for given `BaseModel` classes. - data = dict( - self._iter( - to_dict=models_as_dict, - by_alias=by_alias, - include=include, - exclude=exclude, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - ) - if self.__custom_root_type__: - data = data[ROOT_KEY] - return self.__config__.json_dumps(data, default=encoder, **dumps_kwargs) - - @classmethod - def _enforce_dict_if_root(cls, obj: Any) -> Any: - if cls.__custom_root_type__ and ( - not (isinstance(obj, dict) and obj.keys() == {ROOT_KEY}) - and not (isinstance(obj, BaseModel) and obj.__fields__.keys() == {ROOT_KEY}) - or cls.__fields__[ROOT_KEY].shape in MAPPING_LIKE_SHAPES - ): - return {ROOT_KEY: obj} - else: - return obj - - @classmethod - def parse_obj(cls: Type['Model'], obj: Any) -> 'Model': - obj = cls._enforce_dict_if_root(obj) - if not isinstance(obj, dict): - try: - obj = dict(obj) - except (TypeError, ValueError) as e: - exc = TypeError(f'{cls.__name__} expected dict not {obj.__class__.__name__}') - raise ValidationError([ErrorWrapper(exc, loc=ROOT_KEY)], cls) from e - return cls(**obj) - - @classmethod - def parse_raw( - cls: Type['Model'], - b: StrBytes, - *, - content_type: str = None, - encoding: str = 'utf8', - proto: Protocol = None, - allow_pickle: bool = False, - ) -> 'Model': - try: - obj = load_str_bytes( - b, - proto=proto, - content_type=content_type, - encoding=encoding, - allow_pickle=allow_pickle, - json_loads=cls.__config__.json_loads, - ) - except (ValueError, TypeError, UnicodeDecodeError) as e: - raise ValidationError([ErrorWrapper(e, loc=ROOT_KEY)], cls) - return cls.parse_obj(obj) - - @classmethod - def parse_file( - cls: Type['Model'], - path: Union[str, Path], - *, - content_type: str = None, - encoding: str = 'utf8', - proto: Protocol = None, - allow_pickle: bool = False, - ) -> 'Model': - obj = load_file( - path, - proto=proto, - content_type=content_type, - encoding=encoding, - allow_pickle=allow_pickle, - json_loads=cls.__config__.json_loads, - ) - return cls.parse_obj(obj) - - @classmethod - def from_orm(cls: Type['Model'], obj: Any) -> 'Model': - if not cls.__config__.orm_mode: - raise ConfigError('You must have the config attribute orm_mode=True to use from_orm') - obj = {ROOT_KEY: obj} if cls.__custom_root_type__ else cls._decompose_class(obj) - m = cls.__new__(cls) - values, fields_set, validation_error = validate_model(cls, obj) - if validation_error: - raise validation_error - object_setattr(m, '__dict__', values) - object_setattr(m, '__fields_set__', fields_set) - m._init_private_attributes() - return m - - @classmethod - def construct(cls: Type['Model'], _fields_set: Optional['SetStr'] = None, **values: Any) -> 'Model': - """ - Creates a new model setting __dict__ and __fields_set__ from trusted or pre-validated data. - Default values are respected, but no other validation is performed. - Behaves as if `Config.extra = 'allow'` was set since it adds all passed values - """ - m = cls.__new__(cls) - fields_values: Dict[str, Any] = {} - for name, field in cls.__fields__.items(): - if field.alt_alias and field.alias in values: - fields_values[name] = values[field.alias] - elif name in values: - fields_values[name] = values[name] - elif not field.required: - fields_values[name] = field.get_default() - fields_values.update(values) - object_setattr(m, '__dict__', fields_values) - if _fields_set is None: - _fields_set = set(values.keys()) - object_setattr(m, '__fields_set__', _fields_set) - m._init_private_attributes() - return m - - def _copy_and_set_values(self: 'Model', values: 'DictStrAny', fields_set: 'SetStr', *, deep: bool) -> 'Model': - if deep: - # chances of having empty dict here are quite low for using smart_deepcopy - values = deepcopy(values) - - cls = self.__class__ - m = cls.__new__(cls) - object_setattr(m, '__dict__', values) - object_setattr(m, '__fields_set__', fields_set) - for name in self.__private_attributes__: - value = getattr(self, name, Undefined) - if value is not Undefined: - if deep: - value = deepcopy(value) - object_setattr(m, name, value) - - return m - - def copy( - self: 'Model', - *, - include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - update: Optional['DictStrAny'] = None, - deep: bool = False, - ) -> 'Model': - """ - Duplicate a model, optionally choose which fields to include, exclude and change. - - :param include: fields to include in new model - :param exclude: fields to exclude from new model, as with values this takes precedence over include - :param update: values to change/add in the new model. Note: the data is not validated before creating - the new model: you should trust this data - :param deep: set to `True` to make a deep copy of the model - :return: new model instance - """ - - values = dict( - self._iter(to_dict=False, by_alias=False, include=include, exclude=exclude, exclude_unset=False), - **(update or {}), - ) - - # new `__fields_set__` can have unset optional fields with a set value in `update` kwarg - if update: - fields_set = self.__fields_set__ | update.keys() - else: - fields_set = set(self.__fields_set__) - - return self._copy_and_set_values(values, fields_set, deep=deep) - - @classmethod - def schema(cls, by_alias: bool = True, ref_template: str = default_ref_template) -> 'DictStrAny': - cached = cls.__schema_cache__.get((by_alias, ref_template)) - if cached is not None: - return cached - s = model_schema(cls, by_alias=by_alias, ref_template=ref_template) - cls.__schema_cache__[(by_alias, ref_template)] = s - return s - - @classmethod - def schema_json( - cls, *, by_alias: bool = True, ref_template: str = default_ref_template, **dumps_kwargs: Any - ) -> str: - from pydantic.v1.json import pydantic_encoder - - return cls.__config__.json_dumps( - cls.schema(by_alias=by_alias, ref_template=ref_template), default=pydantic_encoder, **dumps_kwargs - ) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls: Type['Model'], value: Any) -> 'Model': - if isinstance(value, cls): - copy_on_model_validation = cls.__config__.copy_on_model_validation - # whether to deep or shallow copy the model on validation, None means do not copy - deep_copy: Optional[bool] = None - if copy_on_model_validation not in {'deep', 'shallow', 'none'}: - # Warn about deprecated behavior - warnings.warn( - "`copy_on_model_validation` should be a string: 'deep', 'shallow' or 'none'", DeprecationWarning - ) - if copy_on_model_validation: - deep_copy = False - - if copy_on_model_validation == 'shallow': - # shallow copy - deep_copy = False - elif copy_on_model_validation == 'deep': - # deep copy - deep_copy = True - - if deep_copy is None: - return value - else: - return value._copy_and_set_values(value.__dict__, value.__fields_set__, deep=deep_copy) - - value = cls._enforce_dict_if_root(value) - - if isinstance(value, dict): - return cls(**value) - elif cls.__config__.orm_mode: - return cls.from_orm(value) - else: - try: - value_as_dict = dict(value) - except (TypeError, ValueError) as e: - raise DictError() from e - return cls(**value_as_dict) - - @classmethod - def _decompose_class(cls: Type['Model'], obj: Any) -> GetterDict: - if isinstance(obj, GetterDict): - return obj - return cls.__config__.getter_dict(obj) - - @classmethod - @no_type_check - def _get_value( - cls, - v: Any, - to_dict: bool, - by_alias: bool, - include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']], - exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']], - exclude_unset: bool, - exclude_defaults: bool, - exclude_none: bool, - ) -> Any: - if isinstance(v, BaseModel): - if to_dict: - v_dict = v.dict( - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - include=include, - exclude=exclude, - exclude_none=exclude_none, - ) - if ROOT_KEY in v_dict: - return v_dict[ROOT_KEY] - return v_dict - else: - return v.copy(include=include, exclude=exclude) - - value_exclude = ValueItems(v, exclude) if exclude else None - value_include = ValueItems(v, include) if include else None - - if isinstance(v, dict): - return { - k_: cls._get_value( - v_, - to_dict=to_dict, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - include=value_include and value_include.for_element(k_), - exclude=value_exclude and value_exclude.for_element(k_), - exclude_none=exclude_none, - ) - for k_, v_ in v.items() - if (not value_exclude or not value_exclude.is_excluded(k_)) - and (not value_include or value_include.is_included(k_)) - } - - elif sequence_like(v): - seq_args = ( - cls._get_value( - v_, - to_dict=to_dict, - by_alias=by_alias, - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - include=value_include and value_include.for_element(i), - exclude=value_exclude and value_exclude.for_element(i), - exclude_none=exclude_none, - ) - for i, v_ in enumerate(v) - if (not value_exclude or not value_exclude.is_excluded(i)) - and (not value_include or value_include.is_included(i)) - ) - - return v.__class__(*seq_args) if is_namedtuple(v.__class__) else v.__class__(seq_args) - - elif isinstance(v, Enum) and getattr(cls.Config, 'use_enum_values', False): - return v.value - - else: - return v - - @classmethod - def __try_update_forward_refs__(cls, **localns: Any) -> None: - """ - Same as update_forward_refs but will not raise exception - when forward references are not defined. - """ - update_model_forward_refs(cls, cls.__fields__.values(), cls.__config__.json_encoders, localns, (NameError,)) - - @classmethod - def update_forward_refs(cls, **localns: Any) -> None: - """ - Try to update ForwardRefs on fields based on this Model, globalns and localns. - """ - update_model_forward_refs(cls, cls.__fields__.values(), cls.__config__.json_encoders, localns) - - def __iter__(self) -> 'TupleGenerator': - """ - so `dict(model)` works - """ - yield from self.__dict__.items() - - def _iter( - self, - to_dict: bool = False, - by_alias: bool = False, - include: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - exclude: Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']] = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - ) -> 'TupleGenerator': - # Merge field set excludes with explicit exclude parameter with explicit overriding field set options. - # The extra "is not None" guards are not logically necessary but optimizes performance for the simple case. - if exclude is not None or self.__exclude_fields__ is not None: - exclude = ValueItems.merge(self.__exclude_fields__, exclude) - - if include is not None or self.__include_fields__ is not None: - include = ValueItems.merge(self.__include_fields__, include, intersect=True) - - allowed_keys = self._calculate_keys( - include=include, exclude=exclude, exclude_unset=exclude_unset # type: ignore - ) - if allowed_keys is None and not (to_dict or by_alias or exclude_unset or exclude_defaults or exclude_none): - # huge boost for plain _iter() - yield from self.__dict__.items() - return - - value_exclude = ValueItems(self, exclude) if exclude is not None else None - value_include = ValueItems(self, include) if include is not None else None - - for field_key, v in self.__dict__.items(): - if (allowed_keys is not None and field_key not in allowed_keys) or (exclude_none and v is None): - continue - - if exclude_defaults: - model_field = self.__fields__.get(field_key) - if not getattr(model_field, 'required', True) and getattr(model_field, 'default', _missing) == v: - continue - - if by_alias and field_key in self.__fields__: - dict_key = self.__fields__[field_key].alias - else: - dict_key = field_key - - if to_dict or value_include or value_exclude: - v = self._get_value( - v, - to_dict=to_dict, - by_alias=by_alias, - include=value_include and value_include.for_element(field_key), - exclude=value_exclude and value_exclude.for_element(field_key), - exclude_unset=exclude_unset, - exclude_defaults=exclude_defaults, - exclude_none=exclude_none, - ) - yield dict_key, v - - def _calculate_keys( - self, - include: Optional['MappingIntStrAny'], - exclude: Optional['MappingIntStrAny'], - exclude_unset: bool, - update: Optional['DictStrAny'] = None, - ) -> Optional[AbstractSet[str]]: - if include is None and exclude is None and exclude_unset is False: - return None - - keys: AbstractSet[str] - if exclude_unset: - keys = self.__fields_set__.copy() - else: - keys = self.__dict__.keys() - - if include is not None: - keys &= include.keys() - - if update: - keys -= update.keys() - - if exclude: - keys -= {k for k, v in exclude.items() if ValueItems.is_true(v)} - - return keys - - def __eq__(self, other: Any) -> bool: - if isinstance(other, BaseModel): - return self.dict() == other.dict() - else: - return self.dict() == other - - def __repr_args__(self) -> 'ReprArgs': - return [ - (k, v) - for k, v in self.__dict__.items() - if k not in DUNDER_ATTRIBUTES and (k not in self.__fields__ or self.__fields__[k].field_info.repr) - ] - - -_is_base_model_class_defined = True - - -@overload -def create_model( - __model_name: str, - *, - __config__: Optional[Type[BaseConfig]] = None, - __base__: None = None, - __module__: str = __name__, - __validators__: Dict[str, 'AnyClassMethod'] = None, - __cls_kwargs__: Dict[str, Any] = None, - **field_definitions: Any, -) -> Type['BaseModel']: - ... - - -@overload -def create_model( - __model_name: str, - *, - __config__: Optional[Type[BaseConfig]] = None, - __base__: Union[Type['Model'], Tuple[Type['Model'], ...]], - __module__: str = __name__, - __validators__: Dict[str, 'AnyClassMethod'] = None, - __cls_kwargs__: Dict[str, Any] = None, - **field_definitions: Any, -) -> Type['Model']: - ... - - -def create_model( - __model_name: str, - *, - __config__: Optional[Type[BaseConfig]] = None, - __base__: Union[None, Type['Model'], Tuple[Type['Model'], ...]] = None, - __module__: str = __name__, - __validators__: Dict[str, 'AnyClassMethod'] = None, - __cls_kwargs__: Dict[str, Any] = None, - __slots__: Optional[Tuple[str, ...]] = None, - **field_definitions: Any, -) -> Type['Model']: - """ - Dynamically create a model. - :param __model_name: name of the created model - :param __config__: config class to use for the new model - :param __base__: base class for the new model to inherit from - :param __module__: module of the created model - :param __validators__: a dict of method names and @validator class methods - :param __cls_kwargs__: a dict for class creation - :param __slots__: Deprecated, `__slots__` should not be passed to `create_model` - :param field_definitions: fields of the model (or extra fields if a base is supplied) - in the format `=(, )` or `=, e.g. - `foobar=(str, ...)` or `foobar=123`, or, for complex use-cases, in the format - `=` or `=(, )`, e.g. - `foo=Field(datetime, default_factory=datetime.utcnow, alias='bar')` or - `foo=(str, FieldInfo(title='Foo'))` - """ - if __slots__ is not None: - # __slots__ will be ignored from here on - warnings.warn('__slots__ should not be passed to create_model', RuntimeWarning) - - if __base__ is not None: - if __config__ is not None: - raise ConfigError('to avoid confusion __config__ and __base__ cannot be used together') - if not isinstance(__base__, tuple): - __base__ = (__base__,) - else: - __base__ = (cast(Type['Model'], BaseModel),) - - __cls_kwargs__ = __cls_kwargs__ or {} - - fields = {} - annotations = {} - - for f_name, f_def in field_definitions.items(): - if not is_valid_field(f_name): - warnings.warn(f'fields may not start with an underscore, ignoring "{f_name}"', RuntimeWarning) - if isinstance(f_def, tuple): - try: - f_annotation, f_value = f_def - except ValueError as e: - raise ConfigError( - 'field definitions should either be a tuple of (, ) or just a ' - 'default value, unfortunately this means tuples as ' - 'default values are not allowed' - ) from e - else: - f_annotation, f_value = None, f_def - - if f_annotation: - annotations[f_name] = f_annotation - fields[f_name] = f_value - - namespace: 'DictStrAny' = {'__annotations__': annotations, '__module__': __module__} - if __validators__: - namespace.update(__validators__) - namespace.update(fields) - if __config__: - namespace['Config'] = inherit_config(__config__, BaseConfig) - resolved_bases = resolve_bases(__base__) - meta, ns, kwds = prepare_class(__model_name, resolved_bases, kwds=__cls_kwargs__) - if resolved_bases is not __base__: - ns['__orig_bases__'] = __base__ - namespace.update(ns) - return meta(__model_name, resolved_bases, namespace, **kwds) - - -_missing = object() - - -def validate_model( # noqa: C901 (ignore complexity) - model: Type[BaseModel], input_data: 'DictStrAny', cls: 'ModelOrDc' = None -) -> Tuple['DictStrAny', 'SetStr', Optional[ValidationError]]: - """ - validate data against a model. - """ - values = {} - errors = [] - # input_data names, possibly alias - names_used = set() - # field names, never aliases - fields_set = set() - config = model.__config__ - check_extra = config.extra is not Extra.ignore - cls_ = cls or model - - for validator in model.__pre_root_validators__: - try: - input_data = validator(cls_, input_data) - except (ValueError, TypeError, AssertionError) as exc: - return {}, set(), ValidationError([ErrorWrapper(exc, loc=ROOT_KEY)], cls_) - - for name, field in model.__fields__.items(): - value = input_data.get(field.alias, _missing) - using_name = False - if value is _missing and config.allow_population_by_field_name and field.alt_alias: - value = input_data.get(field.name, _missing) - using_name = True - - if value is _missing: - if field.required: - errors.append(ErrorWrapper(MissingError(), loc=field.alias)) - continue - - value = field.get_default() - - if not config.validate_all and not field.validate_always: - values[name] = value - continue - else: - fields_set.add(name) - if check_extra: - names_used.add(field.name if using_name else field.alias) - - v_, errors_ = field.validate(value, values, loc=field.alias, cls=cls_) - if isinstance(errors_, ErrorWrapper): - errors.append(errors_) - elif isinstance(errors_, list): - errors.extend(errors_) - else: - values[name] = v_ - - if check_extra: - if isinstance(input_data, GetterDict): - extra = input_data.extra_keys() - names_used - else: - extra = input_data.keys() - names_used - if extra: - fields_set |= extra - if config.extra is Extra.allow: - for f in extra: - values[f] = input_data[f] - else: - for f in sorted(extra): - errors.append(ErrorWrapper(ExtraError(), loc=f)) - - for skip_on_failure, validator in model.__post_root_validators__: - if skip_on_failure and errors: - continue - try: - values = validator(cls_, values) - except (ValueError, TypeError, AssertionError) as exc: - errors.append(ErrorWrapper(exc, loc=ROOT_KEY)) - - if errors: - return values, fields_set, ValidationError(errors, cls_) - else: - return values, fields_set, None diff --git a/functions/python/data-processor/pydantic/v1/mypy.py b/functions/python/data-processor/pydantic/v1/mypy.py deleted file mode 100644 index 0a7756921..000000000 --- a/functions/python/data-processor/pydantic/v1/mypy.py +++ /dev/null @@ -1,949 +0,0 @@ -import sys -from configparser import ConfigParser -from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type as TypingType, Union - -from mypy.errorcodes import ErrorCode -from mypy.nodes import ( - ARG_NAMED, - ARG_NAMED_OPT, - ARG_OPT, - ARG_POS, - ARG_STAR2, - MDEF, - Argument, - AssignmentStmt, - Block, - CallExpr, - ClassDef, - Context, - Decorator, - EllipsisExpr, - FuncBase, - FuncDef, - JsonDict, - MemberExpr, - NameExpr, - PassStmt, - PlaceholderNode, - RefExpr, - StrExpr, - SymbolNode, - SymbolTableNode, - TempNode, - TypeInfo, - TypeVarExpr, - Var, -) -from mypy.options import Options -from mypy.plugin import ( - CheckerPluginInterface, - ClassDefContext, - FunctionContext, - MethodContext, - Plugin, - ReportConfigContext, - SemanticAnalyzerPluginInterface, -) -from mypy.plugins import dataclasses -from mypy.semanal import set_callable_name # type: ignore -from mypy.server.trigger import make_wildcard_trigger -from mypy.types import ( - AnyType, - CallableType, - Instance, - NoneType, - Overloaded, - ProperType, - Type, - TypeOfAny, - TypeType, - TypeVarId, - TypeVarType, - UnionType, - get_proper_type, -) -from mypy.typevars import fill_typevars -from mypy.util import get_unique_redefinition_name -from mypy.version import __version__ as mypy_version - -from pydantic.v1.utils import is_valid_field - -try: - from mypy.types import TypeVarDef # type: ignore[attr-defined] -except ImportError: # pragma: no cover - # Backward-compatible with TypeVarDef from Mypy 0.910. - from mypy.types import TypeVarType as TypeVarDef - -CONFIGFILE_KEY = 'pydantic-mypy' -METADATA_KEY = 'pydantic-mypy-metadata' -_NAMESPACE = __name__[:-5] # 'pydantic' in 1.10.X, 'pydantic.v1' in v2.X -BASEMODEL_FULLNAME = f'{_NAMESPACE}.main.BaseModel' -BASESETTINGS_FULLNAME = f'{_NAMESPACE}.env_settings.BaseSettings' -MODEL_METACLASS_FULLNAME = f'{_NAMESPACE}.main.ModelMetaclass' -FIELD_FULLNAME = f'{_NAMESPACE}.fields.Field' -DATACLASS_FULLNAME = f'{_NAMESPACE}.dataclasses.dataclass' - - -def parse_mypy_version(version: str) -> Tuple[int, ...]: - return tuple(map(int, version.partition('+')[0].split('.'))) - - -MYPY_VERSION_TUPLE = parse_mypy_version(mypy_version) -BUILTINS_NAME = 'builtins' if MYPY_VERSION_TUPLE >= (0, 930) else '__builtins__' - -# Increment version if plugin changes and mypy caches should be invalidated -__version__ = 2 - - -def plugin(version: str) -> 'TypingType[Plugin]': - """ - `version` is the mypy version string - - We might want to use this to print a warning if the mypy version being used is - newer, or especially older, than we expect (or need). - """ - return PydanticPlugin - - -class PydanticPlugin(Plugin): - def __init__(self, options: Options) -> None: - self.plugin_config = PydanticPluginConfig(options) - self._plugin_data = self.plugin_config.to_data() - super().__init__(options) - - def get_base_class_hook(self, fullname: str) -> 'Optional[Callable[[ClassDefContext], None]]': - sym = self.lookup_fully_qualified(fullname) - if sym and isinstance(sym.node, TypeInfo): # pragma: no branch - # No branching may occur if the mypy cache has not been cleared - if any(get_fullname(base) == BASEMODEL_FULLNAME for base in sym.node.mro): - return self._pydantic_model_class_maker_callback - return None - - def get_metaclass_hook(self, fullname: str) -> Optional[Callable[[ClassDefContext], None]]: - if fullname == MODEL_METACLASS_FULLNAME: - return self._pydantic_model_metaclass_marker_callback - return None - - def get_function_hook(self, fullname: str) -> 'Optional[Callable[[FunctionContext], Type]]': - sym = self.lookup_fully_qualified(fullname) - if sym and sym.fullname == FIELD_FULLNAME: - return self._pydantic_field_callback - return None - - def get_method_hook(self, fullname: str) -> Optional[Callable[[MethodContext], Type]]: - if fullname.endswith('.from_orm'): - return from_orm_callback - return None - - def get_class_decorator_hook(self, fullname: str) -> Optional[Callable[[ClassDefContext], None]]: - """Mark pydantic.dataclasses as dataclass. - - Mypy version 1.1.1 added support for `@dataclass_transform` decorator. - """ - if fullname == DATACLASS_FULLNAME and MYPY_VERSION_TUPLE < (1, 1): - return dataclasses.dataclass_class_maker_callback # type: ignore[return-value] - return None - - def report_config_data(self, ctx: ReportConfigContext) -> Dict[str, Any]: - """Return all plugin config data. - - Used by mypy to determine if cache needs to be discarded. - """ - return self._plugin_data - - def _pydantic_model_class_maker_callback(self, ctx: ClassDefContext) -> None: - transformer = PydanticModelTransformer(ctx, self.plugin_config) - transformer.transform() - - def _pydantic_model_metaclass_marker_callback(self, ctx: ClassDefContext) -> None: - """Reset dataclass_transform_spec attribute of ModelMetaclass. - - Let the plugin handle it. This behavior can be disabled - if 'debug_dataclass_transform' is set to True', for testing purposes. - """ - if self.plugin_config.debug_dataclass_transform: - return - info_metaclass = ctx.cls.info.declared_metaclass - assert info_metaclass, "callback not passed from 'get_metaclass_hook'" - if getattr(info_metaclass.type, 'dataclass_transform_spec', None): - info_metaclass.type.dataclass_transform_spec = None # type: ignore[attr-defined] - - def _pydantic_field_callback(self, ctx: FunctionContext) -> 'Type': - """ - Extract the type of the `default` argument from the Field function, and use it as the return type. - - In particular: - * Check whether the default and default_factory argument is specified. - * Output an error if both are specified. - * Retrieve the type of the argument which is specified, and use it as return type for the function. - """ - default_any_type = ctx.default_return_type - - assert ctx.callee_arg_names[0] == 'default', '"default" is no longer first argument in Field()' - assert ctx.callee_arg_names[1] == 'default_factory', '"default_factory" is no longer second argument in Field()' - default_args = ctx.args[0] - default_factory_args = ctx.args[1] - - if default_args and default_factory_args: - error_default_and_default_factory_specified(ctx.api, ctx.context) - return default_any_type - - if default_args: - default_type = ctx.arg_types[0][0] - default_arg = default_args[0] - - # Fallback to default Any type if the field is required - if not isinstance(default_arg, EllipsisExpr): - return default_type - - elif default_factory_args: - default_factory_type = ctx.arg_types[1][0] - - # Functions which use `ParamSpec` can be overloaded, exposing the callable's types as a parameter - # Pydantic calls the default factory without any argument, so we retrieve the first item - if isinstance(default_factory_type, Overloaded): - if MYPY_VERSION_TUPLE > (0, 910): - default_factory_type = default_factory_type.items[0] - else: - # Mypy0.910 exposes the items of overloaded types in a function - default_factory_type = default_factory_type.items()[0] # type: ignore[operator] - - if isinstance(default_factory_type, CallableType): - ret_type = get_proper_type(default_factory_type.ret_type) - if ( - isinstance(ret_type, Instance) - and ret_type.args - and all(isinstance(arg, TypeVarType) for arg in ret_type.args) - ): - # Looks like the default factory is a type like `list` or `dict`, replace all args with `Any` - ret_type = ret_type.copy_modified(args=[default_any_type] * len(ret_type.args)) - return ret_type - - return default_any_type - - -class PydanticPluginConfig: - __slots__ = ( - 'init_forbid_extra', - 'init_typed', - 'warn_required_dynamic_aliases', - 'warn_untyped_fields', - 'debug_dataclass_transform', - ) - init_forbid_extra: bool - init_typed: bool - warn_required_dynamic_aliases: bool - warn_untyped_fields: bool - debug_dataclass_transform: bool # undocumented - - def __init__(self, options: Options) -> None: - if options.config_file is None: # pragma: no cover - return - - toml_config = parse_toml(options.config_file) - if toml_config is not None: - config = toml_config.get('tool', {}).get('pydantic-mypy', {}) - for key in self.__slots__: - setting = config.get(key, False) - if not isinstance(setting, bool): - raise ValueError(f'Configuration value must be a boolean for key: {key}') - setattr(self, key, setting) - else: - plugin_config = ConfigParser() - plugin_config.read(options.config_file) - for key in self.__slots__: - setting = plugin_config.getboolean(CONFIGFILE_KEY, key, fallback=False) - setattr(self, key, setting) - - def to_data(self) -> Dict[str, Any]: - return {key: getattr(self, key) for key in self.__slots__} - - -def from_orm_callback(ctx: MethodContext) -> Type: - """ - Raise an error if orm_mode is not enabled - """ - model_type: Instance - ctx_type = ctx.type - if isinstance(ctx_type, TypeType): - ctx_type = ctx_type.item - if isinstance(ctx_type, CallableType) and isinstance(ctx_type.ret_type, Instance): - model_type = ctx_type.ret_type # called on the class - elif isinstance(ctx_type, Instance): - model_type = ctx_type # called on an instance (unusual, but still valid) - else: # pragma: no cover - detail = f'ctx.type: {ctx_type} (of type {ctx_type.__class__.__name__})' - error_unexpected_behavior(detail, ctx.api, ctx.context) - return ctx.default_return_type - pydantic_metadata = model_type.type.metadata.get(METADATA_KEY) - if pydantic_metadata is None: - return ctx.default_return_type - orm_mode = pydantic_metadata.get('config', {}).get('orm_mode') - if orm_mode is not True: - error_from_orm(get_name(model_type.type), ctx.api, ctx.context) - return ctx.default_return_type - - -class PydanticModelTransformer: - tracked_config_fields: Set[str] = { - 'extra', - 'allow_mutation', - 'frozen', - 'orm_mode', - 'allow_population_by_field_name', - 'alias_generator', - } - - def __init__(self, ctx: ClassDefContext, plugin_config: PydanticPluginConfig) -> None: - self._ctx = ctx - self.plugin_config = plugin_config - - def transform(self) -> None: - """ - Configures the BaseModel subclass according to the plugin settings. - - In particular: - * determines the model config and fields, - * adds a fields-aware signature for the initializer and construct methods - * freezes the class if allow_mutation = False or frozen = True - * stores the fields, config, and if the class is settings in the mypy metadata for access by subclasses - """ - ctx = self._ctx - info = ctx.cls.info - - self.adjust_validator_signatures() - config = self.collect_config() - fields = self.collect_fields(config) - is_settings = any(get_fullname(base) == BASESETTINGS_FULLNAME for base in info.mro[:-1]) - self.add_initializer(fields, config, is_settings) - self.add_construct_method(fields) - self.set_frozen(fields, frozen=config.allow_mutation is False or config.frozen is True) - info.metadata[METADATA_KEY] = { - 'fields': {field.name: field.serialize() for field in fields}, - 'config': config.set_values_dict(), - } - - def adjust_validator_signatures(self) -> None: - """When we decorate a function `f` with `pydantic.validator(...), mypy sees - `f` as a regular method taking a `self` instance, even though pydantic - internally wraps `f` with `classmethod` if necessary. - - Teach mypy this by marking any function whose outermost decorator is a - `validator()` call as a classmethod. - """ - for name, sym in self._ctx.cls.info.names.items(): - if isinstance(sym.node, Decorator): - first_dec = sym.node.original_decorators[0] - if ( - isinstance(first_dec, CallExpr) - and isinstance(first_dec.callee, NameExpr) - and first_dec.callee.fullname == f'{_NAMESPACE}.class_validators.validator' - ): - sym.node.func.is_class = True - - def collect_config(self) -> 'ModelConfigData': - """ - Collects the values of the config attributes that are used by the plugin, accounting for parent classes. - """ - ctx = self._ctx - cls = ctx.cls - config = ModelConfigData() - for stmt in cls.defs.body: - if not isinstance(stmt, ClassDef): - continue - if stmt.name == 'Config': - for substmt in stmt.defs.body: - if not isinstance(substmt, AssignmentStmt): - continue - config.update(self.get_config_update(substmt)) - if ( - config.has_alias_generator - and not config.allow_population_by_field_name - and self.plugin_config.warn_required_dynamic_aliases - ): - error_required_dynamic_aliases(ctx.api, stmt) - for info in cls.info.mro[1:]: # 0 is the current class - if METADATA_KEY not in info.metadata: - continue - - # Each class depends on the set of fields in its ancestors - ctx.api.add_plugin_dependency(make_wildcard_trigger(get_fullname(info))) - for name, value in info.metadata[METADATA_KEY]['config'].items(): - config.setdefault(name, value) - return config - - def collect_fields(self, model_config: 'ModelConfigData') -> List['PydanticModelField']: - """ - Collects the fields for the model, accounting for parent classes - """ - # First, collect fields belonging to the current class. - ctx = self._ctx - cls = self._ctx.cls - fields = [] # type: List[PydanticModelField] - known_fields = set() # type: Set[str] - for stmt in cls.defs.body: - if not isinstance(stmt, AssignmentStmt): # `and stmt.new_syntax` to require annotation - continue - - lhs = stmt.lvalues[0] - if not isinstance(lhs, NameExpr) or not is_valid_field(lhs.name): - continue - - if not stmt.new_syntax and self.plugin_config.warn_untyped_fields: - error_untyped_fields(ctx.api, stmt) - - # if lhs.name == '__config__': # BaseConfig not well handled; I'm not sure why yet - # continue - - sym = cls.info.names.get(lhs.name) - if sym is None: # pragma: no cover - # This is likely due to a star import (see the dataclasses plugin for a more detailed explanation) - # This is the same logic used in the dataclasses plugin - continue - - node = sym.node - if isinstance(node, PlaceholderNode): # pragma: no cover - # See the PlaceholderNode docstring for more detail about how this can occur - # Basically, it is an edge case when dealing with complex import logic - # This is the same logic used in the dataclasses plugin - continue - if not isinstance(node, Var): # pragma: no cover - # Don't know if this edge case still happens with the `is_valid_field` check above - # but better safe than sorry - continue - - # x: ClassVar[int] is ignored by dataclasses. - if node.is_classvar: - continue - - is_required = self.get_is_required(cls, stmt, lhs) - alias, has_dynamic_alias = self.get_alias_info(stmt) - if ( - has_dynamic_alias - and not model_config.allow_population_by_field_name - and self.plugin_config.warn_required_dynamic_aliases - ): - error_required_dynamic_aliases(ctx.api, stmt) - fields.append( - PydanticModelField( - name=lhs.name, - is_required=is_required, - alias=alias, - has_dynamic_alias=has_dynamic_alias, - line=stmt.line, - column=stmt.column, - ) - ) - known_fields.add(lhs.name) - all_fields = fields.copy() - for info in cls.info.mro[1:]: # 0 is the current class, -2 is BaseModel, -1 is object - if METADATA_KEY not in info.metadata: - continue - - superclass_fields = [] - # Each class depends on the set of fields in its ancestors - ctx.api.add_plugin_dependency(make_wildcard_trigger(get_fullname(info))) - - for name, data in info.metadata[METADATA_KEY]['fields'].items(): - if name not in known_fields: - field = PydanticModelField.deserialize(info, data) - known_fields.add(name) - superclass_fields.append(field) - else: - (field,) = (a for a in all_fields if a.name == name) - all_fields.remove(field) - superclass_fields.append(field) - all_fields = superclass_fields + all_fields - return all_fields - - def add_initializer(self, fields: List['PydanticModelField'], config: 'ModelConfigData', is_settings: bool) -> None: - """ - Adds a fields-aware `__init__` method to the class. - - The added `__init__` will be annotated with types vs. all `Any` depending on the plugin settings. - """ - ctx = self._ctx - typed = self.plugin_config.init_typed - use_alias = config.allow_population_by_field_name is not True - force_all_optional = is_settings or bool( - config.has_alias_generator and not config.allow_population_by_field_name - ) - init_arguments = self.get_field_arguments( - fields, typed=typed, force_all_optional=force_all_optional, use_alias=use_alias - ) - if not self.should_init_forbid_extra(fields, config): - var = Var('kwargs') - init_arguments.append(Argument(var, AnyType(TypeOfAny.explicit), None, ARG_STAR2)) - - if '__init__' not in ctx.cls.info.names: - add_method(ctx, '__init__', init_arguments, NoneType()) - - def add_construct_method(self, fields: List['PydanticModelField']) -> None: - """ - Adds a fully typed `construct` classmethod to the class. - - Similar to the fields-aware __init__ method, but always uses the field names (not aliases), - and does not treat settings fields as optional. - """ - ctx = self._ctx - set_str = ctx.api.named_type(f'{BUILTINS_NAME}.set', [ctx.api.named_type(f'{BUILTINS_NAME}.str')]) - optional_set_str = UnionType([set_str, NoneType()]) - fields_set_argument = Argument(Var('_fields_set', optional_set_str), optional_set_str, None, ARG_OPT) - construct_arguments = self.get_field_arguments(fields, typed=True, force_all_optional=False, use_alias=False) - construct_arguments = [fields_set_argument] + construct_arguments - - obj_type = ctx.api.named_type(f'{BUILTINS_NAME}.object') - self_tvar_name = '_PydanticBaseModel' # Make sure it does not conflict with other names in the class - tvar_fullname = ctx.cls.fullname + '.' + self_tvar_name - if MYPY_VERSION_TUPLE >= (1, 4): - tvd = TypeVarType( - self_tvar_name, - tvar_fullname, - ( - TypeVarId(-1, namespace=ctx.cls.fullname + '.construct') - if MYPY_VERSION_TUPLE >= (1, 11) - else TypeVarId(-1) - ), - [], - obj_type, - AnyType(TypeOfAny.from_omitted_generics), # type: ignore[arg-type] - ) - self_tvar_expr = TypeVarExpr( - self_tvar_name, - tvar_fullname, - [], - obj_type, - AnyType(TypeOfAny.from_omitted_generics), # type: ignore[arg-type] - ) - else: - tvd = TypeVarDef(self_tvar_name, tvar_fullname, -1, [], obj_type) - self_tvar_expr = TypeVarExpr(self_tvar_name, tvar_fullname, [], obj_type) - ctx.cls.info.names[self_tvar_name] = SymbolTableNode(MDEF, self_tvar_expr) - - # Backward-compatible with TypeVarDef from Mypy 0.910. - if isinstance(tvd, TypeVarType): - self_type = tvd - else: - self_type = TypeVarType(tvd) - - add_method( - ctx, - 'construct', - construct_arguments, - return_type=self_type, - self_type=self_type, - tvar_def=tvd, - is_classmethod=True, - ) - - def set_frozen(self, fields: List['PydanticModelField'], frozen: bool) -> None: - """ - Marks all fields as properties so that attempts to set them trigger mypy errors. - - This is the same approach used by the attrs and dataclasses plugins. - """ - ctx = self._ctx - info = ctx.cls.info - for field in fields: - sym_node = info.names.get(field.name) - if sym_node is not None: - var = sym_node.node - if isinstance(var, Var): - var.is_property = frozen - elif isinstance(var, PlaceholderNode) and not ctx.api.final_iteration: - # See https://github.com/pydantic/pydantic/issues/5191 to hit this branch for test coverage - ctx.api.defer() - else: # pragma: no cover - # I don't know whether it's possible to hit this branch, but I've added it for safety - try: - var_str = str(var) - except TypeError: - # This happens for PlaceholderNode; perhaps it will happen for other types in the future.. - var_str = repr(var) - detail = f'sym_node.node: {var_str} (of type {var.__class__})' - error_unexpected_behavior(detail, ctx.api, ctx.cls) - else: - var = field.to_var(info, use_alias=False) - var.info = info - var.is_property = frozen - var._fullname = get_fullname(info) + '.' + get_name(var) - info.names[get_name(var)] = SymbolTableNode(MDEF, var) - - def get_config_update(self, substmt: AssignmentStmt) -> Optional['ModelConfigData']: - """ - Determines the config update due to a single statement in the Config class definition. - - Warns if a tracked config attribute is set to a value the plugin doesn't know how to interpret (e.g., an int) - """ - lhs = substmt.lvalues[0] - if not (isinstance(lhs, NameExpr) and lhs.name in self.tracked_config_fields): - return None - if lhs.name == 'extra': - if isinstance(substmt.rvalue, StrExpr): - forbid_extra = substmt.rvalue.value == 'forbid' - elif isinstance(substmt.rvalue, MemberExpr): - forbid_extra = substmt.rvalue.name == 'forbid' - else: - error_invalid_config_value(lhs.name, self._ctx.api, substmt) - return None - return ModelConfigData(forbid_extra=forbid_extra) - if lhs.name == 'alias_generator': - has_alias_generator = True - if isinstance(substmt.rvalue, NameExpr) and substmt.rvalue.fullname == 'builtins.None': - has_alias_generator = False - return ModelConfigData(has_alias_generator=has_alias_generator) - if isinstance(substmt.rvalue, NameExpr) and substmt.rvalue.fullname in ('builtins.True', 'builtins.False'): - return ModelConfigData(**{lhs.name: substmt.rvalue.fullname == 'builtins.True'}) - error_invalid_config_value(lhs.name, self._ctx.api, substmt) - return None - - @staticmethod - def get_is_required(cls: ClassDef, stmt: AssignmentStmt, lhs: NameExpr) -> bool: - """ - Returns a boolean indicating whether the field defined in `stmt` is a required field. - """ - expr = stmt.rvalue - if isinstance(expr, TempNode): - # TempNode means annotation-only, so only non-required if Optional - value_type = get_proper_type(cls.info[lhs.name].type) - return not PydanticModelTransformer.type_has_implicit_default(value_type) - if isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME: - # The "default value" is a call to `Field`; at this point, the field is - # only required if default is Ellipsis (i.e., `field_name: Annotation = Field(...)`) or if default_factory - # is specified. - for arg, name in zip(expr.args, expr.arg_names): - # If name is None, then this arg is the default because it is the only positional argument. - if name is None or name == 'default': - return arg.__class__ is EllipsisExpr - if name == 'default_factory': - return False - # In this case, default and default_factory are not specified, so we need to look at the annotation - value_type = get_proper_type(cls.info[lhs.name].type) - return not PydanticModelTransformer.type_has_implicit_default(value_type) - # Only required if the "default value" is Ellipsis (i.e., `field_name: Annotation = ...`) - return isinstance(expr, EllipsisExpr) - - @staticmethod - def type_has_implicit_default(type_: Optional[ProperType]) -> bool: - """ - Returns True if the passed type will be given an implicit default value. - - In pydantic v1, this is the case for Optional types and Any (with default value None). - """ - if isinstance(type_, AnyType): - # Annotated as Any - return True - if isinstance(type_, UnionType) and any( - isinstance(item, NoneType) or isinstance(item, AnyType) for item in type_.items - ): - # Annotated as Optional, or otherwise having NoneType or AnyType in the union - return True - return False - - @staticmethod - def get_alias_info(stmt: AssignmentStmt) -> Tuple[Optional[str], bool]: - """ - Returns a pair (alias, has_dynamic_alias), extracted from the declaration of the field defined in `stmt`. - - `has_dynamic_alias` is True if and only if an alias is provided, but not as a string literal. - If `has_dynamic_alias` is True, `alias` will be None. - """ - expr = stmt.rvalue - if isinstance(expr, TempNode): - # TempNode means annotation-only - return None, False - - if not ( - isinstance(expr, CallExpr) and isinstance(expr.callee, RefExpr) and expr.callee.fullname == FIELD_FULLNAME - ): - # Assigned value is not a call to pydantic.fields.Field - return None, False - - for i, arg_name in enumerate(expr.arg_names): - if arg_name != 'alias': - continue - arg = expr.args[i] - if isinstance(arg, StrExpr): - return arg.value, False - else: - return None, True - return None, False - - def get_field_arguments( - self, fields: List['PydanticModelField'], typed: bool, force_all_optional: bool, use_alias: bool - ) -> List[Argument]: - """ - Helper function used during the construction of the `__init__` and `construct` method signatures. - - Returns a list of mypy Argument instances for use in the generated signatures. - """ - info = self._ctx.cls.info - arguments = [ - field.to_argument(info, typed=typed, force_optional=force_all_optional, use_alias=use_alias) - for field in fields - if not (use_alias and field.has_dynamic_alias) - ] - return arguments - - def should_init_forbid_extra(self, fields: List['PydanticModelField'], config: 'ModelConfigData') -> bool: - """ - Indicates whether the generated `__init__` should get a `**kwargs` at the end of its signature - - We disallow arbitrary kwargs if the extra config setting is "forbid", or if the plugin config says to, - *unless* a required dynamic alias is present (since then we can't determine a valid signature). - """ - if not config.allow_population_by_field_name: - if self.is_dynamic_alias_present(fields, bool(config.has_alias_generator)): - return False - if config.forbid_extra: - return True - return self.plugin_config.init_forbid_extra - - @staticmethod - def is_dynamic_alias_present(fields: List['PydanticModelField'], has_alias_generator: bool) -> bool: - """ - Returns whether any fields on the model have a "dynamic alias", i.e., an alias that cannot be - determined during static analysis. - """ - for field in fields: - if field.has_dynamic_alias: - return True - if has_alias_generator: - for field in fields: - if field.alias is None: - return True - return False - - -class PydanticModelField: - def __init__( - self, name: str, is_required: bool, alias: Optional[str], has_dynamic_alias: bool, line: int, column: int - ): - self.name = name - self.is_required = is_required - self.alias = alias - self.has_dynamic_alias = has_dynamic_alias - self.line = line - self.column = column - - def to_var(self, info: TypeInfo, use_alias: bool) -> Var: - name = self.name - if use_alias and self.alias is not None: - name = self.alias - return Var(name, info[self.name].type) - - def to_argument(self, info: TypeInfo, typed: bool, force_optional: bool, use_alias: bool) -> Argument: - if typed and info[self.name].type is not None: - type_annotation = info[self.name].type - else: - type_annotation = AnyType(TypeOfAny.explicit) - return Argument( - variable=self.to_var(info, use_alias), - type_annotation=type_annotation, - initializer=None, - kind=ARG_NAMED_OPT if force_optional or not self.is_required else ARG_NAMED, - ) - - def serialize(self) -> JsonDict: - return self.__dict__ - - @classmethod - def deserialize(cls, info: TypeInfo, data: JsonDict) -> 'PydanticModelField': - return cls(**data) - - -class ModelConfigData: - def __init__( - self, - forbid_extra: Optional[bool] = None, - allow_mutation: Optional[bool] = None, - frozen: Optional[bool] = None, - orm_mode: Optional[bool] = None, - allow_population_by_field_name: Optional[bool] = None, - has_alias_generator: Optional[bool] = None, - ): - self.forbid_extra = forbid_extra - self.allow_mutation = allow_mutation - self.frozen = frozen - self.orm_mode = orm_mode - self.allow_population_by_field_name = allow_population_by_field_name - self.has_alias_generator = has_alias_generator - - def set_values_dict(self) -> Dict[str, Any]: - return {k: v for k, v in self.__dict__.items() if v is not None} - - def update(self, config: Optional['ModelConfigData']) -> None: - if config is None: - return - for k, v in config.set_values_dict().items(): - setattr(self, k, v) - - def setdefault(self, key: str, value: Any) -> None: - if getattr(self, key) is None: - setattr(self, key, value) - - -ERROR_ORM = ErrorCode('pydantic-orm', 'Invalid from_orm call', 'Pydantic') -ERROR_CONFIG = ErrorCode('pydantic-config', 'Invalid config value', 'Pydantic') -ERROR_ALIAS = ErrorCode('pydantic-alias', 'Dynamic alias disallowed', 'Pydantic') -ERROR_UNEXPECTED = ErrorCode('pydantic-unexpected', 'Unexpected behavior', 'Pydantic') -ERROR_UNTYPED = ErrorCode('pydantic-field', 'Untyped field disallowed', 'Pydantic') -ERROR_FIELD_DEFAULTS = ErrorCode('pydantic-field', 'Invalid Field defaults', 'Pydantic') - - -def error_from_orm(model_name: str, api: CheckerPluginInterface, context: Context) -> None: - api.fail(f'"{model_name}" does not have orm_mode=True', context, code=ERROR_ORM) - - -def error_invalid_config_value(name: str, api: SemanticAnalyzerPluginInterface, context: Context) -> None: - api.fail(f'Invalid value for "Config.{name}"', context, code=ERROR_CONFIG) - - -def error_required_dynamic_aliases(api: SemanticAnalyzerPluginInterface, context: Context) -> None: - api.fail('Required dynamic aliases disallowed', context, code=ERROR_ALIAS) - - -def error_unexpected_behavior( - detail: str, api: Union[CheckerPluginInterface, SemanticAnalyzerPluginInterface], context: Context -) -> None: # pragma: no cover - # Can't think of a good way to test this, but I confirmed it renders as desired by adding to a non-error path - link = 'https://github.com/pydantic/pydantic/issues/new/choose' - full_message = f'The pydantic mypy plugin ran into unexpected behavior: {detail}\n' - full_message += f'Please consider reporting this bug at {link} so we can try to fix it!' - api.fail(full_message, context, code=ERROR_UNEXPECTED) - - -def error_untyped_fields(api: SemanticAnalyzerPluginInterface, context: Context) -> None: - api.fail('Untyped fields disallowed', context, code=ERROR_UNTYPED) - - -def error_default_and_default_factory_specified(api: CheckerPluginInterface, context: Context) -> None: - api.fail('Field default and default_factory cannot be specified together', context, code=ERROR_FIELD_DEFAULTS) - - -def add_method( - ctx: ClassDefContext, - name: str, - args: List[Argument], - return_type: Type, - self_type: Optional[Type] = None, - tvar_def: Optional[TypeVarDef] = None, - is_classmethod: bool = False, - is_new: bool = False, - # is_staticmethod: bool = False, -) -> None: - """ - Adds a new method to a class. - - This can be dropped if/when https://github.com/python/mypy/issues/7301 is merged - """ - info = ctx.cls.info - - # First remove any previously generated methods with the same name - # to avoid clashes and problems in the semantic analyzer. - if name in info.names: - sym = info.names[name] - if sym.plugin_generated and isinstance(sym.node, FuncDef): - ctx.cls.defs.body.remove(sym.node) # pragma: no cover - - self_type = self_type or fill_typevars(info) - if is_classmethod or is_new: - first = [Argument(Var('_cls'), TypeType.make_normalized(self_type), None, ARG_POS)] - # elif is_staticmethod: - # first = [] - else: - self_type = self_type or fill_typevars(info) - first = [Argument(Var('__pydantic_self__'), self_type, None, ARG_POS)] - args = first + args - arg_types, arg_names, arg_kinds = [], [], [] - for arg in args: - assert arg.type_annotation, 'All arguments must be fully typed.' - arg_types.append(arg.type_annotation) - arg_names.append(get_name(arg.variable)) - arg_kinds.append(arg.kind) - - function_type = ctx.api.named_type(f'{BUILTINS_NAME}.function') - signature = CallableType( - arg_types, arg_kinds, arg_names, return_type, function_type, variables=[tvar_def] if tvar_def else None - ) - - func = FuncDef(name, args, Block([PassStmt()])) - func.info = info - func.type = set_callable_name(signature, func) - func.is_class = is_classmethod - # func.is_static = is_staticmethod - func._fullname = get_fullname(info) + '.' + name - func.line = info.line - - # NOTE: we would like the plugin generated node to dominate, but we still - # need to keep any existing definitions so they get semantically analyzed. - if name in info.names: - # Get a nice unique name instead. - r_name = get_unique_redefinition_name(name, info.names) - info.names[r_name] = info.names[name] - - if is_classmethod: # or is_staticmethod: - func.is_decorated = True - v = Var(name, func.type) - v.info = info - v._fullname = func._fullname - # if is_classmethod: - v.is_classmethod = True - dec = Decorator(func, [NameExpr('classmethod')], v) - # else: - # v.is_staticmethod = True - # dec = Decorator(func, [NameExpr('staticmethod')], v) - - dec.line = info.line - sym = SymbolTableNode(MDEF, dec) - else: - sym = SymbolTableNode(MDEF, func) - sym.plugin_generated = True - - info.names[name] = sym - info.defn.defs.body.append(func) - - -def get_fullname(x: Union[FuncBase, SymbolNode]) -> str: - """ - Used for compatibility with mypy 0.740; can be dropped once support for 0.740 is dropped. - """ - fn = x.fullname - if callable(fn): # pragma: no cover - return fn() - return fn - - -def get_name(x: Union[FuncBase, SymbolNode]) -> str: - """ - Used for compatibility with mypy 0.740; can be dropped once support for 0.740 is dropped. - """ - fn = x.name - if callable(fn): # pragma: no cover - return fn() - return fn - - -def parse_toml(config_file: str) -> Optional[Dict[str, Any]]: - if not config_file.endswith('.toml'): - return None - - read_mode = 'rb' - if sys.version_info >= (3, 11): - import tomllib as toml_ - else: - try: - import tomli as toml_ - except ImportError: - # older versions of mypy have toml as a dependency, not tomli - read_mode = 'r' - try: - import toml as toml_ # type: ignore[no-redef] - except ImportError: # pragma: no cover - import warnings - - warnings.warn('No TOML parser installed, cannot read configuration from `pyproject.toml`.') - return None - - with open(config_file, read_mode) as rf: - return toml_.load(rf) # type: ignore[arg-type] diff --git a/functions/python/data-processor/pydantic/v1/networks.py b/functions/python/data-processor/pydantic/v1/networks.py deleted file mode 100644 index ba07b7486..000000000 --- a/functions/python/data-processor/pydantic/v1/networks.py +++ /dev/null @@ -1,747 +0,0 @@ -import re -from ipaddress import ( - IPv4Address, - IPv4Interface, - IPv4Network, - IPv6Address, - IPv6Interface, - IPv6Network, - _BaseAddress, - _BaseNetwork, -) -from typing import ( - TYPE_CHECKING, - Any, - Collection, - Dict, - Generator, - List, - Match, - Optional, - Pattern, - Set, - Tuple, - Type, - Union, - cast, - no_type_check, -) - -from pydantic.v1 import errors -from pydantic.v1.utils import Representation, update_not_none -from pydantic.v1.validators import constr_length_validator, str_validator - -if TYPE_CHECKING: - import email_validator - from typing_extensions import TypedDict - - from pydantic.v1.config import BaseConfig - from pydantic.v1.fields import ModelField - from pydantic.v1.typing import AnyCallable - - CallableGenerator = Generator[AnyCallable, None, None] - - class Parts(TypedDict, total=False): - scheme: str - user: Optional[str] - password: Optional[str] - ipv4: Optional[str] - ipv6: Optional[str] - domain: Optional[str] - port: Optional[str] - path: Optional[str] - query: Optional[str] - fragment: Optional[str] - - class HostParts(TypedDict, total=False): - host: str - tld: Optional[str] - host_type: Optional[str] - port: Optional[str] - rebuild: bool - -else: - email_validator = None - - class Parts(dict): - pass - - -NetworkType = Union[str, bytes, int, Tuple[Union[str, bytes, int], Union[str, int]]] - -__all__ = [ - 'AnyUrl', - 'AnyHttpUrl', - 'FileUrl', - 'HttpUrl', - 'stricturl', - 'EmailStr', - 'NameEmail', - 'IPvAnyAddress', - 'IPvAnyInterface', - 'IPvAnyNetwork', - 'PostgresDsn', - 'CockroachDsn', - 'AmqpDsn', - 'RedisDsn', - 'MongoDsn', - 'KafkaDsn', - 'validate_email', -] - -_url_regex_cache = None -_multi_host_url_regex_cache = None -_ascii_domain_regex_cache = None -_int_domain_regex_cache = None -_host_regex_cache = None - -_host_regex = ( - r'(?:' - r'(?P(?:\d{1,3}\.){3}\d{1,3})(?=$|[/:#?])|' # ipv4 - r'(?P\[[A-F0-9]*:[A-F0-9:]+\])(?=$|[/:#?])|' # ipv6 - r'(?P[^\s/:?#]+)' # domain, validation occurs later - r')?' - r'(?::(?P\d+))?' # port -) -_scheme_regex = r'(?:(?P[a-z][a-z0-9+\-.]+)://)?' # scheme https://tools.ietf.org/html/rfc3986#appendix-A -_user_info_regex = r'(?:(?P[^\s:/]*)(?::(?P[^\s/]*))?@)?' -_path_regex = r'(?P/[^\s?#]*)?' -_query_regex = r'(?:\?(?P[^\s#]*))?' -_fragment_regex = r'(?:#(?P[^\s#]*))?' - - -def url_regex() -> Pattern[str]: - global _url_regex_cache - if _url_regex_cache is None: - _url_regex_cache = re.compile( - rf'{_scheme_regex}{_user_info_regex}{_host_regex}{_path_regex}{_query_regex}{_fragment_regex}', - re.IGNORECASE, - ) - return _url_regex_cache - - -def multi_host_url_regex() -> Pattern[str]: - """ - Compiled multi host url regex. - - Additionally to `url_regex` it allows to match multiple hosts. - E.g. host1.db.net,host2.db.net - """ - global _multi_host_url_regex_cache - if _multi_host_url_regex_cache is None: - _multi_host_url_regex_cache = re.compile( - rf'{_scheme_regex}{_user_info_regex}' - r'(?P([^/]*))' # validation occurs later - rf'{_path_regex}{_query_regex}{_fragment_regex}', - re.IGNORECASE, - ) - return _multi_host_url_regex_cache - - -def ascii_domain_regex() -> Pattern[str]: - global _ascii_domain_regex_cache - if _ascii_domain_regex_cache is None: - ascii_chunk = r'[_0-9a-z](?:[-_0-9a-z]{0,61}[_0-9a-z])?' - ascii_domain_ending = r'(?P\.[a-z]{2,63})?\.?' - _ascii_domain_regex_cache = re.compile( - fr'(?:{ascii_chunk}\.)*?{ascii_chunk}{ascii_domain_ending}', re.IGNORECASE - ) - return _ascii_domain_regex_cache - - -def int_domain_regex() -> Pattern[str]: - global _int_domain_regex_cache - if _int_domain_regex_cache is None: - int_chunk = r'[_0-9a-\U00040000](?:[-_0-9a-\U00040000]{0,61}[_0-9a-\U00040000])?' - int_domain_ending = r'(?P(\.[^\W\d_]{2,63})|(\.(?:xn--)[_0-9a-z-]{2,63}))?\.?' - _int_domain_regex_cache = re.compile(fr'(?:{int_chunk}\.)*?{int_chunk}{int_domain_ending}', re.IGNORECASE) - return _int_domain_regex_cache - - -def host_regex() -> Pattern[str]: - global _host_regex_cache - if _host_regex_cache is None: - _host_regex_cache = re.compile( - _host_regex, - re.IGNORECASE, - ) - return _host_regex_cache - - -class AnyUrl(str): - strip_whitespace = True - min_length = 1 - max_length = 2**16 - allowed_schemes: Optional[Collection[str]] = None - tld_required: bool = False - user_required: bool = False - host_required: bool = True - hidden_parts: Set[str] = set() - - __slots__ = ('scheme', 'user', 'password', 'host', 'tld', 'host_type', 'port', 'path', 'query', 'fragment') - - @no_type_check - def __new__(cls, url: Optional[str], **kwargs) -> object: - return str.__new__(cls, cls.build(**kwargs) if url is None else url) - - def __init__( - self, - url: str, - *, - scheme: str, - user: Optional[str] = None, - password: Optional[str] = None, - host: Optional[str] = None, - tld: Optional[str] = None, - host_type: str = 'domain', - port: Optional[str] = None, - path: Optional[str] = None, - query: Optional[str] = None, - fragment: Optional[str] = None, - ) -> None: - str.__init__(url) - self.scheme = scheme - self.user = user - self.password = password - self.host = host - self.tld = tld - self.host_type = host_type - self.port = port - self.path = path - self.query = query - self.fragment = fragment - - @classmethod - def build( - cls, - *, - scheme: str, - user: Optional[str] = None, - password: Optional[str] = None, - host: str, - port: Optional[str] = None, - path: Optional[str] = None, - query: Optional[str] = None, - fragment: Optional[str] = None, - **_kwargs: str, - ) -> str: - parts = Parts( - scheme=scheme, - user=user, - password=password, - host=host, - port=port, - path=path, - query=query, - fragment=fragment, - **_kwargs, # type: ignore[misc] - ) - - url = scheme + '://' - if user: - url += user - if password: - url += ':' + password - if user or password: - url += '@' - url += host - if port and ('port' not in cls.hidden_parts or cls.get_default_parts(parts).get('port') != port): - url += ':' + port - if path: - url += path - if query: - url += '?' + query - if fragment: - url += '#' + fragment - return url - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none(field_schema, minLength=cls.min_length, maxLength=cls.max_length, format='uri') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, value: Any, field: 'ModelField', config: 'BaseConfig') -> 'AnyUrl': - if value.__class__ == cls: - return value - value = str_validator(value) - if cls.strip_whitespace: - value = value.strip() - url: str = cast(str, constr_length_validator(value, field, config)) - - m = cls._match_url(url) - # the regex should always match, if it doesn't please report with details of the URL tried - assert m, 'URL regex failed unexpectedly' - - original_parts = cast('Parts', m.groupdict()) - parts = cls.apply_default_parts(original_parts) - parts = cls.validate_parts(parts) - - if m.end() != len(url): - raise errors.UrlExtraError(extra=url[m.end() :]) - - return cls._build_url(m, url, parts) - - @classmethod - def _build_url(cls, m: Match[str], url: str, parts: 'Parts') -> 'AnyUrl': - """ - Validate hosts and build the AnyUrl object. Split from `validate` so this method - can be altered in `MultiHostDsn`. - """ - host, tld, host_type, rebuild = cls.validate_host(parts) - - return cls( - None if rebuild else url, - scheme=parts['scheme'], - user=parts['user'], - password=parts['password'], - host=host, - tld=tld, - host_type=host_type, - port=parts['port'], - path=parts['path'], - query=parts['query'], - fragment=parts['fragment'], - ) - - @staticmethod - def _match_url(url: str) -> Optional[Match[str]]: - return url_regex().match(url) - - @staticmethod - def _validate_port(port: Optional[str]) -> None: - if port is not None and int(port) > 65_535: - raise errors.UrlPortError() - - @classmethod - def validate_parts(cls, parts: 'Parts', validate_port: bool = True) -> 'Parts': - """ - A method used to validate parts of a URL. - Could be overridden to set default values for parts if missing - """ - scheme = parts['scheme'] - if scheme is None: - raise errors.UrlSchemeError() - - if cls.allowed_schemes and scheme.lower() not in cls.allowed_schemes: - raise errors.UrlSchemePermittedError(set(cls.allowed_schemes)) - - if validate_port: - cls._validate_port(parts['port']) - - user = parts['user'] - if cls.user_required and user is None: - raise errors.UrlUserInfoError() - - return parts - - @classmethod - def validate_host(cls, parts: 'Parts') -> Tuple[str, Optional[str], str, bool]: - tld, host_type, rebuild = None, None, False - for f in ('domain', 'ipv4', 'ipv6'): - host = parts[f] # type: ignore[literal-required] - if host: - host_type = f - break - - if host is None: - if cls.host_required: - raise errors.UrlHostError() - elif host_type == 'domain': - is_international = False - d = ascii_domain_regex().fullmatch(host) - if d is None: - d = int_domain_regex().fullmatch(host) - if d is None: - raise errors.UrlHostError() - is_international = True - - tld = d.group('tld') - if tld is None and not is_international: - d = int_domain_regex().fullmatch(host) - assert d is not None - tld = d.group('tld') - is_international = True - - if tld is not None: - tld = tld[1:] - elif cls.tld_required: - raise errors.UrlHostTldError() - - if is_international: - host_type = 'int_domain' - rebuild = True - host = host.encode('idna').decode('ascii') - if tld is not None: - tld = tld.encode('idna').decode('ascii') - - return host, tld, host_type, rebuild # type: ignore - - @staticmethod - def get_default_parts(parts: 'Parts') -> 'Parts': - return {} - - @classmethod - def apply_default_parts(cls, parts: 'Parts') -> 'Parts': - for key, value in cls.get_default_parts(parts).items(): - if not parts[key]: # type: ignore[literal-required] - parts[key] = value # type: ignore[literal-required] - return parts - - def __repr__(self) -> str: - extra = ', '.join(f'{n}={getattr(self, n)!r}' for n in self.__slots__ if getattr(self, n) is not None) - return f'{self.__class__.__name__}({super().__repr__()}, {extra})' - - -class AnyHttpUrl(AnyUrl): - allowed_schemes = {'http', 'https'} - - __slots__ = () - - -class HttpUrl(AnyHttpUrl): - tld_required = True - # https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers - max_length = 2083 - hidden_parts = {'port'} - - @staticmethod - def get_default_parts(parts: 'Parts') -> 'Parts': - return {'port': '80' if parts['scheme'] == 'http' else '443'} - - -class FileUrl(AnyUrl): - allowed_schemes = {'file'} - host_required = False - - __slots__ = () - - -class MultiHostDsn(AnyUrl): - __slots__ = AnyUrl.__slots__ + ('hosts',) - - def __init__(self, *args: Any, hosts: Optional[List['HostParts']] = None, **kwargs: Any): - super().__init__(*args, **kwargs) - self.hosts = hosts - - @staticmethod - def _match_url(url: str) -> Optional[Match[str]]: - return multi_host_url_regex().match(url) - - @classmethod - def validate_parts(cls, parts: 'Parts', validate_port: bool = True) -> 'Parts': - return super().validate_parts(parts, validate_port=False) - - @classmethod - def _build_url(cls, m: Match[str], url: str, parts: 'Parts') -> 'MultiHostDsn': - hosts_parts: List['HostParts'] = [] - host_re = host_regex() - for host in m.groupdict()['hosts'].split(','): - d: Parts = host_re.match(host).groupdict() # type: ignore - host, tld, host_type, rebuild = cls.validate_host(d) - port = d.get('port') - cls._validate_port(port) - hosts_parts.append( - { - 'host': host, - 'host_type': host_type, - 'tld': tld, - 'rebuild': rebuild, - 'port': port, - } - ) - - if len(hosts_parts) > 1: - return cls( - None if any([hp['rebuild'] for hp in hosts_parts]) else url, - scheme=parts['scheme'], - user=parts['user'], - password=parts['password'], - path=parts['path'], - query=parts['query'], - fragment=parts['fragment'], - host_type=None, - hosts=hosts_parts, - ) - else: - # backwards compatibility with single host - host_part = hosts_parts[0] - return cls( - None if host_part['rebuild'] else url, - scheme=parts['scheme'], - user=parts['user'], - password=parts['password'], - host=host_part['host'], - tld=host_part['tld'], - host_type=host_part['host_type'], - port=host_part.get('port'), - path=parts['path'], - query=parts['query'], - fragment=parts['fragment'], - ) - - -class PostgresDsn(MultiHostDsn): - allowed_schemes = { - 'postgres', - 'postgresql', - 'postgresql+asyncpg', - 'postgresql+pg8000', - 'postgresql+psycopg', - 'postgresql+psycopg2', - 'postgresql+psycopg2cffi', - 'postgresql+py-postgresql', - 'postgresql+pygresql', - } - user_required = True - - __slots__ = () - - -class CockroachDsn(AnyUrl): - allowed_schemes = { - 'cockroachdb', - 'cockroachdb+psycopg2', - 'cockroachdb+asyncpg', - } - user_required = True - - -class AmqpDsn(AnyUrl): - allowed_schemes = {'amqp', 'amqps'} - host_required = False - - -class RedisDsn(AnyUrl): - __slots__ = () - allowed_schemes = {'redis', 'rediss'} - host_required = False - - @staticmethod - def get_default_parts(parts: 'Parts') -> 'Parts': - return { - 'domain': 'localhost' if not (parts['ipv4'] or parts['ipv6']) else '', - 'port': '6379', - 'path': '/0', - } - - -class MongoDsn(AnyUrl): - allowed_schemes = {'mongodb'} - - # TODO: Needed to generic "Parts" for "Replica Set", "Sharded Cluster", and other mongodb deployment modes - @staticmethod - def get_default_parts(parts: 'Parts') -> 'Parts': - return { - 'port': '27017', - } - - -class KafkaDsn(AnyUrl): - allowed_schemes = {'kafka'} - - @staticmethod - def get_default_parts(parts: 'Parts') -> 'Parts': - return { - 'domain': 'localhost', - 'port': '9092', - } - - -def stricturl( - *, - strip_whitespace: bool = True, - min_length: int = 1, - max_length: int = 2**16, - tld_required: bool = True, - host_required: bool = True, - allowed_schemes: Optional[Collection[str]] = None, -) -> Type[AnyUrl]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict( - strip_whitespace=strip_whitespace, - min_length=min_length, - max_length=max_length, - tld_required=tld_required, - host_required=host_required, - allowed_schemes=allowed_schemes, - ) - return type('UrlValue', (AnyUrl,), namespace) - - -def import_email_validator() -> None: - global email_validator - try: - import email_validator - except ImportError as e: - raise ImportError('email-validator is not installed, run `pip install pydantic[email]`') from e - - -class EmailStr(str): - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='email') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - # included here and below so the error happens straight away - import_email_validator() - - yield str_validator - yield cls.validate - - @classmethod - def validate(cls, value: Union[str]) -> str: - return validate_email(value)[1] - - -class NameEmail(Representation): - __slots__ = 'name', 'email' - - def __init__(self, name: str, email: str): - self.name = name - self.email = email - - def __eq__(self, other: Any) -> bool: - return isinstance(other, NameEmail) and (self.name, self.email) == (other.name, other.email) - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='name-email') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - import_email_validator() - - yield cls.validate - - @classmethod - def validate(cls, value: Any) -> 'NameEmail': - if value.__class__ == cls: - return value - value = str_validator(value) - return cls(*validate_email(value)) - - def __str__(self) -> str: - return f'{self.name} <{self.email}>' - - -class IPvAnyAddress(_BaseAddress): - __slots__ = () - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='ipvanyaddress') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, value: Union[str, bytes, int]) -> Union[IPv4Address, IPv6Address]: - try: - return IPv4Address(value) - except ValueError: - pass - - try: - return IPv6Address(value) - except ValueError: - raise errors.IPvAnyAddressError() - - -class IPvAnyInterface(_BaseAddress): - __slots__ = () - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='ipvanyinterface') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, value: NetworkType) -> Union[IPv4Interface, IPv6Interface]: - try: - return IPv4Interface(value) - except ValueError: - pass - - try: - return IPv6Interface(value) - except ValueError: - raise errors.IPvAnyInterfaceError() - - -class IPvAnyNetwork(_BaseNetwork): # type: ignore - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='ipvanynetwork') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, value: NetworkType) -> Union[IPv4Network, IPv6Network]: - # Assume IP Network is defined with a default value for ``strict`` argument. - # Define your own class if you want to specify network address check strictness. - try: - return IPv4Network(value) - except ValueError: - pass - - try: - return IPv6Network(value) - except ValueError: - raise errors.IPvAnyNetworkError() - - -pretty_email_regex = re.compile(r'([\w ]*?) *<(.*)> *') -MAX_EMAIL_LENGTH = 2048 -"""Maximum length for an email. -A somewhat arbitrary but very generous number compared to what is allowed by most implementations. -""" - - -def validate_email(value: Union[str]) -> Tuple[str, str]: - """ - Email address validation using https://pypi.org/project/email-validator/ - Notes: - * raw ip address (literal) domain parts are not allowed. - * "John Doe " style "pretty" email addresses are processed - * spaces are striped from the beginning and end of addresses but no error is raised - """ - if email_validator is None: - import_email_validator() - - if len(value) > MAX_EMAIL_LENGTH: - raise errors.EmailError() - - m = pretty_email_regex.fullmatch(value) - name: Union[str, None] = None - if m: - name, value = m.groups() - email = value.strip() - try: - parts = email_validator.validate_email(email, check_deliverability=False) - except email_validator.EmailNotValidError as e: - raise errors.EmailError from e - - if hasattr(parts, 'normalized'): - # email-validator >= 2 - email = parts.normalized - assert email is not None - name = name or parts.local_part - return name, email - else: - # email-validator >1, <2 - at_index = email.index('@') - local_part = email[:at_index] # RFC 5321, local part must be case-sensitive. - global_part = email[at_index:].lower() - - return name or local_part, local_part + global_part diff --git a/functions/python/data-processor/pydantic/v1/parse.py b/functions/python/data-processor/pydantic/v1/parse.py deleted file mode 100644 index 431d75a64..000000000 --- a/functions/python/data-processor/pydantic/v1/parse.py +++ /dev/null @@ -1,66 +0,0 @@ -import json -import pickle -from enum import Enum -from pathlib import Path -from typing import Any, Callable, Union - -from pydantic.v1.types import StrBytes - - -class Protocol(str, Enum): - json = 'json' - pickle = 'pickle' - - -def load_str_bytes( - b: StrBytes, - *, - content_type: str = None, - encoding: str = 'utf8', - proto: Protocol = None, - allow_pickle: bool = False, - json_loads: Callable[[str], Any] = json.loads, -) -> Any: - if proto is None and content_type: - if content_type.endswith(('json', 'javascript')): - pass - elif allow_pickle and content_type.endswith('pickle'): - proto = Protocol.pickle - else: - raise TypeError(f'Unknown content-type: {content_type}') - - proto = proto or Protocol.json - - if proto == Protocol.json: - if isinstance(b, bytes): - b = b.decode(encoding) - return json_loads(b) - elif proto == Protocol.pickle: - if not allow_pickle: - raise RuntimeError('Trying to decode with pickle with allow_pickle=False') - bb = b if isinstance(b, bytes) else b.encode() - return pickle.loads(bb) - else: - raise TypeError(f'Unknown protocol: {proto}') - - -def load_file( - path: Union[str, Path], - *, - content_type: str = None, - encoding: str = 'utf8', - proto: Protocol = None, - allow_pickle: bool = False, - json_loads: Callable[[str], Any] = json.loads, -) -> Any: - path = Path(path) - b = path.read_bytes() - if content_type is None: - if path.suffix in ('.js', '.json'): - proto = Protocol.json - elif path.suffix == '.pkl': - proto = Protocol.pickle - - return load_str_bytes( - b, proto=proto, content_type=content_type, encoding=encoding, allow_pickle=allow_pickle, json_loads=json_loads - ) diff --git a/functions/python/data-processor/pydantic/v1/py.typed b/functions/python/data-processor/pydantic/v1/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/pydantic/v1/schema.py b/functions/python/data-processor/pydantic/v1/schema.py deleted file mode 100644 index a91fe2cd4..000000000 --- a/functions/python/data-processor/pydantic/v1/schema.py +++ /dev/null @@ -1,1163 +0,0 @@ -import re -import warnings -from collections import defaultdict -from dataclasses import is_dataclass -from datetime import date, datetime, time, timedelta -from decimal import Decimal -from enum import Enum -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - ForwardRef, - FrozenSet, - Generic, - Iterable, - List, - Optional, - Pattern, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, -) -from uuid import UUID - -from typing_extensions import Annotated, Literal - -from pydantic.v1.fields import ( - MAPPING_LIKE_SHAPES, - SHAPE_DEQUE, - SHAPE_FROZENSET, - SHAPE_GENERIC, - SHAPE_ITERABLE, - SHAPE_LIST, - SHAPE_SEQUENCE, - SHAPE_SET, - SHAPE_SINGLETON, - SHAPE_TUPLE, - SHAPE_TUPLE_ELLIPSIS, - FieldInfo, - ModelField, -) -from pydantic.v1.json import pydantic_encoder -from pydantic.v1.networks import AnyUrl, EmailStr -from pydantic.v1.types import ( - ConstrainedDecimal, - ConstrainedFloat, - ConstrainedFrozenSet, - ConstrainedInt, - ConstrainedList, - ConstrainedSet, - ConstrainedStr, - SecretBytes, - SecretStr, - StrictBytes, - StrictStr, - conbytes, - condecimal, - confloat, - confrozenset, - conint, - conlist, - conset, - constr, -) -from pydantic.v1.typing import ( - all_literal_values, - get_args, - get_origin, - get_sub_types, - is_callable_type, - is_literal_type, - is_namedtuple, - is_none_type, - is_union, -) -from pydantic.v1.utils import ROOT_KEY, get_model, lenient_issubclass - -if TYPE_CHECKING: - from pydantic.v1.dataclasses import Dataclass - from pydantic.v1.main import BaseModel - -default_prefix = '#/definitions/' -default_ref_template = '#/definitions/{model}' - -TypeModelOrEnum = Union[Type['BaseModel'], Type[Enum]] -TypeModelSet = Set[TypeModelOrEnum] - - -def _apply_modify_schema( - modify_schema: Callable[..., None], field: Optional[ModelField], field_schema: Dict[str, Any] -) -> None: - from inspect import signature - - sig = signature(modify_schema) - args = set(sig.parameters.keys()) - if 'field' in args or 'kwargs' in args: - modify_schema(field_schema, field=field) - else: - modify_schema(field_schema) - - -def schema( - models: Sequence[Union[Type['BaseModel'], Type['Dataclass']]], - *, - by_alias: bool = True, - title: Optional[str] = None, - description: Optional[str] = None, - ref_prefix: Optional[str] = None, - ref_template: str = default_ref_template, -) -> Dict[str, Any]: - """ - Process a list of models and generate a single JSON Schema with all of them defined in the ``definitions`` - top-level JSON key, including their sub-models. - - :param models: a list of models to include in the generated JSON Schema - :param by_alias: generate the schemas using the aliases defined, if any - :param title: title for the generated schema that includes the definitions - :param description: description for the generated schema - :param ref_prefix: the JSON Pointer prefix for schema references with ``$ref``, if None, will be set to the - default of ``#/definitions/``. Update it if you want the schemas to reference the definitions somewhere - else, e.g. for OpenAPI use ``#/components/schemas/``. The resulting generated schemas will still be at the - top-level key ``definitions``, so you can extract them from there. But all the references will have the set - prefix. - :param ref_template: Use a ``string.format()`` template for ``$ref`` instead of a prefix. This can be useful - for references that cannot be represented by ``ref_prefix`` such as a definition stored in another file. For - a sibling json file in a ``/schemas`` directory use ``"/schemas/${model}.json#"``. - :return: dict with the JSON Schema with a ``definitions`` top-level key including the schema definitions for - the models and sub-models passed in ``models``. - """ - clean_models = [get_model(model) for model in models] - flat_models = get_flat_models_from_models(clean_models) - model_name_map = get_model_name_map(flat_models) - definitions = {} - output_schema: Dict[str, Any] = {} - if title: - output_schema['title'] = title - if description: - output_schema['description'] = description - for model in clean_models: - m_schema, m_definitions, m_nested_models = model_process_schema( - model, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - ) - definitions.update(m_definitions) - model_name = model_name_map[model] - definitions[model_name] = m_schema - if definitions: - output_schema['definitions'] = definitions - return output_schema - - -def model_schema( - model: Union[Type['BaseModel'], Type['Dataclass']], - by_alias: bool = True, - ref_prefix: Optional[str] = None, - ref_template: str = default_ref_template, -) -> Dict[str, Any]: - """ - Generate a JSON Schema for one model. With all the sub-models defined in the ``definitions`` top-level - JSON key. - - :param model: a Pydantic model (a class that inherits from BaseModel) - :param by_alias: generate the schemas using the aliases defined, if any - :param ref_prefix: the JSON Pointer prefix for schema references with ``$ref``, if None, will be set to the - default of ``#/definitions/``. Update it if you want the schemas to reference the definitions somewhere - else, e.g. for OpenAPI use ``#/components/schemas/``. The resulting generated schemas will still be at the - top-level key ``definitions``, so you can extract them from there. But all the references will have the set - prefix. - :param ref_template: Use a ``string.format()`` template for ``$ref`` instead of a prefix. This can be useful for - references that cannot be represented by ``ref_prefix`` such as a definition stored in another file. For a - sibling json file in a ``/schemas`` directory use ``"/schemas/${model}.json#"``. - :return: dict with the JSON Schema for the passed ``model`` - """ - model = get_model(model) - flat_models = get_flat_models_from_model(model) - model_name_map = get_model_name_map(flat_models) - model_name = model_name_map[model] - m_schema, m_definitions, nested_models = model_process_schema( - model, by_alias=by_alias, model_name_map=model_name_map, ref_prefix=ref_prefix, ref_template=ref_template - ) - if model_name in nested_models: - # model_name is in Nested models, it has circular references - m_definitions[model_name] = m_schema - m_schema = get_schema_ref(model_name, ref_prefix, ref_template, False) - if m_definitions: - m_schema.update({'definitions': m_definitions}) - return m_schema - - -def get_field_info_schema(field: ModelField, schema_overrides: bool = False) -> Tuple[Dict[str, Any], bool]: - # If no title is explicitly set, we don't set title in the schema for enums. - # The behaviour is the same as `BaseModel` reference, where the default title - # is in the definitions part of the schema. - schema_: Dict[str, Any] = {} - if field.field_info.title or not lenient_issubclass(field.type_, Enum): - schema_['title'] = field.field_info.title or field.alias.title().replace('_', ' ') - - if field.field_info.title: - schema_overrides = True - - if field.field_info.description: - schema_['description'] = field.field_info.description - schema_overrides = True - - if not field.required and field.default is not None and not is_callable_type(field.outer_type_): - schema_['default'] = encode_default(field.default) - schema_overrides = True - - return schema_, schema_overrides - - -def field_schema( - field: ModelField, - *, - by_alias: bool = True, - model_name_map: Dict[TypeModelOrEnum, str], - ref_prefix: Optional[str] = None, - ref_template: str = default_ref_template, - known_models: Optional[TypeModelSet] = None, -) -> Tuple[Dict[str, Any], Dict[str, Any], Set[str]]: - """ - Process a Pydantic field and return a tuple with a JSON Schema for it as the first item. - Also return a dictionary of definitions with models as keys and their schemas as values. If the passed field - is a model and has sub-models, and those sub-models don't have overrides (as ``title``, ``default``, etc), they - will be included in the definitions and referenced in the schema instead of included recursively. - - :param field: a Pydantic ``ModelField`` - :param by_alias: use the defined alias (if any) in the returned schema - :param model_name_map: used to generate the JSON Schema references to other models included in the definitions - :param ref_prefix: the JSON Pointer prefix to use for references to other schemas, if None, the default of - #/definitions/ will be used - :param ref_template: Use a ``string.format()`` template for ``$ref`` instead of a prefix. This can be useful for - references that cannot be represented by ``ref_prefix`` such as a definition stored in another file. For a - sibling json file in a ``/schemas`` directory use ``"/schemas/${model}.json#"``. - :param known_models: used to solve circular references - :return: tuple of the schema for this field and additional definitions - """ - s, schema_overrides = get_field_info_schema(field) - - validation_schema = get_field_schema_validations(field) - if validation_schema: - s.update(validation_schema) - schema_overrides = True - - f_schema, f_definitions, f_nested_models = field_type_schema( - field, - by_alias=by_alias, - model_name_map=model_name_map, - schema_overrides=schema_overrides, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models or set(), - ) - - # $ref will only be returned when there are no schema_overrides - if '$ref' in f_schema: - return f_schema, f_definitions, f_nested_models - else: - s.update(f_schema) - return s, f_definitions, f_nested_models - - -numeric_types = (int, float, Decimal) -_str_types_attrs: Tuple[Tuple[str, Union[type, Tuple[type, ...]], str], ...] = ( - ('max_length', numeric_types, 'maxLength'), - ('min_length', numeric_types, 'minLength'), - ('regex', str, 'pattern'), -) - -_numeric_types_attrs: Tuple[Tuple[str, Union[type, Tuple[type, ...]], str], ...] = ( - ('gt', numeric_types, 'exclusiveMinimum'), - ('lt', numeric_types, 'exclusiveMaximum'), - ('ge', numeric_types, 'minimum'), - ('le', numeric_types, 'maximum'), - ('multiple_of', numeric_types, 'multipleOf'), -) - - -def get_field_schema_validations(field: ModelField) -> Dict[str, Any]: - """ - Get the JSON Schema validation keywords for a ``field`` with an annotation of - a Pydantic ``FieldInfo`` with validation arguments. - """ - f_schema: Dict[str, Any] = {} - - if lenient_issubclass(field.type_, Enum): - # schema is already updated by `enum_process_schema`; just update with field extra - if field.field_info.extra: - f_schema.update(field.field_info.extra) - return f_schema - - if lenient_issubclass(field.type_, (str, bytes)): - for attr_name, t, keyword in _str_types_attrs: - attr = getattr(field.field_info, attr_name, None) - if isinstance(attr, t): - f_schema[keyword] = attr - if lenient_issubclass(field.type_, numeric_types) and not issubclass(field.type_, bool): - for attr_name, t, keyword in _numeric_types_attrs: - attr = getattr(field.field_info, attr_name, None) - if isinstance(attr, t): - f_schema[keyword] = attr - if field.field_info is not None and field.field_info.const: - f_schema['const'] = field.default - if field.field_info.extra: - f_schema.update(field.field_info.extra) - modify_schema = getattr(field.outer_type_, '__modify_schema__', None) - if modify_schema: - _apply_modify_schema(modify_schema, field, f_schema) - return f_schema - - -def get_model_name_map(unique_models: TypeModelSet) -> Dict[TypeModelOrEnum, str]: - """ - Process a set of models and generate unique names for them to be used as keys in the JSON Schema - definitions. By default the names are the same as the class name. But if two models in different Python - modules have the same name (e.g. "users.Model" and "items.Model"), the generated names will be - based on the Python module path for those conflicting models to prevent name collisions. - - :param unique_models: a Python set of models - :return: dict mapping models to names - """ - name_model_map = {} - conflicting_names: Set[str] = set() - for model in unique_models: - model_name = normalize_name(model.__name__) - if model_name in conflicting_names: - model_name = get_long_model_name(model) - name_model_map[model_name] = model - elif model_name in name_model_map: - conflicting_names.add(model_name) - conflicting_model = name_model_map.pop(model_name) - name_model_map[get_long_model_name(conflicting_model)] = conflicting_model - name_model_map[get_long_model_name(model)] = model - else: - name_model_map[model_name] = model - return {v: k for k, v in name_model_map.items()} - - -def get_flat_models_from_model(model: Type['BaseModel'], known_models: Optional[TypeModelSet] = None) -> TypeModelSet: - """ - Take a single ``model`` and generate a set with itself and all the sub-models in the tree. I.e. if you pass - model ``Foo`` (subclass of Pydantic ``BaseModel``) as ``model``, and it has a field of type ``Bar`` (also - subclass of ``BaseModel``) and that model ``Bar`` has a field of type ``Baz`` (also subclass of ``BaseModel``), - the return value will be ``set([Foo, Bar, Baz])``. - - :param model: a Pydantic ``BaseModel`` subclass - :param known_models: used to solve circular references - :return: a set with the initial model and all its sub-models - """ - known_models = known_models or set() - flat_models: TypeModelSet = set() - flat_models.add(model) - known_models |= flat_models - fields = cast(Sequence[ModelField], model.__fields__.values()) - flat_models |= get_flat_models_from_fields(fields, known_models=known_models) - return flat_models - - -def get_flat_models_from_field(field: ModelField, known_models: TypeModelSet) -> TypeModelSet: - """ - Take a single Pydantic ``ModelField`` (from a model) that could have been declared as a subclass of BaseModel - (so, it could be a submodel), and generate a set with its model and all the sub-models in the tree. - I.e. if you pass a field that was declared to be of type ``Foo`` (subclass of BaseModel) as ``field``, and that - model ``Foo`` has a field of type ``Bar`` (also subclass of ``BaseModel``) and that model ``Bar`` has a field of - type ``Baz`` (also subclass of ``BaseModel``), the return value will be ``set([Foo, Bar, Baz])``. - - :param field: a Pydantic ``ModelField`` - :param known_models: used to solve circular references - :return: a set with the model used in the declaration for this field, if any, and all its sub-models - """ - from pydantic.v1.main import BaseModel - - flat_models: TypeModelSet = set() - - field_type = field.type_ - if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel): - field_type = field_type.__pydantic_model__ - - if field.sub_fields and not lenient_issubclass(field_type, BaseModel): - flat_models |= get_flat_models_from_fields(field.sub_fields, known_models=known_models) - elif lenient_issubclass(field_type, BaseModel) and field_type not in known_models: - flat_models |= get_flat_models_from_model(field_type, known_models=known_models) - elif lenient_issubclass(field_type, Enum): - flat_models.add(field_type) - return flat_models - - -def get_flat_models_from_fields(fields: Sequence[ModelField], known_models: TypeModelSet) -> TypeModelSet: - """ - Take a list of Pydantic ``ModelField``s (from a model) that could have been declared as subclasses of ``BaseModel`` - (so, any of them could be a submodel), and generate a set with their models and all the sub-models in the tree. - I.e. if you pass a the fields of a model ``Foo`` (subclass of ``BaseModel``) as ``fields``, and on of them has a - field of type ``Bar`` (also subclass of ``BaseModel``) and that model ``Bar`` has a field of type ``Baz`` (also - subclass of ``BaseModel``), the return value will be ``set([Foo, Bar, Baz])``. - - :param fields: a list of Pydantic ``ModelField``s - :param known_models: used to solve circular references - :return: a set with any model declared in the fields, and all their sub-models - """ - flat_models: TypeModelSet = set() - for field in fields: - flat_models |= get_flat_models_from_field(field, known_models=known_models) - return flat_models - - -def get_flat_models_from_models(models: Sequence[Type['BaseModel']]) -> TypeModelSet: - """ - Take a list of ``models`` and generate a set with them and all their sub-models in their trees. I.e. if you pass - a list of two models, ``Foo`` and ``Bar``, both subclasses of Pydantic ``BaseModel`` as models, and ``Bar`` has - a field of type ``Baz`` (also subclass of ``BaseModel``), the return value will be ``set([Foo, Bar, Baz])``. - """ - flat_models: TypeModelSet = set() - for model in models: - flat_models |= get_flat_models_from_model(model) - return flat_models - - -def get_long_model_name(model: TypeModelOrEnum) -> str: - return f'{model.__module__}__{model.__qualname__}'.replace('.', '__') - - -def field_type_schema( - field: ModelField, - *, - by_alias: bool, - model_name_map: Dict[TypeModelOrEnum, str], - ref_template: str, - schema_overrides: bool = False, - ref_prefix: Optional[str] = None, - known_models: TypeModelSet, -) -> Tuple[Dict[str, Any], Dict[str, Any], Set[str]]: - """ - Used by ``field_schema()``, you probably should be using that function. - - Take a single ``field`` and generate the schema for its type only, not including additional - information as title, etc. Also return additional schema definitions, from sub-models. - """ - from pydantic.v1.main import BaseModel # noqa: F811 - - definitions = {} - nested_models: Set[str] = set() - f_schema: Dict[str, Any] - if field.shape in { - SHAPE_LIST, - SHAPE_TUPLE_ELLIPSIS, - SHAPE_SEQUENCE, - SHAPE_SET, - SHAPE_FROZENSET, - SHAPE_ITERABLE, - SHAPE_DEQUE, - }: - items_schema, f_definitions, f_nested_models = field_singleton_schema( - field, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - definitions.update(f_definitions) - nested_models.update(f_nested_models) - f_schema = {'type': 'array', 'items': items_schema} - if field.shape in {SHAPE_SET, SHAPE_FROZENSET}: - f_schema['uniqueItems'] = True - - elif field.shape in MAPPING_LIKE_SHAPES: - f_schema = {'type': 'object'} - key_field = cast(ModelField, field.key_field) - regex = getattr(key_field.type_, 'regex', None) - items_schema, f_definitions, f_nested_models = field_singleton_schema( - field, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - definitions.update(f_definitions) - nested_models.update(f_nested_models) - if regex: - # Dict keys have a regex pattern - # items_schema might be a schema or empty dict, add it either way - f_schema['patternProperties'] = {ConstrainedStr._get_pattern(regex): items_schema} - if items_schema: - # The dict values are not simply Any, so they need a schema - f_schema['additionalProperties'] = items_schema - elif field.shape == SHAPE_TUPLE or (field.shape == SHAPE_GENERIC and not issubclass(field.type_, BaseModel)): - sub_schema = [] - sub_fields = cast(List[ModelField], field.sub_fields) - for sf in sub_fields: - sf_schema, sf_definitions, sf_nested_models = field_type_schema( - sf, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - definitions.update(sf_definitions) - nested_models.update(sf_nested_models) - sub_schema.append(sf_schema) - - sub_fields_len = len(sub_fields) - if field.shape == SHAPE_GENERIC: - all_of_schemas = sub_schema[0] if sub_fields_len == 1 else {'type': 'array', 'items': sub_schema} - f_schema = {'allOf': [all_of_schemas]} - else: - f_schema = { - 'type': 'array', - 'minItems': sub_fields_len, - 'maxItems': sub_fields_len, - } - if sub_fields_len >= 1: - f_schema['items'] = sub_schema - else: - assert field.shape in {SHAPE_SINGLETON, SHAPE_GENERIC}, field.shape - f_schema, f_definitions, f_nested_models = field_singleton_schema( - field, - by_alias=by_alias, - model_name_map=model_name_map, - schema_overrides=schema_overrides, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - definitions.update(f_definitions) - nested_models.update(f_nested_models) - - # check field type to avoid repeated calls to the same __modify_schema__ method - if field.type_ != field.outer_type_: - if field.shape == SHAPE_GENERIC: - field_type = field.type_ - else: - field_type = field.outer_type_ - modify_schema = getattr(field_type, '__modify_schema__', None) - if modify_schema: - _apply_modify_schema(modify_schema, field, f_schema) - return f_schema, definitions, nested_models - - -def model_process_schema( - model: TypeModelOrEnum, - *, - by_alias: bool = True, - model_name_map: Dict[TypeModelOrEnum, str], - ref_prefix: Optional[str] = None, - ref_template: str = default_ref_template, - known_models: Optional[TypeModelSet] = None, - field: Optional[ModelField] = None, -) -> Tuple[Dict[str, Any], Dict[str, Any], Set[str]]: - """ - Used by ``model_schema()``, you probably should be using that function. - - Take a single ``model`` and generate its schema. Also return additional schema definitions, from sub-models. The - sub-models of the returned schema will be referenced, but their definitions will not be included in the schema. All - the definitions are returned as the second value. - """ - from inspect import getdoc, signature - - known_models = known_models or set() - if lenient_issubclass(model, Enum): - model = cast(Type[Enum], model) - s = enum_process_schema(model, field=field) - return s, {}, set() - model = cast(Type['BaseModel'], model) - s = {'title': model.__config__.title or model.__name__} - doc = getdoc(model) - if doc: - s['description'] = doc - known_models.add(model) - m_schema, m_definitions, nested_models = model_type_schema( - model, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - s.update(m_schema) - schema_extra = model.__config__.schema_extra - if callable(schema_extra): - if len(signature(schema_extra).parameters) == 1: - schema_extra(s) - else: - schema_extra(s, model) - else: - s.update(schema_extra) - return s, m_definitions, nested_models - - -def model_type_schema( - model: Type['BaseModel'], - *, - by_alias: bool, - model_name_map: Dict[TypeModelOrEnum, str], - ref_template: str, - ref_prefix: Optional[str] = None, - known_models: TypeModelSet, -) -> Tuple[Dict[str, Any], Dict[str, Any], Set[str]]: - """ - You probably should be using ``model_schema()``, this function is indirectly used by that function. - - Take a single ``model`` and generate the schema for its type only, not including additional - information as title, etc. Also return additional schema definitions, from sub-models. - """ - properties = {} - required = [] - definitions: Dict[str, Any] = {} - nested_models: Set[str] = set() - for k, f in model.__fields__.items(): - try: - f_schema, f_definitions, f_nested_models = field_schema( - f, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - except SkipField as skip: - warnings.warn(skip.message, UserWarning) - continue - definitions.update(f_definitions) - nested_models.update(f_nested_models) - if by_alias: - properties[f.alias] = f_schema - if f.required: - required.append(f.alias) - else: - properties[k] = f_schema - if f.required: - required.append(k) - if ROOT_KEY in properties: - out_schema = properties[ROOT_KEY] - out_schema['title'] = model.__config__.title or model.__name__ - else: - out_schema = {'type': 'object', 'properties': properties} - if required: - out_schema['required'] = required - if model.__config__.extra == 'forbid': - out_schema['additionalProperties'] = False - return out_schema, definitions, nested_models - - -def enum_process_schema(enum: Type[Enum], *, field: Optional[ModelField] = None) -> Dict[str, Any]: - """ - Take a single `enum` and generate its schema. - - This is similar to the `model_process_schema` function, but applies to ``Enum`` objects. - """ - import inspect - - schema_: Dict[str, Any] = { - 'title': enum.__name__, - # Python assigns all enums a default docstring value of 'An enumeration', so - # all enums will have a description field even if not explicitly provided. - 'description': inspect.cleandoc(enum.__doc__ or 'An enumeration.'), - # Add enum values and the enum field type to the schema. - 'enum': [item.value for item in cast(Iterable[Enum], enum)], - } - - add_field_type_to_schema(enum, schema_) - - modify_schema = getattr(enum, '__modify_schema__', None) - if modify_schema: - _apply_modify_schema(modify_schema, field, schema_) - - return schema_ - - -def field_singleton_sub_fields_schema( - field: ModelField, - *, - by_alias: bool, - model_name_map: Dict[TypeModelOrEnum, str], - ref_template: str, - schema_overrides: bool = False, - ref_prefix: Optional[str] = None, - known_models: TypeModelSet, -) -> Tuple[Dict[str, Any], Dict[str, Any], Set[str]]: - """ - This function is indirectly used by ``field_schema()``, you probably should be using that function. - - Take a list of Pydantic ``ModelField`` from the declaration of a type with parameters, and generate their - schema. I.e., fields used as "type parameters", like ``str`` and ``int`` in ``Tuple[str, int]``. - """ - sub_fields = cast(List[ModelField], field.sub_fields) - definitions = {} - nested_models: Set[str] = set() - if len(sub_fields) == 1: - return field_type_schema( - sub_fields[0], - by_alias=by_alias, - model_name_map=model_name_map, - schema_overrides=schema_overrides, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - else: - s: Dict[str, Any] = {} - # https://github.com/OAI/OpenAPI-Specification/blob/master/versions/3.0.2.md#discriminator-object - field_has_discriminator: bool = field.discriminator_key is not None - if field_has_discriminator: - assert field.sub_fields_mapping is not None - - discriminator_models_refs: Dict[str, Union[str, Dict[str, Any]]] = {} - - for discriminator_value, sub_field in field.sub_fields_mapping.items(): - if isinstance(discriminator_value, Enum): - discriminator_value = str(discriminator_value.value) - # sub_field is either a `BaseModel` or directly an `Annotated` `Union` of many - if is_union(get_origin(sub_field.type_)): - sub_models = get_sub_types(sub_field.type_) - discriminator_models_refs[discriminator_value] = { - model_name_map[sub_model]: get_schema_ref( - model_name_map[sub_model], ref_prefix, ref_template, False - ) - for sub_model in sub_models - } - else: - sub_field_type = sub_field.type_ - if hasattr(sub_field_type, '__pydantic_model__'): - sub_field_type = sub_field_type.__pydantic_model__ - - discriminator_model_name = model_name_map[sub_field_type] - discriminator_model_ref = get_schema_ref(discriminator_model_name, ref_prefix, ref_template, False) - discriminator_models_refs[discriminator_value] = discriminator_model_ref['$ref'] - - s['discriminator'] = { - 'propertyName': field.discriminator_alias if by_alias else field.discriminator_key, - 'mapping': discriminator_models_refs, - } - - sub_field_schemas = [] - for sf in sub_fields: - sub_schema, sub_definitions, sub_nested_models = field_type_schema( - sf, - by_alias=by_alias, - model_name_map=model_name_map, - schema_overrides=schema_overrides, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - definitions.update(sub_definitions) - if schema_overrides and 'allOf' in sub_schema: - # if the sub_field is a referenced schema we only need the referenced - # object. Otherwise we will end up with several allOf inside anyOf/oneOf. - # See https://github.com/pydantic/pydantic/issues/1209 - sub_schema = sub_schema['allOf'][0] - - if sub_schema.keys() == {'discriminator', 'oneOf'}: - # we don't want discriminator information inside oneOf choices, this is dealt with elsewhere - sub_schema.pop('discriminator') - sub_field_schemas.append(sub_schema) - nested_models.update(sub_nested_models) - s['oneOf' if field_has_discriminator else 'anyOf'] = sub_field_schemas - return s, definitions, nested_models - - -# Order is important, e.g. subclasses of str must go before str -# this is used only for standard library types, custom types should use __modify_schema__ instead -field_class_to_schema: Tuple[Tuple[Any, Dict[str, Any]], ...] = ( - (Path, {'type': 'string', 'format': 'path'}), - (datetime, {'type': 'string', 'format': 'date-time'}), - (date, {'type': 'string', 'format': 'date'}), - (time, {'type': 'string', 'format': 'time'}), - (timedelta, {'type': 'number', 'format': 'time-delta'}), - (IPv4Network, {'type': 'string', 'format': 'ipv4network'}), - (IPv6Network, {'type': 'string', 'format': 'ipv6network'}), - (IPv4Interface, {'type': 'string', 'format': 'ipv4interface'}), - (IPv6Interface, {'type': 'string', 'format': 'ipv6interface'}), - (IPv4Address, {'type': 'string', 'format': 'ipv4'}), - (IPv6Address, {'type': 'string', 'format': 'ipv6'}), - (Pattern, {'type': 'string', 'format': 'regex'}), - (str, {'type': 'string'}), - (bytes, {'type': 'string', 'format': 'binary'}), - (bool, {'type': 'boolean'}), - (int, {'type': 'integer'}), - (float, {'type': 'number'}), - (Decimal, {'type': 'number'}), - (UUID, {'type': 'string', 'format': 'uuid'}), - (dict, {'type': 'object'}), - (list, {'type': 'array', 'items': {}}), - (tuple, {'type': 'array', 'items': {}}), - (set, {'type': 'array', 'items': {}, 'uniqueItems': True}), - (frozenset, {'type': 'array', 'items': {}, 'uniqueItems': True}), -) - -json_scheme = {'type': 'string', 'format': 'json-string'} - - -def add_field_type_to_schema(field_type: Any, schema_: Dict[str, Any]) -> None: - """ - Update the given `schema` with the type-specific metadata for the given `field_type`. - - This function looks through `field_class_to_schema` for a class that matches the given `field_type`, - and then modifies the given `schema` with the information from that type. - """ - for type_, t_schema in field_class_to_schema: - # Fallback for `typing.Pattern` and `re.Pattern` as they are not a valid class - if lenient_issubclass(field_type, type_) or field_type is type_ is Pattern: - schema_.update(t_schema) - break - - -def get_schema_ref(name: str, ref_prefix: Optional[str], ref_template: str, schema_overrides: bool) -> Dict[str, Any]: - if ref_prefix: - schema_ref = {'$ref': ref_prefix + name} - else: - schema_ref = {'$ref': ref_template.format(model=name)} - return {'allOf': [schema_ref]} if schema_overrides else schema_ref - - -def field_singleton_schema( # noqa: C901 (ignore complexity) - field: ModelField, - *, - by_alias: bool, - model_name_map: Dict[TypeModelOrEnum, str], - ref_template: str, - schema_overrides: bool = False, - ref_prefix: Optional[str] = None, - known_models: TypeModelSet, -) -> Tuple[Dict[str, Any], Dict[str, Any], Set[str]]: - """ - This function is indirectly used by ``field_schema()``, you should probably be using that function. - - Take a single Pydantic ``ModelField``, and return its schema and any additional definitions from sub-models. - """ - from pydantic.v1.main import BaseModel - - definitions: Dict[str, Any] = {} - nested_models: Set[str] = set() - field_type = field.type_ - - # Recurse into this field if it contains sub_fields and is NOT a - # BaseModel OR that BaseModel is a const - if field.sub_fields and ( - (field.field_info and field.field_info.const) or not lenient_issubclass(field_type, BaseModel) - ): - return field_singleton_sub_fields_schema( - field, - by_alias=by_alias, - model_name_map=model_name_map, - schema_overrides=schema_overrides, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - if field_type is Any or field_type is object or field_type.__class__ == TypeVar or get_origin(field_type) is type: - return {}, definitions, nested_models # no restrictions - if is_none_type(field_type): - return {'type': 'null'}, definitions, nested_models - if is_callable_type(field_type): - raise SkipField(f'Callable {field.name} was excluded from schema since JSON schema has no equivalent type.') - f_schema: Dict[str, Any] = {} - if field.field_info is not None and field.field_info.const: - f_schema['const'] = field.default - - if is_literal_type(field_type): - values = tuple(x.value if isinstance(x, Enum) else x for x in all_literal_values(field_type)) - - if len({v.__class__ for v in values}) > 1: - return field_schema( - multitypes_literal_field_for_schema(values, field), - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - ) - - # All values have the same type - field_type = values[0].__class__ - f_schema['enum'] = list(values) - add_field_type_to_schema(field_type, f_schema) - elif lenient_issubclass(field_type, Enum): - enum_name = model_name_map[field_type] - f_schema, schema_overrides = get_field_info_schema(field, schema_overrides) - f_schema.update(get_schema_ref(enum_name, ref_prefix, ref_template, schema_overrides)) - definitions[enum_name] = enum_process_schema(field_type, field=field) - elif is_namedtuple(field_type): - sub_schema, *_ = model_process_schema( - field_type.__pydantic_model__, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - field=field, - ) - items_schemas = list(sub_schema['properties'].values()) - f_schema.update( - { - 'type': 'array', - 'items': items_schemas, - 'minItems': len(items_schemas), - 'maxItems': len(items_schemas), - } - ) - elif not hasattr(field_type, '__pydantic_model__'): - add_field_type_to_schema(field_type, f_schema) - - modify_schema = getattr(field_type, '__modify_schema__', None) - if modify_schema: - _apply_modify_schema(modify_schema, field, f_schema) - - if f_schema: - return f_schema, definitions, nested_models - - # Handle dataclass-based models - if lenient_issubclass(getattr(field_type, '__pydantic_model__', None), BaseModel): - field_type = field_type.__pydantic_model__ - - if issubclass(field_type, BaseModel): - model_name = model_name_map[field_type] - if field_type not in known_models: - sub_schema, sub_definitions, sub_nested_models = model_process_schema( - field_type, - by_alias=by_alias, - model_name_map=model_name_map, - ref_prefix=ref_prefix, - ref_template=ref_template, - known_models=known_models, - field=field, - ) - definitions.update(sub_definitions) - definitions[model_name] = sub_schema - nested_models.update(sub_nested_models) - else: - nested_models.add(model_name) - schema_ref = get_schema_ref(model_name, ref_prefix, ref_template, schema_overrides) - return schema_ref, definitions, nested_models - - # For generics with no args - args = get_args(field_type) - if args is not None and not args and Generic in field_type.__bases__: - return f_schema, definitions, nested_models - - raise ValueError(f'Value not declarable with JSON Schema, field: {field}') - - -def multitypes_literal_field_for_schema(values: Tuple[Any, ...], field: ModelField) -> ModelField: - """ - To support `Literal` with values of different types, we split it into multiple `Literal` with same type - e.g. `Literal['qwe', 'asd', 1, 2]` becomes `Union[Literal['qwe', 'asd'], Literal[1, 2]]` - """ - literal_distinct_types = defaultdict(list) - for v in values: - literal_distinct_types[v.__class__].append(v) - distinct_literals = (Literal[tuple(same_type_values)] for same_type_values in literal_distinct_types.values()) - - return ModelField( - name=field.name, - type_=Union[tuple(distinct_literals)], # type: ignore - class_validators=field.class_validators, - model_config=field.model_config, - default=field.default, - required=field.required, - alias=field.alias, - field_info=field.field_info, - ) - - -def encode_default(dft: Any) -> Any: - from pydantic.v1.main import BaseModel - - if isinstance(dft, BaseModel) or is_dataclass(dft): - dft = cast('dict[str, Any]', pydantic_encoder(dft)) - - if isinstance(dft, dict): - return {encode_default(k): encode_default(v) for k, v in dft.items()} - elif isinstance(dft, Enum): - return dft.value - elif isinstance(dft, (int, float, str)): - return dft - elif isinstance(dft, (list, tuple)): - t = dft.__class__ - seq_args = (encode_default(v) for v in dft) - return t(*seq_args) if is_namedtuple(t) else t(seq_args) - elif dft is None: - return None - else: - return pydantic_encoder(dft) - - -_map_types_constraint: Dict[Any, Callable[..., type]] = {int: conint, float: confloat, Decimal: condecimal} - - -def get_annotation_from_field_info( - annotation: Any, field_info: FieldInfo, field_name: str, validate_assignment: bool = False -) -> Type[Any]: - """ - Get an annotation with validation implemented for numbers and strings based on the field_info. - :param annotation: an annotation from a field specification, as ``str``, ``ConstrainedStr`` - :param field_info: an instance of FieldInfo, possibly with declarations for validations and JSON Schema - :param field_name: name of the field for use in error messages - :param validate_assignment: default False, flag for BaseModel Config value of validate_assignment - :return: the same ``annotation`` if unmodified or a new annotation with validation in place - """ - constraints = field_info.get_constraints() - used_constraints: Set[str] = set() - if constraints: - annotation, used_constraints = get_annotation_with_constraints(annotation, field_info) - if validate_assignment: - used_constraints.add('allow_mutation') - - unused_constraints = constraints - used_constraints - if unused_constraints: - raise ValueError( - f'On field "{field_name}" the following field constraints are set but not enforced: ' - f'{", ".join(unused_constraints)}. ' - f'\nFor more details see https://docs.pydantic.dev/usage/schema/#unenforced-field-constraints' - ) - - return annotation - - -def get_annotation_with_constraints(annotation: Any, field_info: FieldInfo) -> Tuple[Type[Any], Set[str]]: # noqa: C901 - """ - Get an annotation with used constraints implemented for numbers and strings based on the field_info. - - :param annotation: an annotation from a field specification, as ``str``, ``ConstrainedStr`` - :param field_info: an instance of FieldInfo, possibly with declarations for validations and JSON Schema - :return: the same ``annotation`` if unmodified or a new annotation along with the used constraints. - """ - used_constraints: Set[str] = set() - - def go(type_: Any) -> Type[Any]: - if ( - is_literal_type(type_) - or isinstance(type_, ForwardRef) - or lenient_issubclass(type_, (ConstrainedList, ConstrainedSet, ConstrainedFrozenSet)) - ): - return type_ - origin = get_origin(type_) - if origin is not None: - args: Tuple[Any, ...] = get_args(type_) - if any(isinstance(a, ForwardRef) for a in args): - # forward refs cause infinite recursion below - return type_ - - if origin is Annotated: - return go(args[0]) - if is_union(origin): - return Union[tuple(go(a) for a in args)] # type: ignore - - if issubclass(origin, List) and ( - field_info.min_items is not None - or field_info.max_items is not None - or field_info.unique_items is not None - ): - used_constraints.update({'min_items', 'max_items', 'unique_items'}) - return conlist( - go(args[0]), - min_items=field_info.min_items, - max_items=field_info.max_items, - unique_items=field_info.unique_items, - ) - - if issubclass(origin, Set) and (field_info.min_items is not None or field_info.max_items is not None): - used_constraints.update({'min_items', 'max_items'}) - return conset(go(args[0]), min_items=field_info.min_items, max_items=field_info.max_items) - - if issubclass(origin, FrozenSet) and (field_info.min_items is not None or field_info.max_items is not None): - used_constraints.update({'min_items', 'max_items'}) - return confrozenset(go(args[0]), min_items=field_info.min_items, max_items=field_info.max_items) - - for t in (Tuple, List, Set, FrozenSet, Sequence): - if issubclass(origin, t): # type: ignore - return t[tuple(go(a) for a in args)] # type: ignore - - if issubclass(origin, Dict): - return Dict[args[0], go(args[1])] # type: ignore - - attrs: Optional[Tuple[str, ...]] = None - constraint_func: Optional[Callable[..., type]] = None - if isinstance(type_, type): - if issubclass(type_, (SecretStr, SecretBytes)): - attrs = ('max_length', 'min_length') - - def constraint_func(**kw: Any) -> Type[Any]: # noqa: F811 - return type(type_.__name__, (type_,), kw) - - elif issubclass(type_, str) and not issubclass(type_, (EmailStr, AnyUrl)): - attrs = ('max_length', 'min_length', 'regex') - if issubclass(type_, StrictStr): - - def constraint_func(**kw: Any) -> Type[Any]: - return type(type_.__name__, (type_,), kw) - - else: - constraint_func = constr - elif issubclass(type_, bytes): - attrs = ('max_length', 'min_length', 'regex') - if issubclass(type_, StrictBytes): - - def constraint_func(**kw: Any) -> Type[Any]: - return type(type_.__name__, (type_,), kw) - - else: - constraint_func = conbytes - elif issubclass(type_, numeric_types) and not issubclass( - type_, - ( - ConstrainedInt, - ConstrainedFloat, - ConstrainedDecimal, - ConstrainedList, - ConstrainedSet, - ConstrainedFrozenSet, - bool, - ), - ): - # Is numeric type - attrs = ('gt', 'lt', 'ge', 'le', 'multiple_of') - if issubclass(type_, float): - attrs += ('allow_inf_nan',) - if issubclass(type_, Decimal): - attrs += ('max_digits', 'decimal_places') - numeric_type = next(t for t in numeric_types if issubclass(type_, t)) # pragma: no branch - constraint_func = _map_types_constraint[numeric_type] - - if attrs: - used_constraints.update(set(attrs)) - kwargs = { - attr_name: attr - for attr_name, attr in ((attr_name, getattr(field_info, attr_name)) for attr_name in attrs) - if attr is not None - } - if kwargs: - constraint_func = cast(Callable[..., type], constraint_func) - return constraint_func(**kwargs) - return type_ - - return go(annotation), used_constraints - - -def normalize_name(name: str) -> str: - """ - Normalizes the given name. This can be applied to either a model *or* enum. - """ - return re.sub(r'[^a-zA-Z0-9.\-_]', '_', name) - - -class SkipField(Exception): - """ - Utility exception used to exclude fields from schema. - """ - - def __init__(self, message: str) -> None: - self.message = message diff --git a/functions/python/data-processor/pydantic/v1/tools.py b/functions/python/data-processor/pydantic/v1/tools.py deleted file mode 100644 index 6838a23ec..000000000 --- a/functions/python/data-processor/pydantic/v1/tools.py +++ /dev/null @@ -1,92 +0,0 @@ -import json -from functools import lru_cache -from pathlib import Path -from typing import TYPE_CHECKING, Any, Callable, Optional, Type, TypeVar, Union - -from pydantic.v1.parse import Protocol, load_file, load_str_bytes -from pydantic.v1.types import StrBytes -from pydantic.v1.typing import display_as_type - -__all__ = ('parse_file_as', 'parse_obj_as', 'parse_raw_as', 'schema_of', 'schema_json_of') - -NameFactory = Union[str, Callable[[Type[Any]], str]] - -if TYPE_CHECKING: - from pydantic.v1.typing import DictStrAny - - -def _generate_parsing_type_name(type_: Any) -> str: - return f'ParsingModel[{display_as_type(type_)}]' - - -@lru_cache(maxsize=2048) -def _get_parsing_type(type_: Any, *, type_name: Optional[NameFactory] = None) -> Any: - from pydantic.v1.main import create_model - - if type_name is None: - type_name = _generate_parsing_type_name - if not isinstance(type_name, str): - type_name = type_name(type_) - return create_model(type_name, __root__=(type_, ...)) - - -T = TypeVar('T') - - -def parse_obj_as(type_: Type[T], obj: Any, *, type_name: Optional[NameFactory] = None) -> T: - model_type = _get_parsing_type(type_, type_name=type_name) # type: ignore[arg-type] - return model_type(__root__=obj).__root__ - - -def parse_file_as( - type_: Type[T], - path: Union[str, Path], - *, - content_type: str = None, - encoding: str = 'utf8', - proto: Protocol = None, - allow_pickle: bool = False, - json_loads: Callable[[str], Any] = json.loads, - type_name: Optional[NameFactory] = None, -) -> T: - obj = load_file( - path, - proto=proto, - content_type=content_type, - encoding=encoding, - allow_pickle=allow_pickle, - json_loads=json_loads, - ) - return parse_obj_as(type_, obj, type_name=type_name) - - -def parse_raw_as( - type_: Type[T], - b: StrBytes, - *, - content_type: str = None, - encoding: str = 'utf8', - proto: Protocol = None, - allow_pickle: bool = False, - json_loads: Callable[[str], Any] = json.loads, - type_name: Optional[NameFactory] = None, -) -> T: - obj = load_str_bytes( - b, - proto=proto, - content_type=content_type, - encoding=encoding, - allow_pickle=allow_pickle, - json_loads=json_loads, - ) - return parse_obj_as(type_, obj, type_name=type_name) - - -def schema_of(type_: Any, *, title: Optional[NameFactory] = None, **schema_kwargs: Any) -> 'DictStrAny': - """Generate a JSON schema (as dict) for the passed model or dynamically generated one""" - return _get_parsing_type(type_, type_name=title).schema(**schema_kwargs) - - -def schema_json_of(type_: Any, *, title: Optional[NameFactory] = None, **schema_json_kwargs: Any) -> str: - """Generate a JSON schema (as JSON) for the passed model or dynamically generated one""" - return _get_parsing_type(type_, type_name=title).schema_json(**schema_json_kwargs) diff --git a/functions/python/data-processor/pydantic/v1/types.py b/functions/python/data-processor/pydantic/v1/types.py deleted file mode 100644 index e1840d99f..000000000 --- a/functions/python/data-processor/pydantic/v1/types.py +++ /dev/null @@ -1,1205 +0,0 @@ -import abc -import math -import re -import warnings -from datetime import date -from decimal import Decimal, InvalidOperation -from enum import Enum -from pathlib import Path -from types import new_class -from typing import ( - TYPE_CHECKING, - Any, - Callable, - ClassVar, - Dict, - FrozenSet, - List, - Optional, - Pattern, - Set, - Tuple, - Type, - TypeVar, - Union, - cast, - overload, -) -from uuid import UUID -from weakref import WeakSet - -from pydantic.v1 import errors -from pydantic.v1.datetime_parse import parse_date -from pydantic.v1.utils import import_string, update_not_none -from pydantic.v1.validators import ( - bytes_validator, - constr_length_validator, - constr_lower, - constr_strip_whitespace, - constr_upper, - decimal_validator, - float_finite_validator, - float_validator, - frozenset_validator, - int_validator, - list_validator, - number_multiple_validator, - number_size_validator, - path_exists_validator, - path_validator, - set_validator, - str_validator, - strict_bytes_validator, - strict_float_validator, - strict_int_validator, - strict_str_validator, -) - -__all__ = [ - 'NoneStr', - 'NoneBytes', - 'StrBytes', - 'NoneStrBytes', - 'StrictStr', - 'ConstrainedBytes', - 'conbytes', - 'ConstrainedList', - 'conlist', - 'ConstrainedSet', - 'conset', - 'ConstrainedFrozenSet', - 'confrozenset', - 'ConstrainedStr', - 'constr', - 'PyObject', - 'ConstrainedInt', - 'conint', - 'PositiveInt', - 'NegativeInt', - 'NonNegativeInt', - 'NonPositiveInt', - 'ConstrainedFloat', - 'confloat', - 'PositiveFloat', - 'NegativeFloat', - 'NonNegativeFloat', - 'NonPositiveFloat', - 'FiniteFloat', - 'ConstrainedDecimal', - 'condecimal', - 'UUID1', - 'UUID3', - 'UUID4', - 'UUID5', - 'FilePath', - 'DirectoryPath', - 'Json', - 'JsonWrapper', - 'SecretField', - 'SecretStr', - 'SecretBytes', - 'StrictBool', - 'StrictBytes', - 'StrictInt', - 'StrictFloat', - 'PaymentCardNumber', - 'ByteSize', - 'PastDate', - 'FutureDate', - 'ConstrainedDate', - 'condate', -] - -NoneStr = Optional[str] -NoneBytes = Optional[bytes] -StrBytes = Union[str, bytes] -NoneStrBytes = Optional[StrBytes] -OptionalInt = Optional[int] -OptionalIntFloat = Union[OptionalInt, float] -OptionalIntFloatDecimal = Union[OptionalIntFloat, Decimal] -OptionalDate = Optional[date] -StrIntFloat = Union[str, int, float] - -if TYPE_CHECKING: - from typing_extensions import Annotated - - from pydantic.v1.dataclasses import Dataclass - from pydantic.v1.main import BaseModel - from pydantic.v1.typing import CallableGenerator - - ModelOrDc = Type[Union[BaseModel, Dataclass]] - -T = TypeVar('T') -_DEFINED_TYPES: 'WeakSet[type]' = WeakSet() - - -@overload -def _registered(typ: Type[T]) -> Type[T]: - pass - - -@overload -def _registered(typ: 'ConstrainedNumberMeta') -> 'ConstrainedNumberMeta': - pass - - -def _registered(typ: Union[Type[T], 'ConstrainedNumberMeta']) -> Union[Type[T], 'ConstrainedNumberMeta']: - # In order to generate valid examples of constrained types, Hypothesis needs - # to inspect the type object - so we keep a weakref to each contype object - # until it can be registered. When (or if) our Hypothesis plugin is loaded, - # it monkeypatches this function. - # If Hypothesis is never used, the total effect is to keep a weak reference - # which has minimal memory usage and doesn't even affect garbage collection. - _DEFINED_TYPES.add(typ) - return typ - - -class ConstrainedNumberMeta(type): - def __new__(cls, name: str, bases: Any, dct: Dict[str, Any]) -> 'ConstrainedInt': # type: ignore - new_cls = cast('ConstrainedInt', type.__new__(cls, name, bases, dct)) - - if new_cls.gt is not None and new_cls.ge is not None: - raise errors.ConfigError('bounds gt and ge cannot be specified at the same time') - if new_cls.lt is not None and new_cls.le is not None: - raise errors.ConfigError('bounds lt and le cannot be specified at the same time') - - return _registered(new_cls) # type: ignore - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BOOLEAN TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -if TYPE_CHECKING: - StrictBool = bool -else: - - class StrictBool(int): - """ - StrictBool to allow for bools which are not type-coerced. - """ - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='boolean') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, value: Any) -> bool: - """ - Ensure that we only allow bools. - """ - if isinstance(value, bool): - return value - - raise errors.StrictBoolError() - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ INTEGER TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class ConstrainedInt(int, metaclass=ConstrainedNumberMeta): - strict: bool = False - gt: OptionalInt = None - ge: OptionalInt = None - lt: OptionalInt = None - le: OptionalInt = None - multiple_of: OptionalInt = None - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none( - field_schema, - exclusiveMinimum=cls.gt, - exclusiveMaximum=cls.lt, - minimum=cls.ge, - maximum=cls.le, - multipleOf=cls.multiple_of, - ) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield strict_int_validator if cls.strict else int_validator - yield number_size_validator - yield number_multiple_validator - - -def conint( - *, - strict: bool = False, - gt: Optional[int] = None, - ge: Optional[int] = None, - lt: Optional[int] = None, - le: Optional[int] = None, - multiple_of: Optional[int] = None, -) -> Type[int]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict(strict=strict, gt=gt, ge=ge, lt=lt, le=le, multiple_of=multiple_of) - return type('ConstrainedIntValue', (ConstrainedInt,), namespace) - - -if TYPE_CHECKING: - PositiveInt = int - NegativeInt = int - NonPositiveInt = int - NonNegativeInt = int - StrictInt = int -else: - - class PositiveInt(ConstrainedInt): - gt = 0 - - class NegativeInt(ConstrainedInt): - lt = 0 - - class NonPositiveInt(ConstrainedInt): - le = 0 - - class NonNegativeInt(ConstrainedInt): - ge = 0 - - class StrictInt(ConstrainedInt): - strict = True - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ FLOAT TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class ConstrainedFloat(float, metaclass=ConstrainedNumberMeta): - strict: bool = False - gt: OptionalIntFloat = None - ge: OptionalIntFloat = None - lt: OptionalIntFloat = None - le: OptionalIntFloat = None - multiple_of: OptionalIntFloat = None - allow_inf_nan: Optional[bool] = None - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none( - field_schema, - exclusiveMinimum=cls.gt, - exclusiveMaximum=cls.lt, - minimum=cls.ge, - maximum=cls.le, - multipleOf=cls.multiple_of, - ) - # Modify constraints to account for differences between IEEE floats and JSON - if field_schema.get('exclusiveMinimum') == -math.inf: - del field_schema['exclusiveMinimum'] - if field_schema.get('minimum') == -math.inf: - del field_schema['minimum'] - if field_schema.get('exclusiveMaximum') == math.inf: - del field_schema['exclusiveMaximum'] - if field_schema.get('maximum') == math.inf: - del field_schema['maximum'] - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield strict_float_validator if cls.strict else float_validator - yield number_size_validator - yield number_multiple_validator - yield float_finite_validator - - -def confloat( - *, - strict: bool = False, - gt: float = None, - ge: float = None, - lt: float = None, - le: float = None, - multiple_of: float = None, - allow_inf_nan: Optional[bool] = None, -) -> Type[float]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict(strict=strict, gt=gt, ge=ge, lt=lt, le=le, multiple_of=multiple_of, allow_inf_nan=allow_inf_nan) - return type('ConstrainedFloatValue', (ConstrainedFloat,), namespace) - - -if TYPE_CHECKING: - PositiveFloat = float - NegativeFloat = float - NonPositiveFloat = float - NonNegativeFloat = float - StrictFloat = float - FiniteFloat = float -else: - - class PositiveFloat(ConstrainedFloat): - gt = 0 - - class NegativeFloat(ConstrainedFloat): - lt = 0 - - class NonPositiveFloat(ConstrainedFloat): - le = 0 - - class NonNegativeFloat(ConstrainedFloat): - ge = 0 - - class StrictFloat(ConstrainedFloat): - strict = True - - class FiniteFloat(ConstrainedFloat): - allow_inf_nan = False - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BYTES TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class ConstrainedBytes(bytes): - strip_whitespace = False - to_upper = False - to_lower = False - min_length: OptionalInt = None - max_length: OptionalInt = None - strict: bool = False - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none(field_schema, minLength=cls.min_length, maxLength=cls.max_length) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield strict_bytes_validator if cls.strict else bytes_validator - yield constr_strip_whitespace - yield constr_upper - yield constr_lower - yield constr_length_validator - - -def conbytes( - *, - strip_whitespace: bool = False, - to_upper: bool = False, - to_lower: bool = False, - min_length: Optional[int] = None, - max_length: Optional[int] = None, - strict: bool = False, -) -> Type[bytes]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict( - strip_whitespace=strip_whitespace, - to_upper=to_upper, - to_lower=to_lower, - min_length=min_length, - max_length=max_length, - strict=strict, - ) - return _registered(type('ConstrainedBytesValue', (ConstrainedBytes,), namespace)) - - -if TYPE_CHECKING: - StrictBytes = bytes -else: - - class StrictBytes(ConstrainedBytes): - strict = True - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ STRING TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class ConstrainedStr(str): - strip_whitespace = False - to_upper = False - to_lower = False - min_length: OptionalInt = None - max_length: OptionalInt = None - curtail_length: OptionalInt = None - regex: Optional[Union[str, Pattern[str]]] = None - strict = False - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none( - field_schema, - minLength=cls.min_length, - maxLength=cls.max_length, - pattern=cls.regex and cls._get_pattern(cls.regex), - ) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield strict_str_validator if cls.strict else str_validator - yield constr_strip_whitespace - yield constr_upper - yield constr_lower - yield constr_length_validator - yield cls.validate - - @classmethod - def validate(cls, value: Union[str]) -> Union[str]: - if cls.curtail_length and len(value) > cls.curtail_length: - value = value[: cls.curtail_length] - - if cls.regex: - if not re.match(cls.regex, value): - raise errors.StrRegexError(pattern=cls._get_pattern(cls.regex)) - - return value - - @staticmethod - def _get_pattern(regex: Union[str, Pattern[str]]) -> str: - return regex if isinstance(regex, str) else regex.pattern - - -def constr( - *, - strip_whitespace: bool = False, - to_upper: bool = False, - to_lower: bool = False, - strict: bool = False, - min_length: Optional[int] = None, - max_length: Optional[int] = None, - curtail_length: Optional[int] = None, - regex: Optional[str] = None, -) -> Type[str]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict( - strip_whitespace=strip_whitespace, - to_upper=to_upper, - to_lower=to_lower, - strict=strict, - min_length=min_length, - max_length=max_length, - curtail_length=curtail_length, - regex=regex and re.compile(regex), - ) - return _registered(type('ConstrainedStrValue', (ConstrainedStr,), namespace)) - - -if TYPE_CHECKING: - StrictStr = str -else: - - class StrictStr(ConstrainedStr): - strict = True - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SET TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -# This types superclass should be Set[T], but cython chokes on that... -class ConstrainedSet(set): # type: ignore - # Needed for pydantic to detect that this is a set - __origin__ = set - __args__: Set[Type[T]] # type: ignore - - min_items: Optional[int] = None - max_items: Optional[int] = None - item_type: Type[T] # type: ignore - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.set_length_validator - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none(field_schema, minItems=cls.min_items, maxItems=cls.max_items) - - @classmethod - def set_length_validator(cls, v: 'Optional[Set[T]]') -> 'Optional[Set[T]]': - if v is None: - return None - - v = set_validator(v) - v_len = len(v) - - if cls.min_items is not None and v_len < cls.min_items: - raise errors.SetMinLengthError(limit_value=cls.min_items) - - if cls.max_items is not None and v_len > cls.max_items: - raise errors.SetMaxLengthError(limit_value=cls.max_items) - - return v - - -def conset(item_type: Type[T], *, min_items: Optional[int] = None, max_items: Optional[int] = None) -> Type[Set[T]]: - # __args__ is needed to conform to typing generics api - namespace = {'min_items': min_items, 'max_items': max_items, 'item_type': item_type, '__args__': [item_type]} - # We use new_class to be able to deal with Generic types - return new_class('ConstrainedSetValue', (ConstrainedSet,), {}, lambda ns: ns.update(namespace)) - - -# This types superclass should be FrozenSet[T], but cython chokes on that... -class ConstrainedFrozenSet(frozenset): # type: ignore - # Needed for pydantic to detect that this is a set - __origin__ = frozenset - __args__: FrozenSet[Type[T]] # type: ignore - - min_items: Optional[int] = None - max_items: Optional[int] = None - item_type: Type[T] # type: ignore - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.frozenset_length_validator - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none(field_schema, minItems=cls.min_items, maxItems=cls.max_items) - - @classmethod - def frozenset_length_validator(cls, v: 'Optional[FrozenSet[T]]') -> 'Optional[FrozenSet[T]]': - if v is None: - return None - - v = frozenset_validator(v) - v_len = len(v) - - if cls.min_items is not None and v_len < cls.min_items: - raise errors.FrozenSetMinLengthError(limit_value=cls.min_items) - - if cls.max_items is not None and v_len > cls.max_items: - raise errors.FrozenSetMaxLengthError(limit_value=cls.max_items) - - return v - - -def confrozenset( - item_type: Type[T], *, min_items: Optional[int] = None, max_items: Optional[int] = None -) -> Type[FrozenSet[T]]: - # __args__ is needed to conform to typing generics api - namespace = {'min_items': min_items, 'max_items': max_items, 'item_type': item_type, '__args__': [item_type]} - # We use new_class to be able to deal with Generic types - return new_class('ConstrainedFrozenSetValue', (ConstrainedFrozenSet,), {}, lambda ns: ns.update(namespace)) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ LIST TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -# This types superclass should be List[T], but cython chokes on that... -class ConstrainedList(list): # type: ignore - # Needed for pydantic to detect that this is a list - __origin__ = list - __args__: Tuple[Type[T], ...] # type: ignore - - min_items: Optional[int] = None - max_items: Optional[int] = None - unique_items: Optional[bool] = None - item_type: Type[T] # type: ignore - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.list_length_validator - if cls.unique_items: - yield cls.unique_items_validator - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none(field_schema, minItems=cls.min_items, maxItems=cls.max_items, uniqueItems=cls.unique_items) - - @classmethod - def list_length_validator(cls, v: 'Optional[List[T]]') -> 'Optional[List[T]]': - if v is None: - return None - - v = list_validator(v) - v_len = len(v) - - if cls.min_items is not None and v_len < cls.min_items: - raise errors.ListMinLengthError(limit_value=cls.min_items) - - if cls.max_items is not None and v_len > cls.max_items: - raise errors.ListMaxLengthError(limit_value=cls.max_items) - - return v - - @classmethod - def unique_items_validator(cls, v: 'Optional[List[T]]') -> 'Optional[List[T]]': - if v is None: - return None - - for i, value in enumerate(v, start=1): - if value in v[i:]: - raise errors.ListUniqueItemsError() - - return v - - -def conlist( - item_type: Type[T], *, min_items: Optional[int] = None, max_items: Optional[int] = None, unique_items: bool = None -) -> Type[List[T]]: - # __args__ is needed to conform to typing generics api - namespace = dict( - min_items=min_items, max_items=max_items, unique_items=unique_items, item_type=item_type, __args__=(item_type,) - ) - # We use new_class to be able to deal with Generic types - return new_class('ConstrainedListValue', (ConstrainedList,), {}, lambda ns: ns.update(namespace)) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PYOBJECT TYPE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -if TYPE_CHECKING: - PyObject = Callable[..., Any] -else: - - class PyObject: - validate_always = True - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, value: Any) -> Any: - if isinstance(value, Callable): - return value - - try: - value = str_validator(value) - except errors.StrError: - raise errors.PyObjectError(error_message='value is neither a valid import path not a valid callable') - - try: - return import_string(value) - except ImportError as e: - raise errors.PyObjectError(error_message=str(e)) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DECIMAL TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class ConstrainedDecimal(Decimal, metaclass=ConstrainedNumberMeta): - gt: OptionalIntFloatDecimal = None - ge: OptionalIntFloatDecimal = None - lt: OptionalIntFloatDecimal = None - le: OptionalIntFloatDecimal = None - max_digits: OptionalInt = None - decimal_places: OptionalInt = None - multiple_of: OptionalIntFloatDecimal = None - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none( - field_schema, - exclusiveMinimum=cls.gt, - exclusiveMaximum=cls.lt, - minimum=cls.ge, - maximum=cls.le, - multipleOf=cls.multiple_of, - ) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield decimal_validator - yield number_size_validator - yield number_multiple_validator - yield cls.validate - - @classmethod - def validate(cls, value: Decimal) -> Decimal: - try: - normalized_value = value.normalize() - except InvalidOperation: - normalized_value = value - digit_tuple, exponent = normalized_value.as_tuple()[1:] - if exponent in {'F', 'n', 'N'}: - raise errors.DecimalIsNotFiniteError() - - if exponent >= 0: - # A positive exponent adds that many trailing zeros. - digits = len(digit_tuple) + exponent - decimals = 0 - else: - # If the absolute value of the negative exponent is larger than the - # number of digits, then it's the same as the number of digits, - # because it'll consume all of the digits in digit_tuple and then - # add abs(exponent) - len(digit_tuple) leading zeros after the - # decimal point. - if abs(exponent) > len(digit_tuple): - digits = decimals = abs(exponent) - else: - digits = len(digit_tuple) - decimals = abs(exponent) - whole_digits = digits - decimals - - if cls.max_digits is not None and digits > cls.max_digits: - raise errors.DecimalMaxDigitsError(max_digits=cls.max_digits) - - if cls.decimal_places is not None and decimals > cls.decimal_places: - raise errors.DecimalMaxPlacesError(decimal_places=cls.decimal_places) - - if cls.max_digits is not None and cls.decimal_places is not None: - expected = cls.max_digits - cls.decimal_places - if whole_digits > expected: - raise errors.DecimalWholeDigitsError(whole_digits=expected) - - return value - - -def condecimal( - *, - gt: Decimal = None, - ge: Decimal = None, - lt: Decimal = None, - le: Decimal = None, - max_digits: Optional[int] = None, - decimal_places: Optional[int] = None, - multiple_of: Decimal = None, -) -> Type[Decimal]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict( - gt=gt, ge=ge, lt=lt, le=le, max_digits=max_digits, decimal_places=decimal_places, multiple_of=multiple_of - ) - return type('ConstrainedDecimalValue', (ConstrainedDecimal,), namespace) - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ UUID TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -if TYPE_CHECKING: - UUID1 = UUID - UUID3 = UUID - UUID4 = UUID - UUID5 = UUID -else: - - class UUID1(UUID): - _required_version = 1 - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format=f'uuid{cls._required_version}') - - class UUID3(UUID1): - _required_version = 3 - - class UUID4(UUID1): - _required_version = 4 - - class UUID5(UUID1): - _required_version = 5 - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PATH TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -if TYPE_CHECKING: - FilePath = Path - DirectoryPath = Path -else: - - class FilePath(Path): - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(format='file-path') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield path_validator - yield path_exists_validator - yield cls.validate - - @classmethod - def validate(cls, value: Path) -> Path: - if not value.is_file(): - raise errors.PathNotAFileError(path=value) - - return value - - class DirectoryPath(Path): - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(format='directory-path') - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield path_validator - yield path_exists_validator - yield cls.validate - - @classmethod - def validate(cls, value: Path) -> Path: - if not value.is_dir(): - raise errors.PathNotADirectoryError(path=value) - - return value - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ JSON TYPE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class JsonWrapper: - pass - - -class JsonMeta(type): - def __getitem__(self, t: Type[Any]) -> Type[JsonWrapper]: - if t is Any: - return Json # allow Json[Any] to replicate plain Json - return _registered(type('JsonWrapperValue', (JsonWrapper,), {'inner_type': t})) - - -if TYPE_CHECKING: - Json = Annotated[T, ...] # Json[list[str]] will be recognized by type checkers as list[str] - -else: - - class Json(metaclass=JsonMeta): - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - field_schema.update(type='string', format='json-string') - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ SECRET TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class SecretField(abc.ABC): - """ - Note: this should be implemented as a generic like `SecretField(ABC, Generic[T])`, - the `__init__()` should be part of the abstract class and the - `get_secret_value()` method should use the generic `T` type. - - However Cython doesn't support very well generics at the moment and - the generated code fails to be imported (see - https://github.com/cython/cython/issues/2753). - """ - - def __eq__(self, other: Any) -> bool: - return isinstance(other, self.__class__) and self.get_secret_value() == other.get_secret_value() - - def __str__(self) -> str: - return '**********' if self.get_secret_value() else '' - - def __hash__(self) -> int: - return hash(self.get_secret_value()) - - @abc.abstractmethod - def get_secret_value(self) -> Any: # pragma: no cover - ... - - -class SecretStr(SecretField): - min_length: OptionalInt = None - max_length: OptionalInt = None - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none( - field_schema, - type='string', - writeOnly=True, - format='password', - minLength=cls.min_length, - maxLength=cls.max_length, - ) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - yield constr_length_validator - - @classmethod - def validate(cls, value: Any) -> 'SecretStr': - if isinstance(value, cls): - return value - value = str_validator(value) - return cls(value) - - def __init__(self, value: str): - self._secret_value = value - - def __repr__(self) -> str: - return f"SecretStr('{self}')" - - def __len__(self) -> int: - return len(self._secret_value) - - def display(self) -> str: - warnings.warn('`secret_str.display()` is deprecated, use `str(secret_str)` instead', DeprecationWarning) - return str(self) - - def get_secret_value(self) -> str: - return self._secret_value - - -class SecretBytes(SecretField): - min_length: OptionalInt = None - max_length: OptionalInt = None - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none( - field_schema, - type='string', - writeOnly=True, - format='password', - minLength=cls.min_length, - maxLength=cls.max_length, - ) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - yield constr_length_validator - - @classmethod - def validate(cls, value: Any) -> 'SecretBytes': - if isinstance(value, cls): - return value - value = bytes_validator(value) - return cls(value) - - def __init__(self, value: bytes): - self._secret_value = value - - def __repr__(self) -> str: - return f"SecretBytes(b'{self}')" - - def __len__(self) -> int: - return len(self._secret_value) - - def display(self) -> str: - warnings.warn('`secret_bytes.display()` is deprecated, use `str(secret_bytes)` instead', DeprecationWarning) - return str(self) - - def get_secret_value(self) -> bytes: - return self._secret_value - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ PAYMENT CARD TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - -class PaymentCardBrand(str, Enum): - # If you add another card type, please also add it to the - # Hypothesis strategy in `pydantic._hypothesis_plugin`. - amex = 'American Express' - mastercard = 'Mastercard' - visa = 'Visa' - other = 'other' - - def __str__(self) -> str: - return self.value - - -class PaymentCardNumber(str): - """ - Based on: https://en.wikipedia.org/wiki/Payment_card_number - """ - - strip_whitespace: ClassVar[bool] = True - min_length: ClassVar[int] = 12 - max_length: ClassVar[int] = 19 - bin: str - last4: str - brand: PaymentCardBrand - - def __init__(self, card_number: str): - self.bin = card_number[:6] - self.last4 = card_number[-4:] - self.brand = self._get_brand(card_number) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield str_validator - yield constr_strip_whitespace - yield constr_length_validator - yield cls.validate_digits - yield cls.validate_luhn_check_digit - yield cls - yield cls.validate_length_for_brand - - @property - def masked(self) -> str: - num_masked = len(self) - 10 # len(bin) + len(last4) == 10 - return f'{self.bin}{"*" * num_masked}{self.last4}' - - @classmethod - def validate_digits(cls, card_number: str) -> str: - if not card_number.isdigit(): - raise errors.NotDigitError - return card_number - - @classmethod - def validate_luhn_check_digit(cls, card_number: str) -> str: - """ - Based on: https://en.wikipedia.org/wiki/Luhn_algorithm - """ - sum_ = int(card_number[-1]) - length = len(card_number) - parity = length % 2 - for i in range(length - 1): - digit = int(card_number[i]) - if i % 2 == parity: - digit *= 2 - if digit > 9: - digit -= 9 - sum_ += digit - valid = sum_ % 10 == 0 - if not valid: - raise errors.LuhnValidationError - return card_number - - @classmethod - def validate_length_for_brand(cls, card_number: 'PaymentCardNumber') -> 'PaymentCardNumber': - """ - Validate length based on BIN for major brands: - https://en.wikipedia.org/wiki/Payment_card_number#Issuer_identification_number_(IIN) - """ - required_length: Union[None, int, str] = None - if card_number.brand in PaymentCardBrand.mastercard: - required_length = 16 - valid = len(card_number) == required_length - elif card_number.brand == PaymentCardBrand.visa: - required_length = '13, 16 or 19' - valid = len(card_number) in {13, 16, 19} - elif card_number.brand == PaymentCardBrand.amex: - required_length = 15 - valid = len(card_number) == required_length - else: - valid = True - if not valid: - raise errors.InvalidLengthForBrand(brand=card_number.brand, required_length=required_length) - return card_number - - @staticmethod - def _get_brand(card_number: str) -> PaymentCardBrand: - if card_number[0] == '4': - brand = PaymentCardBrand.visa - elif 51 <= int(card_number[:2]) <= 55: - brand = PaymentCardBrand.mastercard - elif card_number[:2] in {'34', '37'}: - brand = PaymentCardBrand.amex - else: - brand = PaymentCardBrand.other - return brand - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ BYTE SIZE TYPE ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -BYTE_SIZES = { - 'b': 1, - 'kb': 10**3, - 'mb': 10**6, - 'gb': 10**9, - 'tb': 10**12, - 'pb': 10**15, - 'eb': 10**18, - 'kib': 2**10, - 'mib': 2**20, - 'gib': 2**30, - 'tib': 2**40, - 'pib': 2**50, - 'eib': 2**60, -} -BYTE_SIZES.update({k.lower()[0]: v for k, v in BYTE_SIZES.items() if 'i' not in k}) -byte_string_re = re.compile(r'^\s*(\d*\.?\d+)\s*(\w+)?', re.IGNORECASE) - - -class ByteSize(int): - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield cls.validate - - @classmethod - def validate(cls, v: StrIntFloat) -> 'ByteSize': - try: - return cls(int(v)) - except ValueError: - pass - - str_match = byte_string_re.match(str(v)) - if str_match is None: - raise errors.InvalidByteSize() - - scalar, unit = str_match.groups() - if unit is None: - unit = 'b' - - try: - unit_mult = BYTE_SIZES[unit.lower()] - except KeyError: - raise errors.InvalidByteSizeUnit(unit=unit) - - return cls(int(float(scalar) * unit_mult)) - - def human_readable(self, decimal: bool = False) -> str: - if decimal: - divisor = 1000 - units = ['B', 'KB', 'MB', 'GB', 'TB', 'PB'] - final_unit = 'EB' - else: - divisor = 1024 - units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'] - final_unit = 'EiB' - - num = float(self) - for unit in units: - if abs(num) < divisor: - return f'{num:0.1f}{unit}' - num /= divisor - - return f'{num:0.1f}{final_unit}' - - def to(self, unit: str) -> float: - try: - unit_div = BYTE_SIZES[unit.lower()] - except KeyError: - raise errors.InvalidByteSizeUnit(unit=unit) - - return self / unit_div - - -# ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ DATE TYPES ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -if TYPE_CHECKING: - PastDate = date - FutureDate = date -else: - - class PastDate(date): - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield parse_date - yield cls.validate - - @classmethod - def validate(cls, value: date) -> date: - if value >= date.today(): - raise errors.DateNotInThePastError() - - return value - - class FutureDate(date): - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield parse_date - yield cls.validate - - @classmethod - def validate(cls, value: date) -> date: - if value <= date.today(): - raise errors.DateNotInTheFutureError() - - return value - - -class ConstrainedDate(date, metaclass=ConstrainedNumberMeta): - gt: OptionalDate = None - ge: OptionalDate = None - lt: OptionalDate = None - le: OptionalDate = None - - @classmethod - def __modify_schema__(cls, field_schema: Dict[str, Any]) -> None: - update_not_none(field_schema, exclusiveMinimum=cls.gt, exclusiveMaximum=cls.lt, minimum=cls.ge, maximum=cls.le) - - @classmethod - def __get_validators__(cls) -> 'CallableGenerator': - yield parse_date - yield number_size_validator - - -def condate( - *, - gt: date = None, - ge: date = None, - lt: date = None, - le: date = None, -) -> Type[date]: - # use kwargs then define conf in a dict to aid with IDE type hinting - namespace = dict(gt=gt, ge=ge, lt=lt, le=le) - return type('ConstrainedDateValue', (ConstrainedDate,), namespace) diff --git a/functions/python/data-processor/pydantic/v1/typing.py b/functions/python/data-processor/pydantic/v1/typing.py deleted file mode 100644 index c5c59794b..000000000 --- a/functions/python/data-processor/pydantic/v1/typing.py +++ /dev/null @@ -1,627 +0,0 @@ -import functools -import operator -import sys -import typing -from collections.abc import Callable -from os import PathLike -from typing import ( # type: ignore - TYPE_CHECKING, - AbstractSet, - Any, - Callable as TypingCallable, - ClassVar, - Dict, - ForwardRef, - Generator, - Iterable, - List, - Mapping, - NewType, - Optional, - Sequence, - Set, - Tuple, - Type, - TypeVar, - Union, - _eval_type, - cast, - get_type_hints, -) - -from typing_extensions import ( - Annotated, - Final, - Literal, - NotRequired as TypedDictNotRequired, - Required as TypedDictRequired, -) - -try: - from typing import _TypingBase as typing_base # type: ignore -except ImportError: - from typing import _Final as typing_base # type: ignore - -try: - from typing import GenericAlias as TypingGenericAlias # type: ignore -except ImportError: - # python < 3.9 does not have GenericAlias (list[int], tuple[str, ...] and so on) - TypingGenericAlias = () - -try: - from types import UnionType as TypesUnionType # type: ignore -except ImportError: - # python < 3.10 does not have UnionType (str | int, byte | bool and so on) - TypesUnionType = () - - -if sys.version_info < (3, 9): - - def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any: - return type_._evaluate(globalns, localns) - -elif sys.version_info < (3, 12, 4): - - def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any: - # Even though it is the right signature for python 3.9, mypy complains with - # `error: Too many arguments for "_evaluate" of "ForwardRef"` hence the cast... - # Python 3.13/3.12.4+ made `recursive_guard` a kwarg, so name it explicitly to avoid: - # TypeError: ForwardRef._evaluate() missing 1 required keyword-only argument: 'recursive_guard' - return cast(Any, type_)._evaluate(globalns, localns, recursive_guard=set()) - -elif sys.version_info < (3, 14): - - def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any: - # Pydantic 1.x will not support PEP 695 syntax, but provide `type_params` to avoid - # warnings: - return cast(Any, type_)._evaluate(globalns, localns, type_params=(), recursive_guard=set()) - -else: - - def evaluate_forwardref(type_: ForwardRef, globalns: Any, localns: Any) -> Any: - # Pydantic 1.x will not support PEP 695 syntax, but provide `type_params` to avoid - # warnings: - return typing.evaluate_forward_ref( - type_, - globals=globalns, - locals=localns, - type_params=(), - _recursive_guard=set(), - ) - - -if sys.version_info < (3, 9): - # Ensure we always get all the whole `Annotated` hint, not just the annotated type. - # For 3.7 to 3.8, `get_type_hints` doesn't recognize `typing_extensions.Annotated`, - # so it already returns the full annotation - get_all_type_hints = get_type_hints - -else: - - def get_all_type_hints(obj: Any, globalns: Any = None, localns: Any = None) -> Any: - return get_type_hints(obj, globalns, localns, include_extras=True) - - -_T = TypeVar('_T') - -AnyCallable = TypingCallable[..., Any] -NoArgAnyCallable = TypingCallable[[], Any] - -# workaround for https://github.com/python/mypy/issues/9496 -AnyArgTCallable = TypingCallable[..., _T] - - -# Annotated[...] is implemented by returning an instance of one of these classes, depending on -# python/typing_extensions version. -AnnotatedTypeNames = {'AnnotatedMeta', '_AnnotatedAlias'} - - -LITERAL_TYPES: Set[Any] = {Literal} -if hasattr(typing, 'Literal'): - LITERAL_TYPES.add(typing.Literal) - - -if sys.version_info < (3, 8): - - def get_origin(t: Type[Any]) -> Optional[Type[Any]]: - if type(t).__name__ in AnnotatedTypeNames: - # weirdly this is a runtime requirement, as well as for mypy - return cast(Type[Any], Annotated) - return getattr(t, '__origin__', None) - -else: - from typing import get_origin as _typing_get_origin - - def get_origin(tp: Type[Any]) -> Optional[Type[Any]]: - """ - We can't directly use `typing.get_origin` since we need a fallback to support - custom generic classes like `ConstrainedList` - It should be useless once https://github.com/cython/cython/issues/3537 is - solved and https://github.com/pydantic/pydantic/pull/1753 is merged. - """ - if type(tp).__name__ in AnnotatedTypeNames: - return cast(Type[Any], Annotated) # mypy complains about _SpecialForm - return _typing_get_origin(tp) or getattr(tp, '__origin__', None) - - -if sys.version_info < (3, 8): - from typing import _GenericAlias - - def get_args(t: Type[Any]) -> Tuple[Any, ...]: - """Compatibility version of get_args for python 3.7. - - Mostly compatible with the python 3.8 `typing` module version - and able to handle almost all use cases. - """ - if type(t).__name__ in AnnotatedTypeNames: - return t.__args__ + t.__metadata__ - if isinstance(t, _GenericAlias): - res = t.__args__ - if t.__origin__ is Callable and res and res[0] is not Ellipsis: - res = (list(res[:-1]), res[-1]) - return res - return getattr(t, '__args__', ()) - -else: - from typing import get_args as _typing_get_args - - def _generic_get_args(tp: Type[Any]) -> Tuple[Any, ...]: - """ - In python 3.9, `typing.Dict`, `typing.List`, ... - do have an empty `__args__` by default (instead of the generic ~T for example). - In order to still support `Dict` for example and consider it as `Dict[Any, Any]`, - we retrieve the `_nparams` value that tells us how many parameters it needs. - """ - if hasattr(tp, '_nparams'): - return (Any,) * tp._nparams - # Special case for `tuple[()]`, which used to return ((),) with `typing.Tuple` - # in python 3.10- but now returns () for `tuple` and `Tuple`. - # This will probably be clarified in pydantic v2 - try: - if tp == Tuple[()] or sys.version_info >= (3, 9) and tp == tuple[()]: # type: ignore[misc] - return ((),) - # there is a TypeError when compiled with cython - except TypeError: # pragma: no cover - pass - return () - - def get_args(tp: Type[Any]) -> Tuple[Any, ...]: - """Get type arguments with all substitutions performed. - - For unions, basic simplifications used by Union constructor are performed. - Examples:: - get_args(Dict[str, int]) == (str, int) - get_args(int) == () - get_args(Union[int, Union[T, int], str][int]) == (int, str) - get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - get_args(Callable[[], T][int]) == ([], int) - """ - if type(tp).__name__ in AnnotatedTypeNames: - return tp.__args__ + tp.__metadata__ - # the fallback is needed for the same reasons as `get_origin` (see above) - return _typing_get_args(tp) or getattr(tp, '__args__', ()) or _generic_get_args(tp) - - -if sys.version_info < (3, 9): - - def convert_generics(tp: Type[Any]) -> Type[Any]: - """Python 3.9 and older only supports generics from `typing` module. - They convert strings to ForwardRef automatically. - - Examples:: - typing.List['Hero'] == typing.List[ForwardRef('Hero')] - """ - return tp - -else: - - def convert_generics(tp: Type[Any]) -> Type[Any]: - """ - Recursively searches for `str` type hints and replaces them with ForwardRef. - - Examples:: - convert_generics(list['Hero']) == list[ForwardRef('Hero')] - convert_generics(dict['Hero', 'Team']) == dict[ForwardRef('Hero'), ForwardRef('Team')] - convert_generics(typing.Dict['Hero', 'Team']) == typing.Dict[ForwardRef('Hero'), ForwardRef('Team')] - convert_generics(list[str | 'Hero'] | int) == list[str | ForwardRef('Hero')] | int - """ - origin = get_origin(tp) - if not origin or not hasattr(tp, '__args__'): - return tp - - args = get_args(tp) - - # typing.Annotated needs special treatment - if origin is Annotated: - return Annotated[(convert_generics(args[0]), *args[1:])] # type: ignore - - # recursively replace `str` instances inside of `GenericAlias` with `ForwardRef(arg)` - converted = tuple( - ForwardRef(arg) if isinstance(arg, str) and isinstance(tp, TypingGenericAlias) else convert_generics(arg) - for arg in args - ) - - if converted == args: - return tp - elif isinstance(tp, TypingGenericAlias): - return TypingGenericAlias(origin, converted) - elif isinstance(tp, TypesUnionType): - # recreate types.UnionType (PEP604, Python >= 3.10) - return functools.reduce(operator.or_, converted) # type: ignore - else: - try: - setattr(tp, '__args__', converted) - except AttributeError: - pass - return tp - - -if sys.version_info < (3, 10): - - def is_union(tp: Optional[Type[Any]]) -> bool: - return tp is Union - - WithArgsTypes = (TypingGenericAlias,) - -else: - import types - import typing - - def is_union(tp: Optional[Type[Any]]) -> bool: - return tp is Union or tp is types.UnionType # noqa: E721 - - WithArgsTypes = (typing._GenericAlias, types.GenericAlias, types.UnionType) - - -StrPath = Union[str, PathLike] - - -if TYPE_CHECKING: - from pydantic.v1.fields import ModelField - - TupleGenerator = Generator[Tuple[str, Any], None, None] - DictStrAny = Dict[str, Any] - DictAny = Dict[Any, Any] - SetStr = Set[str] - ListStr = List[str] - IntStr = Union[int, str] - AbstractSetIntStr = AbstractSet[IntStr] - DictIntStrAny = Dict[IntStr, Any] - MappingIntStrAny = Mapping[IntStr, Any] - CallableGenerator = Generator[AnyCallable, None, None] - ReprArgs = Sequence[Tuple[Optional[str], Any]] - - MYPY = False - if MYPY: - AnyClassMethod = classmethod[Any] - else: - # classmethod[TargetType, CallableParamSpecType, CallableReturnType] - AnyClassMethod = classmethod[Any, Any, Any] - -__all__ = ( - 'AnyCallable', - 'NoArgAnyCallable', - 'NoneType', - 'is_none_type', - 'display_as_type', - 'resolve_annotations', - 'is_callable_type', - 'is_literal_type', - 'all_literal_values', - 'is_namedtuple', - 'is_typeddict', - 'is_typeddict_special', - 'is_new_type', - 'new_type_supertype', - 'is_classvar', - 'is_finalvar', - 'update_field_forward_refs', - 'update_model_forward_refs', - 'TupleGenerator', - 'DictStrAny', - 'DictAny', - 'SetStr', - 'ListStr', - 'IntStr', - 'AbstractSetIntStr', - 'DictIntStrAny', - 'CallableGenerator', - 'ReprArgs', - 'AnyClassMethod', - 'CallableGenerator', - 'WithArgsTypes', - 'get_args', - 'get_origin', - 'get_sub_types', - 'typing_base', - 'get_all_type_hints', - 'is_union', - 'StrPath', - 'MappingIntStrAny', -) - - -NoneType = None.__class__ - - -NONE_TYPES: Tuple[Any, Any, Any] = (None, NoneType, Literal[None]) - - -if sys.version_info < (3, 8): - # Even though this implementation is slower, we need it for python 3.7: - # In python 3.7 "Literal" is not a builtin type and uses a different - # mechanism. - # for this reason `Literal[None] is Literal[None]` evaluates to `False`, - # breaking the faster implementation used for the other python versions. - - def is_none_type(type_: Any) -> bool: - return type_ in NONE_TYPES - -elif sys.version_info[:2] == (3, 8): - - def is_none_type(type_: Any) -> bool: - for none_type in NONE_TYPES: - if type_ is none_type: - return True - # With python 3.8, specifically 3.8.10, Literal "is" check sare very flakey - # can change on very subtle changes like use of types in other modules, - # hopefully this check avoids that issue. - if is_literal_type(type_): # pragma: no cover - return all_literal_values(type_) == (None,) - return False - -else: - - def is_none_type(type_: Any) -> bool: - return type_ in NONE_TYPES - - -def display_as_type(v: Type[Any]) -> str: - if not isinstance(v, typing_base) and not isinstance(v, WithArgsTypes) and not isinstance(v, type): - v = v.__class__ - - if is_union(get_origin(v)): - return f'Union[{", ".join(map(display_as_type, get_args(v)))}]' - - if isinstance(v, WithArgsTypes): - # Generic alias are constructs like `list[int]` - return str(v).replace('typing.', '') - - try: - return v.__name__ - except AttributeError: - # happens with typing objects - return str(v).replace('typing.', '') - - -def resolve_annotations(raw_annotations: Dict[str, Type[Any]], module_name: Optional[str]) -> Dict[str, Type[Any]]: - """ - Partially taken from typing.get_type_hints. - - Resolve string or ForwardRef annotations into type objects if possible. - """ - base_globals: Optional[Dict[str, Any]] = None - if module_name: - try: - module = sys.modules[module_name] - except KeyError: - # happens occasionally, see https://github.com/pydantic/pydantic/issues/2363 - pass - else: - base_globals = module.__dict__ - - annotations = {} - for name, value in raw_annotations.items(): - if isinstance(value, str): - if (3, 10) > sys.version_info >= (3, 9, 8) or sys.version_info >= (3, 10, 1): - value = ForwardRef(value, is_argument=False, is_class=True) - else: - value = ForwardRef(value, is_argument=False) - try: - if sys.version_info >= (3, 13): - value = _eval_type(value, base_globals, None, type_params=()) - else: - value = _eval_type(value, base_globals, None) - except NameError: - # this is ok, it can be fixed with update_forward_refs - pass - annotations[name] = value - return annotations - - -def is_callable_type(type_: Type[Any]) -> bool: - return type_ is Callable or get_origin(type_) is Callable - - -def is_literal_type(type_: Type[Any]) -> bool: - return Literal is not None and get_origin(type_) in LITERAL_TYPES - - -def literal_values(type_: Type[Any]) -> Tuple[Any, ...]: - return get_args(type_) - - -def all_literal_values(type_: Type[Any]) -> Tuple[Any, ...]: - """ - This method is used to retrieve all Literal values as - Literal can be used recursively (see https://www.python.org/dev/peps/pep-0586) - e.g. `Literal[Literal[Literal[1, 2, 3], "foo"], 5, None]` - """ - if not is_literal_type(type_): - return (type_,) - - values = literal_values(type_) - return tuple(x for value in values for x in all_literal_values(value)) - - -def is_namedtuple(type_: Type[Any]) -> bool: - """ - Check if a given class is a named tuple. - It can be either a `typing.NamedTuple` or `collections.namedtuple` - """ - from pydantic.v1.utils import lenient_issubclass - - return lenient_issubclass(type_, tuple) and hasattr(type_, '_fields') - - -def is_typeddict(type_: Type[Any]) -> bool: - """ - Check if a given class is a typed dict (from `typing` or `typing_extensions`) - In 3.10, there will be a public method (https://docs.python.org/3.10/library/typing.html#typing.is_typeddict) - """ - from pydantic.v1.utils import lenient_issubclass - - return lenient_issubclass(type_, dict) and hasattr(type_, '__total__') - - -def _check_typeddict_special(type_: Any) -> bool: - return type_ is TypedDictRequired or type_ is TypedDictNotRequired - - -def is_typeddict_special(type_: Any) -> bool: - """ - Check if type is a TypedDict special form (Required or NotRequired). - """ - return _check_typeddict_special(type_) or _check_typeddict_special(get_origin(type_)) - - -test_type = NewType('test_type', str) - - -def is_new_type(type_: Type[Any]) -> bool: - """ - Check whether type_ was created using typing.NewType - """ - return isinstance(type_, test_type.__class__) and hasattr(type_, '__supertype__') # type: ignore - - -def new_type_supertype(type_: Type[Any]) -> Type[Any]: - while hasattr(type_, '__supertype__'): - type_ = type_.__supertype__ - return type_ - - -def _check_classvar(v: Optional[Type[Any]]) -> bool: - if v is None: - return False - - return v.__class__ == ClassVar.__class__ and getattr(v, '_name', None) == 'ClassVar' - - -def _check_finalvar(v: Optional[Type[Any]]) -> bool: - """ - Check if a given type is a `typing.Final` type. - """ - if v is None: - return False - - return v.__class__ == Final.__class__ and (sys.version_info < (3, 8) or getattr(v, '_name', None) == 'Final') - - -def is_classvar(ann_type: Type[Any]) -> bool: - if _check_classvar(ann_type) or _check_classvar(get_origin(ann_type)): - return True - - # this is an ugly workaround for class vars that contain forward references and are therefore themselves - # forward references, see #3679 - if ann_type.__class__ == ForwardRef and ann_type.__forward_arg__.startswith('ClassVar['): - return True - - return False - - -def is_finalvar(ann_type: Type[Any]) -> bool: - return _check_finalvar(ann_type) or _check_finalvar(get_origin(ann_type)) - - -def update_field_forward_refs(field: 'ModelField', globalns: Any, localns: Any) -> None: - """ - Try to update ForwardRefs on fields based on this ModelField, globalns and localns. - """ - prepare = False - if field.type_.__class__ == ForwardRef: - prepare = True - field.type_ = evaluate_forwardref(field.type_, globalns, localns or None) - if field.outer_type_.__class__ == ForwardRef: - prepare = True - field.outer_type_ = evaluate_forwardref(field.outer_type_, globalns, localns or None) - if prepare: - field.prepare() - - if field.sub_fields: - for sub_f in field.sub_fields: - update_field_forward_refs(sub_f, globalns=globalns, localns=localns) - - if field.discriminator_key is not None: - field.prepare_discriminated_union_sub_fields() - - -def update_model_forward_refs( - model: Type[Any], - fields: Iterable['ModelField'], - json_encoders: Dict[Union[Type[Any], str, ForwardRef], AnyCallable], - localns: 'DictStrAny', - exc_to_suppress: Tuple[Type[BaseException], ...] = (), -) -> None: - """ - Try to update model fields ForwardRefs based on model and localns. - """ - if model.__module__ in sys.modules: - globalns = sys.modules[model.__module__].__dict__.copy() - else: - globalns = {} - - globalns.setdefault(model.__name__, model) - - for f in fields: - try: - update_field_forward_refs(f, globalns=globalns, localns=localns) - except exc_to_suppress: - pass - - for key in set(json_encoders.keys()): - if isinstance(key, str): - fr: ForwardRef = ForwardRef(key) - elif isinstance(key, ForwardRef): - fr = key - else: - continue - - try: - new_key = evaluate_forwardref(fr, globalns, localns or None) - except exc_to_suppress: # pragma: no cover - continue - - json_encoders[new_key] = json_encoders.pop(key) - - -def get_class(type_: Type[Any]) -> Union[None, bool, Type[Any]]: - """ - Tries to get the class of a Type[T] annotation. Returns True if Type is used - without brackets. Otherwise returns None. - """ - if type_ is type: - return True - - if get_origin(type_) is None: - return None - - args = get_args(type_) - if not args or not isinstance(args[0], type): - return True - else: - return args[0] - - -def get_sub_types(tp: Any) -> List[Any]: - """ - Return all the types that are allowed by type `tp` - `tp` can be a `Union` of allowed types or an `Annotated` type - """ - origin = get_origin(tp) - if origin is Annotated: - return get_sub_types(get_args(tp)[0]) - elif is_union(origin): - return [x for t in get_args(tp) for x in get_sub_types(t)] - else: - return [tp] diff --git a/functions/python/data-processor/pydantic/v1/utils.py b/functions/python/data-processor/pydantic/v1/utils.py deleted file mode 100644 index 2094e84fa..000000000 --- a/functions/python/data-processor/pydantic/v1/utils.py +++ /dev/null @@ -1,807 +0,0 @@ -import keyword -import warnings -import weakref -from collections import OrderedDict, defaultdict, deque -from copy import deepcopy -from itertools import islice, zip_longest -from types import BuiltinFunctionType, CodeType, FunctionType, GeneratorType, LambdaType, ModuleType -from typing import ( - TYPE_CHECKING, - AbstractSet, - Any, - Callable, - Collection, - Dict, - Generator, - Iterable, - Iterator, - List, - Mapping, - NoReturn, - Optional, - Set, - Tuple, - Type, - TypeVar, - Union, -) - -from typing_extensions import Annotated - -from pydantic.v1.errors import ConfigError -from pydantic.v1.typing import ( - NoneType, - WithArgsTypes, - all_literal_values, - display_as_type, - get_args, - get_origin, - is_literal_type, - is_union, -) -from pydantic.v1.version import version_info - -if TYPE_CHECKING: - from inspect import Signature - from pathlib import Path - - from pydantic.v1.config import BaseConfig - from pydantic.v1.dataclasses import Dataclass - from pydantic.v1.fields import ModelField - from pydantic.v1.main import BaseModel - from pydantic.v1.typing import AbstractSetIntStr, DictIntStrAny, IntStr, MappingIntStrAny, ReprArgs - - RichReprResult = Iterable[Union[Any, Tuple[Any], Tuple[str, Any], Tuple[str, Any, Any]]] - -__all__ = ( - 'import_string', - 'sequence_like', - 'validate_field_name', - 'lenient_isinstance', - 'lenient_issubclass', - 'in_ipython', - 'is_valid_identifier', - 'deep_update', - 'update_not_none', - 'almost_equal_floats', - 'get_model', - 'to_camel', - 'to_lower_camel', - 'is_valid_field', - 'smart_deepcopy', - 'PyObjectStr', - 'Representation', - 'GetterDict', - 'ValueItems', - 'version_info', # required here to match behaviour in v1.3 - 'ClassAttribute', - 'path_type', - 'ROOT_KEY', - 'get_unique_discriminator_alias', - 'get_discriminator_alias_and_values', - 'DUNDER_ATTRIBUTES', -) - -ROOT_KEY = '__root__' -# these are types that are returned unchanged by deepcopy -IMMUTABLE_NON_COLLECTIONS_TYPES: Set[Type[Any]] = { - int, - float, - complex, - str, - bool, - bytes, - type, - NoneType, - FunctionType, - BuiltinFunctionType, - LambdaType, - weakref.ref, - CodeType, - # note: including ModuleType will differ from behaviour of deepcopy by not producing error. - # It might be not a good idea in general, but considering that this function used only internally - # against default values of fields, this will allow to actually have a field with module as default value - ModuleType, - NotImplemented.__class__, - Ellipsis.__class__, -} - -# these are types that if empty, might be copied with simple copy() instead of deepcopy() -BUILTIN_COLLECTIONS: Set[Type[Any]] = { - list, - set, - tuple, - frozenset, - dict, - OrderedDict, - defaultdict, - deque, -} - - -def import_string(dotted_path: str) -> Any: - """ - Stolen approximately from django. Import a dotted module path and return the attribute/class designated by the - last name in the path. Raise ImportError if the import fails. - """ - from importlib import import_module - - try: - module_path, class_name = dotted_path.strip(' ').rsplit('.', 1) - except ValueError as e: - raise ImportError(f'"{dotted_path}" doesn\'t look like a module path') from e - - module = import_module(module_path) - try: - return getattr(module, class_name) - except AttributeError as e: - raise ImportError(f'Module "{module_path}" does not define a "{class_name}" attribute') from e - - -def truncate(v: Union[str], *, max_len: int = 80) -> str: - """ - Truncate a value and add a unicode ellipsis (three dots) to the end if it was too long - """ - warnings.warn('`truncate` is no-longer used by pydantic and is deprecated', DeprecationWarning) - if isinstance(v, str) and len(v) > (max_len - 2): - # -3 so quote + string + … + quote has correct length - return (v[: (max_len - 3)] + '…').__repr__() - try: - v = v.__repr__() - except TypeError: - v = v.__class__.__repr__(v) # in case v is a type - if len(v) > max_len: - v = v[: max_len - 1] + '…' - return v - - -def sequence_like(v: Any) -> bool: - return isinstance(v, (list, tuple, set, frozenset, GeneratorType, deque)) - - -def validate_field_name(bases: Iterable[Type[Any]], field_name: str) -> None: - """ - Ensure that the field's name does not shadow an existing attribute of the model. - """ - for base in bases: - if getattr(base, field_name, None): - raise NameError( - f'Field name "{field_name}" shadows a BaseModel attribute; ' - f'use a different field name with "alias=\'{field_name}\'".' - ) - - -def lenient_isinstance(o: Any, class_or_tuple: Union[Type[Any], Tuple[Type[Any], ...], None]) -> bool: - try: - return isinstance(o, class_or_tuple) # type: ignore[arg-type] - except TypeError: - return False - - -def lenient_issubclass(cls: Any, class_or_tuple: Union[Type[Any], Tuple[Type[Any], ...], None]) -> bool: - try: - return isinstance(cls, type) and issubclass(cls, class_or_tuple) # type: ignore[arg-type] - except TypeError: - if isinstance(cls, WithArgsTypes): - return False - raise # pragma: no cover - - -def in_ipython() -> bool: - """ - Check whether we're in an ipython environment, including jupyter notebooks. - """ - try: - eval('__IPYTHON__') - except NameError: - return False - else: # pragma: no cover - return True - - -def is_valid_identifier(identifier: str) -> bool: - """ - Checks that a string is a valid identifier and not a Python keyword. - :param identifier: The identifier to test. - :return: True if the identifier is valid. - """ - return identifier.isidentifier() and not keyword.iskeyword(identifier) - - -KeyType = TypeVar('KeyType') - - -def deep_update(mapping: Dict[KeyType, Any], *updating_mappings: Dict[KeyType, Any]) -> Dict[KeyType, Any]: - updated_mapping = mapping.copy() - for updating_mapping in updating_mappings: - for k, v in updating_mapping.items(): - if k in updated_mapping and isinstance(updated_mapping[k], dict) and isinstance(v, dict): - updated_mapping[k] = deep_update(updated_mapping[k], v) - else: - updated_mapping[k] = v - return updated_mapping - - -def update_not_none(mapping: Dict[Any, Any], **update: Any) -> None: - mapping.update({k: v for k, v in update.items() if v is not None}) - - -def almost_equal_floats(value_1: float, value_2: float, *, delta: float = 1e-8) -> bool: - """ - Return True if two floats are almost equal - """ - return abs(value_1 - value_2) <= delta - - -def generate_model_signature( - init: Callable[..., None], fields: Dict[str, 'ModelField'], config: Type['BaseConfig'] -) -> 'Signature': - """ - Generate signature for model based on its fields - """ - from inspect import Parameter, Signature, signature - - from pydantic.v1.config import Extra - - present_params = signature(init).parameters.values() - merged_params: Dict[str, Parameter] = {} - var_kw = None - use_var_kw = False - - for param in islice(present_params, 1, None): # skip self arg - if param.kind is param.VAR_KEYWORD: - var_kw = param - continue - merged_params[param.name] = param - - if var_kw: # if custom init has no var_kw, fields which are not declared in it cannot be passed through - allow_names = config.allow_population_by_field_name - for field_name, field in fields.items(): - param_name = field.alias - if field_name in merged_params or param_name in merged_params: - continue - elif not is_valid_identifier(param_name): - if allow_names and is_valid_identifier(field_name): - param_name = field_name - else: - use_var_kw = True - continue - - # TODO: replace annotation with actual expected types once #1055 solved - kwargs = {'default': field.default} if not field.required else {} - merged_params[param_name] = Parameter( - param_name, Parameter.KEYWORD_ONLY, annotation=field.annotation, **kwargs - ) - - if config.extra is Extra.allow: - use_var_kw = True - - if var_kw and use_var_kw: - # Make sure the parameter for extra kwargs - # does not have the same name as a field - default_model_signature = [ - ('__pydantic_self__', Parameter.POSITIONAL_OR_KEYWORD), - ('data', Parameter.VAR_KEYWORD), - ] - if [(p.name, p.kind) for p in present_params] == default_model_signature: - # if this is the standard model signature, use extra_data as the extra args name - var_kw_name = 'extra_data' - else: - # else start from var_kw - var_kw_name = var_kw.name - - # generate a name that's definitely unique - while var_kw_name in fields: - var_kw_name += '_' - merged_params[var_kw_name] = var_kw.replace(name=var_kw_name) - - return Signature(parameters=list(merged_params.values()), return_annotation=None) - - -def get_model(obj: Union[Type['BaseModel'], Type['Dataclass']]) -> Type['BaseModel']: - from pydantic.v1.main import BaseModel - - try: - model_cls = obj.__pydantic_model__ # type: ignore - except AttributeError: - model_cls = obj - - if not issubclass(model_cls, BaseModel): - raise TypeError('Unsupported type, must be either BaseModel or dataclass') - return model_cls - - -def to_camel(string: str) -> str: - return ''.join(word.capitalize() for word in string.split('_')) - - -def to_lower_camel(string: str) -> str: - if len(string) >= 1: - pascal_string = to_camel(string) - return pascal_string[0].lower() + pascal_string[1:] - return string.lower() - - -T = TypeVar('T') - - -def unique_list( - input_list: Union[List[T], Tuple[T, ...]], - *, - name_factory: Callable[[T], str] = str, -) -> List[T]: - """ - Make a list unique while maintaining order. - We update the list if another one with the same name is set - (e.g. root validator overridden in subclass) - """ - result: List[T] = [] - result_names: List[str] = [] - for v in input_list: - v_name = name_factory(v) - if v_name not in result_names: - result_names.append(v_name) - result.append(v) - else: - result[result_names.index(v_name)] = v - - return result - - -class PyObjectStr(str): - """ - String class where repr doesn't include quotes. Useful with Representation when you want to return a string - representation of something that valid (or pseudo-valid) python. - """ - - def __repr__(self) -> str: - return str(self) - - -class Representation: - """ - Mixin to provide __str__, __repr__, and __pretty__ methods. See #884 for more details. - - __pretty__ is used by [devtools](https://python-devtools.helpmanual.io/) to provide human readable representations - of objects. - """ - - __slots__: Tuple[str, ...] = tuple() - - def __repr_args__(self) -> 'ReprArgs': - """ - Returns the attributes to show in __str__, __repr__, and __pretty__ this is generally overridden. - - Can either return: - * name - value pairs, e.g.: `[('foo_name', 'foo'), ('bar_name', ['b', 'a', 'r'])]` - * or, just values, e.g.: `[(None, 'foo'), (None, ['b', 'a', 'r'])]` - """ - attrs = ((s, getattr(self, s)) for s in self.__slots__) - return [(a, v) for a, v in attrs if v is not None] - - def __repr_name__(self) -> str: - """ - Name of the instance's class, used in __repr__. - """ - return self.__class__.__name__ - - def __repr_str__(self, join_str: str) -> str: - return join_str.join(repr(v) if a is None else f'{a}={v!r}' for a, v in self.__repr_args__()) - - def __pretty__(self, fmt: Callable[[Any], Any], **kwargs: Any) -> Generator[Any, None, None]: - """ - Used by devtools (https://python-devtools.helpmanual.io/) to provide a human readable representations of objects - """ - yield self.__repr_name__() + '(' - yield 1 - for name, value in self.__repr_args__(): - if name is not None: - yield name + '=' - yield fmt(value) - yield ',' - yield 0 - yield -1 - yield ')' - - def __str__(self) -> str: - return self.__repr_str__(' ') - - def __repr__(self) -> str: - return f'{self.__repr_name__()}({self.__repr_str__(", ")})' - - def __rich_repr__(self) -> 'RichReprResult': - """Get fields for Rich library""" - for name, field_repr in self.__repr_args__(): - if name is None: - yield field_repr - else: - yield name, field_repr - - -class GetterDict(Representation): - """ - Hack to make object's smell just enough like dicts for validate_model. - - We can't inherit from Mapping[str, Any] because it upsets cython so we have to implement all methods ourselves. - """ - - __slots__ = ('_obj',) - - def __init__(self, obj: Any): - self._obj = obj - - def __getitem__(self, key: str) -> Any: - try: - return getattr(self._obj, key) - except AttributeError as e: - raise KeyError(key) from e - - def get(self, key: Any, default: Any = None) -> Any: - return getattr(self._obj, key, default) - - def extra_keys(self) -> Set[Any]: - """ - We don't want to get any other attributes of obj if the model didn't explicitly ask for them - """ - return set() - - def keys(self) -> List[Any]: - """ - Keys of the pseudo dictionary, uses a list not set so order information can be maintained like python - dictionaries. - """ - return list(self) - - def values(self) -> List[Any]: - return [self[k] for k in self] - - def items(self) -> Iterator[Tuple[str, Any]]: - for k in self: - yield k, self.get(k) - - def __iter__(self) -> Iterator[str]: - for name in dir(self._obj): - if not name.startswith('_'): - yield name - - def __len__(self) -> int: - return sum(1 for _ in self) - - def __contains__(self, item: Any) -> bool: - return item in self.keys() - - def __eq__(self, other: Any) -> bool: - return dict(self) == dict(other.items()) - - def __repr_args__(self) -> 'ReprArgs': - return [(None, dict(self))] - - def __repr_name__(self) -> str: - return f'GetterDict[{display_as_type(self._obj)}]' - - -class ValueItems(Representation): - """ - Class for more convenient calculation of excluded or included fields on values. - """ - - __slots__ = ('_items', '_type') - - def __init__(self, value: Any, items: Union['AbstractSetIntStr', 'MappingIntStrAny']) -> None: - items = self._coerce_items(items) - - if isinstance(value, (list, tuple)): - items = self._normalize_indexes(items, len(value)) - - self._items: 'MappingIntStrAny' = items - - def is_excluded(self, item: Any) -> bool: - """ - Check if item is fully excluded. - - :param item: key or index of a value - """ - return self.is_true(self._items.get(item)) - - def is_included(self, item: Any) -> bool: - """ - Check if value is contained in self._items - - :param item: key or index of value - """ - return item in self._items - - def for_element(self, e: 'IntStr') -> Optional[Union['AbstractSetIntStr', 'MappingIntStrAny']]: - """ - :param e: key or index of element on value - :return: raw values for element if self._items is dict and contain needed element - """ - - item = self._items.get(e) - return item if not self.is_true(item) else None - - def _normalize_indexes(self, items: 'MappingIntStrAny', v_length: int) -> 'DictIntStrAny': - """ - :param items: dict or set of indexes which will be normalized - :param v_length: length of sequence indexes of which will be - - >>> self._normalize_indexes({0: True, -2: True, -1: True}, 4) - {0: True, 2: True, 3: True} - >>> self._normalize_indexes({'__all__': True}, 4) - {0: True, 1: True, 2: True, 3: True} - """ - - normalized_items: 'DictIntStrAny' = {} - all_items = None - for i, v in items.items(): - if not (isinstance(v, Mapping) or isinstance(v, AbstractSet) or self.is_true(v)): - raise TypeError(f'Unexpected type of exclude value for index "{i}" {v.__class__}') - if i == '__all__': - all_items = self._coerce_value(v) - continue - if not isinstance(i, int): - raise TypeError( - 'Excluding fields from a sequence of sub-models or dicts must be performed index-wise: ' - 'expected integer keys or keyword "__all__"' - ) - normalized_i = v_length + i if i < 0 else i - normalized_items[normalized_i] = self.merge(v, normalized_items.get(normalized_i)) - - if not all_items: - return normalized_items - if self.is_true(all_items): - for i in range(v_length): - normalized_items.setdefault(i, ...) - return normalized_items - for i in range(v_length): - normalized_item = normalized_items.setdefault(i, {}) - if not self.is_true(normalized_item): - normalized_items[i] = self.merge(all_items, normalized_item) - return normalized_items - - @classmethod - def merge(cls, base: Any, override: Any, intersect: bool = False) -> Any: - """ - Merge a ``base`` item with an ``override`` item. - - Both ``base`` and ``override`` are converted to dictionaries if possible. - Sets are converted to dictionaries with the sets entries as keys and - Ellipsis as values. - - Each key-value pair existing in ``base`` is merged with ``override``, - while the rest of the key-value pairs are updated recursively with this function. - - Merging takes place based on the "union" of keys if ``intersect`` is - set to ``False`` (default) and on the intersection of keys if - ``intersect`` is set to ``True``. - """ - override = cls._coerce_value(override) - base = cls._coerce_value(base) - if override is None: - return base - if cls.is_true(base) or base is None: - return override - if cls.is_true(override): - return base if intersect else override - - # intersection or union of keys while preserving ordering: - if intersect: - merge_keys = [k for k in base if k in override] + [k for k in override if k in base] - else: - merge_keys = list(base) + [k for k in override if k not in base] - - merged: 'DictIntStrAny' = {} - for k in merge_keys: - merged_item = cls.merge(base.get(k), override.get(k), intersect=intersect) - if merged_item is not None: - merged[k] = merged_item - - return merged - - @staticmethod - def _coerce_items(items: Union['AbstractSetIntStr', 'MappingIntStrAny']) -> 'MappingIntStrAny': - if isinstance(items, Mapping): - pass - elif isinstance(items, AbstractSet): - items = dict.fromkeys(items, ...) - else: - class_name = getattr(items, '__class__', '???') - assert_never( - items, - f'Unexpected type of exclude value {class_name}', - ) - return items - - @classmethod - def _coerce_value(cls, value: Any) -> Any: - if value is None or cls.is_true(value): - return value - return cls._coerce_items(value) - - @staticmethod - def is_true(v: Any) -> bool: - return v is True or v is ... - - def __repr_args__(self) -> 'ReprArgs': - return [(None, self._items)] - - -class ClassAttribute: - """ - Hide class attribute from its instances - """ - - __slots__ = ( - 'name', - 'value', - ) - - def __init__(self, name: str, value: Any) -> None: - self.name = name - self.value = value - - def __get__(self, instance: Any, owner: Type[Any]) -> None: - if instance is None: - return self.value - raise AttributeError(f'{self.name!r} attribute of {owner.__name__!r} is class-only') - - -path_types = { - 'is_dir': 'directory', - 'is_file': 'file', - 'is_mount': 'mount point', - 'is_symlink': 'symlink', - 'is_block_device': 'block device', - 'is_char_device': 'char device', - 'is_fifo': 'FIFO', - 'is_socket': 'socket', -} - - -def path_type(p: 'Path') -> str: - """ - Find out what sort of thing a path is. - """ - assert p.exists(), 'path does not exist' - for method, name in path_types.items(): - if getattr(p, method)(): - return name - - return 'unknown' - - -Obj = TypeVar('Obj') - - -def smart_deepcopy(obj: Obj) -> Obj: - """ - Return type as is for immutable built-in types - Use obj.copy() for built-in empty collections - Use copy.deepcopy() for non-empty collections and unknown objects - """ - - obj_type = obj.__class__ - if obj_type in IMMUTABLE_NON_COLLECTIONS_TYPES: - return obj # fastest case: obj is immutable and not collection therefore will not be copied anyway - try: - if not obj and obj_type in BUILTIN_COLLECTIONS: - # faster way for empty collections, no need to copy its members - return obj if obj_type is tuple else obj.copy() # type: ignore # tuple doesn't have copy method - except (TypeError, ValueError, RuntimeError): - # do we really dare to catch ALL errors? Seems a bit risky - pass - - return deepcopy(obj) # slowest way when we actually might need a deepcopy - - -def is_valid_field(name: str) -> bool: - if not name.startswith('_'): - return True - return ROOT_KEY == name - - -DUNDER_ATTRIBUTES = { - '__annotations__', - '__classcell__', - '__doc__', - '__module__', - '__orig_bases__', - '__orig_class__', - '__qualname__', - '__firstlineno__', - '__static_attributes__', - '__classdictcell__', -} - - -def is_valid_private_name(name: str) -> bool: - return not is_valid_field(name) and name not in DUNDER_ATTRIBUTES - - -_EMPTY = object() - - -def all_identical(left: Iterable[Any], right: Iterable[Any]) -> bool: - """ - Check that the items of `left` are the same objects as those in `right`. - - >>> a, b = object(), object() - >>> all_identical([a, b, a], [a, b, a]) - True - >>> all_identical([a, b, [a]], [a, b, [a]]) # new list object, while "equal" is not "identical" - False - """ - for left_item, right_item in zip_longest(left, right, fillvalue=_EMPTY): - if left_item is not right_item: - return False - return True - - -def assert_never(obj: NoReturn, msg: str) -> NoReturn: - """ - Helper to make sure that we have covered all possible types. - - This is mostly useful for ``mypy``, docs: - https://mypy.readthedocs.io/en/latest/literal_types.html#exhaustive-checks - """ - raise TypeError(msg) - - -def get_unique_discriminator_alias(all_aliases: Collection[str], discriminator_key: str) -> str: - """Validate that all aliases are the same and if that's the case return the alias""" - unique_aliases = set(all_aliases) - if len(unique_aliases) > 1: - raise ConfigError( - f'Aliases for discriminator {discriminator_key!r} must be the same (got {", ".join(sorted(all_aliases))})' - ) - return unique_aliases.pop() - - -def get_discriminator_alias_and_values(tp: Any, discriminator_key: str) -> Tuple[str, Tuple[str, ...]]: - """ - Get alias and all valid values in the `Literal` type of the discriminator field - `tp` can be a `BaseModel` class or directly an `Annotated` `Union` of many. - """ - is_root_model = getattr(tp, '__custom_root_type__', False) - - if get_origin(tp) is Annotated: - tp = get_args(tp)[0] - - if hasattr(tp, '__pydantic_model__'): - tp = tp.__pydantic_model__ - - if is_union(get_origin(tp)): - alias, all_values = _get_union_alias_and_all_values(tp, discriminator_key) - return alias, tuple(v for values in all_values for v in values) - elif is_root_model: - union_type = tp.__fields__[ROOT_KEY].type_ - alias, all_values = _get_union_alias_and_all_values(union_type, discriminator_key) - - if len(set(all_values)) > 1: - raise ConfigError( - f'Field {discriminator_key!r} is not the same for all submodels of {display_as_type(tp)!r}' - ) - - return alias, all_values[0] - - else: - try: - t_discriminator_type = tp.__fields__[discriminator_key].type_ - except AttributeError as e: - raise TypeError(f'Type {tp.__name__!r} is not a valid `BaseModel` or `dataclass`') from e - except KeyError as e: - raise ConfigError(f'Model {tp.__name__!r} needs a discriminator field for key {discriminator_key!r}') from e - - if not is_literal_type(t_discriminator_type): - raise ConfigError(f'Field {discriminator_key!r} of model {tp.__name__!r} needs to be a `Literal`') - - return tp.__fields__[discriminator_key].alias, all_literal_values(t_discriminator_type) - - -def _get_union_alias_and_all_values( - union_type: Type[Any], discriminator_key: str -) -> Tuple[str, Tuple[Tuple[str, ...], ...]]: - zipped_aliases_values = [get_discriminator_alias_and_values(t, discriminator_key) for t in get_args(union_type)] - # unzip: [('alias_a',('v1', 'v2)), ('alias_b', ('v3',))] => [('alias_a', 'alias_b'), (('v1', 'v2'), ('v3',))] - all_aliases, all_values = zip(*zipped_aliases_values) - return get_unique_discriminator_alias(all_aliases, discriminator_key), all_values diff --git a/functions/python/data-processor/pydantic/v1/validators.py b/functions/python/data-processor/pydantic/v1/validators.py deleted file mode 100644 index c0940e811..000000000 --- a/functions/python/data-processor/pydantic/v1/validators.py +++ /dev/null @@ -1,768 +0,0 @@ -import math -import re -from collections import OrderedDict, deque -from collections.abc import Hashable as CollectionsHashable -from datetime import date, datetime, time, timedelta -from decimal import Decimal, DecimalException -from enum import Enum, IntEnum -from ipaddress import IPv4Address, IPv4Interface, IPv4Network, IPv6Address, IPv6Interface, IPv6Network -from pathlib import Path -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Deque, - Dict, - ForwardRef, - FrozenSet, - Generator, - Hashable, - List, - NamedTuple, - Pattern, - Set, - Tuple, - Type, - TypeVar, - Union, -) -from uuid import UUID -from warnings import warn - -from pydantic.v1 import errors -from pydantic.v1.datetime_parse import parse_date, parse_datetime, parse_duration, parse_time -from pydantic.v1.typing import ( - AnyCallable, - all_literal_values, - display_as_type, - get_class, - is_callable_type, - is_literal_type, - is_namedtuple, - is_none_type, - is_typeddict, -) -from pydantic.v1.utils import almost_equal_floats, lenient_issubclass, sequence_like - -if TYPE_CHECKING: - from typing_extensions import Literal, TypedDict - - from pydantic.v1.config import BaseConfig - from pydantic.v1.fields import ModelField - from pydantic.v1.types import ConstrainedDecimal, ConstrainedFloat, ConstrainedInt - - ConstrainedNumber = Union[ConstrainedDecimal, ConstrainedFloat, ConstrainedInt] - AnyOrderedDict = OrderedDict[Any, Any] - Number = Union[int, float, Decimal] - StrBytes = Union[str, bytes] - - -def str_validator(v: Any) -> Union[str]: - if isinstance(v, str): - if isinstance(v, Enum): - return v.value - else: - return v - elif isinstance(v, (float, int, Decimal)): - # is there anything else we want to add here? If you think so, create an issue. - return str(v) - elif isinstance(v, (bytes, bytearray)): - return v.decode() - else: - raise errors.StrError() - - -def strict_str_validator(v: Any) -> Union[str]: - if isinstance(v, str) and not isinstance(v, Enum): - return v - raise errors.StrError() - - -def bytes_validator(v: Any) -> Union[bytes]: - if isinstance(v, bytes): - return v - elif isinstance(v, bytearray): - return bytes(v) - elif isinstance(v, str): - return v.encode() - elif isinstance(v, (float, int, Decimal)): - return str(v).encode() - else: - raise errors.BytesError() - - -def strict_bytes_validator(v: Any) -> Union[bytes]: - if isinstance(v, bytes): - return v - elif isinstance(v, bytearray): - return bytes(v) - else: - raise errors.BytesError() - - -BOOL_FALSE = {0, '0', 'off', 'f', 'false', 'n', 'no'} -BOOL_TRUE = {1, '1', 'on', 't', 'true', 'y', 'yes'} - - -def bool_validator(v: Any) -> bool: - if v is True or v is False: - return v - if isinstance(v, bytes): - v = v.decode() - if isinstance(v, str): - v = v.lower() - try: - if v in BOOL_TRUE: - return True - if v in BOOL_FALSE: - return False - except TypeError: - raise errors.BoolError() - raise errors.BoolError() - - -# matches the default limit cpython, see https://github.com/python/cpython/pull/96500 -max_str_int = 4_300 - - -def int_validator(v: Any) -> int: - if isinstance(v, int) and not (v is True or v is False): - return v - - # see https://github.com/pydantic/pydantic/issues/1477 and in turn, https://github.com/python/cpython/issues/95778 - # this check should be unnecessary once patch releases are out for 3.7, 3.8, 3.9 and 3.10 - # but better to check here until then. - # NOTICE: this does not fully protect user from the DOS risk since the standard library JSON implementation - # (and other std lib modules like xml) use `int()` and are likely called before this, the best workaround is to - # 1. update to the latest patch release of python once released, 2. use a different JSON library like ujson - if isinstance(v, (str, bytes, bytearray)) and len(v) > max_str_int: - raise errors.IntegerError() - - try: - return int(v) - except (TypeError, ValueError, OverflowError): - raise errors.IntegerError() - - -def strict_int_validator(v: Any) -> int: - if isinstance(v, int) and not (v is True or v is False): - return v - raise errors.IntegerError() - - -def float_validator(v: Any) -> float: - if isinstance(v, float): - return v - - try: - return float(v) - except (TypeError, ValueError): - raise errors.FloatError() - - -def strict_float_validator(v: Any) -> float: - if isinstance(v, float): - return v - raise errors.FloatError() - - -def float_finite_validator(v: 'Number', field: 'ModelField', config: 'BaseConfig') -> 'Number': - allow_inf_nan = getattr(field.type_, 'allow_inf_nan', None) - if allow_inf_nan is None: - allow_inf_nan = config.allow_inf_nan - - if allow_inf_nan is False and (math.isnan(v) or math.isinf(v)): - raise errors.NumberNotFiniteError() - return v - - -def number_multiple_validator(v: 'Number', field: 'ModelField') -> 'Number': - field_type: ConstrainedNumber = field.type_ - if field_type.multiple_of is not None: - mod = float(v) / float(field_type.multiple_of) % 1 - if not almost_equal_floats(mod, 0.0) and not almost_equal_floats(mod, 1.0): - raise errors.NumberNotMultipleError(multiple_of=field_type.multiple_of) - return v - - -def number_size_validator(v: 'Number', field: 'ModelField') -> 'Number': - field_type: ConstrainedNumber = field.type_ - if field_type.gt is not None and not v > field_type.gt: - raise errors.NumberNotGtError(limit_value=field_type.gt) - elif field_type.ge is not None and not v >= field_type.ge: - raise errors.NumberNotGeError(limit_value=field_type.ge) - - if field_type.lt is not None and not v < field_type.lt: - raise errors.NumberNotLtError(limit_value=field_type.lt) - if field_type.le is not None and not v <= field_type.le: - raise errors.NumberNotLeError(limit_value=field_type.le) - - return v - - -def constant_validator(v: 'Any', field: 'ModelField') -> 'Any': - """Validate ``const`` fields. - - The value provided for a ``const`` field must be equal to the default value - of the field. This is to support the keyword of the same name in JSON - Schema. - """ - if v != field.default: - raise errors.WrongConstantError(given=v, permitted=[field.default]) - - return v - - -def anystr_length_validator(v: 'StrBytes', config: 'BaseConfig') -> 'StrBytes': - v_len = len(v) - - min_length = config.min_anystr_length - if v_len < min_length: - raise errors.AnyStrMinLengthError(limit_value=min_length) - - max_length = config.max_anystr_length - if max_length is not None and v_len > max_length: - raise errors.AnyStrMaxLengthError(limit_value=max_length) - - return v - - -def anystr_strip_whitespace(v: 'StrBytes') -> 'StrBytes': - return v.strip() - - -def anystr_upper(v: 'StrBytes') -> 'StrBytes': - return v.upper() - - -def anystr_lower(v: 'StrBytes') -> 'StrBytes': - return v.lower() - - -def ordered_dict_validator(v: Any) -> 'AnyOrderedDict': - if isinstance(v, OrderedDict): - return v - - try: - return OrderedDict(v) - except (TypeError, ValueError): - raise errors.DictError() - - -def dict_validator(v: Any) -> Dict[Any, Any]: - if isinstance(v, dict): - return v - - try: - return dict(v) - except (TypeError, ValueError): - raise errors.DictError() - - -def list_validator(v: Any) -> List[Any]: - if isinstance(v, list): - return v - elif sequence_like(v): - return list(v) - else: - raise errors.ListError() - - -def tuple_validator(v: Any) -> Tuple[Any, ...]: - if isinstance(v, tuple): - return v - elif sequence_like(v): - return tuple(v) - else: - raise errors.TupleError() - - -def set_validator(v: Any) -> Set[Any]: - if isinstance(v, set): - return v - elif sequence_like(v): - return set(v) - else: - raise errors.SetError() - - -def frozenset_validator(v: Any) -> FrozenSet[Any]: - if isinstance(v, frozenset): - return v - elif sequence_like(v): - return frozenset(v) - else: - raise errors.FrozenSetError() - - -def deque_validator(v: Any) -> Deque[Any]: - if isinstance(v, deque): - return v - elif sequence_like(v): - return deque(v) - else: - raise errors.DequeError() - - -def enum_member_validator(v: Any, field: 'ModelField', config: 'BaseConfig') -> Enum: - try: - enum_v = field.type_(v) - except ValueError: - # field.type_ should be an enum, so will be iterable - raise errors.EnumMemberError(enum_values=list(field.type_)) - return enum_v.value if config.use_enum_values else enum_v - - -def uuid_validator(v: Any, field: 'ModelField') -> UUID: - try: - if isinstance(v, str): - v = UUID(v) - elif isinstance(v, (bytes, bytearray)): - try: - v = UUID(v.decode()) - except ValueError: - # 16 bytes in big-endian order as the bytes argument fail - # the above check - v = UUID(bytes=v) - except ValueError: - raise errors.UUIDError() - - if not isinstance(v, UUID): - raise errors.UUIDError() - - required_version = getattr(field.type_, '_required_version', None) - if required_version and v.version != required_version: - raise errors.UUIDVersionError(required_version=required_version) - - return v - - -def decimal_validator(v: Any) -> Decimal: - if isinstance(v, Decimal): - return v - elif isinstance(v, (bytes, bytearray)): - v = v.decode() - - v = str(v).strip() - - try: - v = Decimal(v) - except DecimalException: - raise errors.DecimalError() - - if not v.is_finite(): - raise errors.DecimalIsNotFiniteError() - - return v - - -def hashable_validator(v: Any) -> Hashable: - if isinstance(v, Hashable): - return v - - raise errors.HashableError() - - -def ip_v4_address_validator(v: Any) -> IPv4Address: - if isinstance(v, IPv4Address): - return v - - try: - return IPv4Address(v) - except ValueError: - raise errors.IPv4AddressError() - - -def ip_v6_address_validator(v: Any) -> IPv6Address: - if isinstance(v, IPv6Address): - return v - - try: - return IPv6Address(v) - except ValueError: - raise errors.IPv6AddressError() - - -def ip_v4_network_validator(v: Any) -> IPv4Network: - """ - Assume IPv4Network initialised with a default ``strict`` argument - - See more: - https://docs.python.org/library/ipaddress.html#ipaddress.IPv4Network - """ - if isinstance(v, IPv4Network): - return v - - try: - return IPv4Network(v) - except ValueError: - raise errors.IPv4NetworkError() - - -def ip_v6_network_validator(v: Any) -> IPv6Network: - """ - Assume IPv6Network initialised with a default ``strict`` argument - - See more: - https://docs.python.org/library/ipaddress.html#ipaddress.IPv6Network - """ - if isinstance(v, IPv6Network): - return v - - try: - return IPv6Network(v) - except ValueError: - raise errors.IPv6NetworkError() - - -def ip_v4_interface_validator(v: Any) -> IPv4Interface: - if isinstance(v, IPv4Interface): - return v - - try: - return IPv4Interface(v) - except ValueError: - raise errors.IPv4InterfaceError() - - -def ip_v6_interface_validator(v: Any) -> IPv6Interface: - if isinstance(v, IPv6Interface): - return v - - try: - return IPv6Interface(v) - except ValueError: - raise errors.IPv6InterfaceError() - - -def path_validator(v: Any) -> Path: - if isinstance(v, Path): - return v - - try: - return Path(v) - except TypeError: - raise errors.PathError() - - -def path_exists_validator(v: Any) -> Path: - if not v.exists(): - raise errors.PathNotExistsError(path=v) - - return v - - -def callable_validator(v: Any) -> AnyCallable: - """ - Perform a simple check if the value is callable. - - Note: complete matching of argument type hints and return types is not performed - """ - if callable(v): - return v - - raise errors.CallableError(value=v) - - -def enum_validator(v: Any) -> Enum: - if isinstance(v, Enum): - return v - - raise errors.EnumError(value=v) - - -def int_enum_validator(v: Any) -> IntEnum: - if isinstance(v, IntEnum): - return v - - raise errors.IntEnumError(value=v) - - -def make_literal_validator(type_: Any) -> Callable[[Any], Any]: - permitted_choices = all_literal_values(type_) - - # To have a O(1) complexity and still return one of the values set inside the `Literal`, - # we create a dict with the set values (a set causes some problems with the way intersection works). - # In some cases the set value and checked value can indeed be different (see `test_literal_validator_str_enum`) - allowed_choices = {v: v for v in permitted_choices} - - def literal_validator(v: Any) -> Any: - try: - return allowed_choices[v] - except (KeyError, TypeError): - raise errors.WrongConstantError(given=v, permitted=permitted_choices) - - return literal_validator - - -def constr_length_validator(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes': - v_len = len(v) - - min_length = field.type_.min_length if field.type_.min_length is not None else config.min_anystr_length - if v_len < min_length: - raise errors.AnyStrMinLengthError(limit_value=min_length) - - max_length = field.type_.max_length if field.type_.max_length is not None else config.max_anystr_length - if max_length is not None and v_len > max_length: - raise errors.AnyStrMaxLengthError(limit_value=max_length) - - return v - - -def constr_strip_whitespace(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes': - strip_whitespace = field.type_.strip_whitespace or config.anystr_strip_whitespace - if strip_whitespace: - v = v.strip() - - return v - - -def constr_upper(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes': - upper = field.type_.to_upper or config.anystr_upper - if upper: - v = v.upper() - - return v - - -def constr_lower(v: 'StrBytes', field: 'ModelField', config: 'BaseConfig') -> 'StrBytes': - lower = field.type_.to_lower or config.anystr_lower - if lower: - v = v.lower() - return v - - -def validate_json(v: Any, config: 'BaseConfig') -> Any: - if v is None: - # pass None through to other validators - return v - try: - return config.json_loads(v) # type: ignore - except ValueError: - raise errors.JsonError() - except TypeError: - raise errors.JsonTypeError() - - -T = TypeVar('T') - - -def make_arbitrary_type_validator(type_: Type[T]) -> Callable[[T], T]: - def arbitrary_type_validator(v: Any) -> T: - if isinstance(v, type_): - return v - raise errors.ArbitraryTypeError(expected_arbitrary_type=type_) - - return arbitrary_type_validator - - -def make_class_validator(type_: Type[T]) -> Callable[[Any], Type[T]]: - def class_validator(v: Any) -> Type[T]: - if lenient_issubclass(v, type_): - return v - raise errors.SubclassError(expected_class=type_) - - return class_validator - - -def any_class_validator(v: Any) -> Type[T]: - if isinstance(v, type): - return v - raise errors.ClassError() - - -def none_validator(v: Any) -> 'Literal[None]': - if v is None: - return v - raise errors.NotNoneError() - - -def pattern_validator(v: Any) -> Pattern[str]: - if isinstance(v, Pattern): - return v - - str_value = str_validator(v) - - try: - return re.compile(str_value) - except re.error: - raise errors.PatternError() - - -NamedTupleT = TypeVar('NamedTupleT', bound=NamedTuple) - - -def make_namedtuple_validator( - namedtuple_cls: Type[NamedTupleT], config: Type['BaseConfig'] -) -> Callable[[Tuple[Any, ...]], NamedTupleT]: - from pydantic.v1.annotated_types import create_model_from_namedtuple - - NamedTupleModel = create_model_from_namedtuple( - namedtuple_cls, - __config__=config, - __module__=namedtuple_cls.__module__, - ) - namedtuple_cls.__pydantic_model__ = NamedTupleModel # type: ignore[attr-defined] - - def namedtuple_validator(values: Tuple[Any, ...]) -> NamedTupleT: - annotations = NamedTupleModel.__annotations__ - - if len(values) > len(annotations): - raise errors.ListMaxLengthError(limit_value=len(annotations)) - - dict_values: Dict[str, Any] = dict(zip(annotations, values)) - validated_dict_values: Dict[str, Any] = dict(NamedTupleModel(**dict_values)) - return namedtuple_cls(**validated_dict_values) - - return namedtuple_validator - - -def make_typeddict_validator( - typeddict_cls: Type['TypedDict'], config: Type['BaseConfig'] # type: ignore[valid-type] -) -> Callable[[Any], Dict[str, Any]]: - from pydantic.v1.annotated_types import create_model_from_typeddict - - TypedDictModel = create_model_from_typeddict( - typeddict_cls, - __config__=config, - __module__=typeddict_cls.__module__, - ) - typeddict_cls.__pydantic_model__ = TypedDictModel # type: ignore[attr-defined] - - def typeddict_validator(values: 'TypedDict') -> Dict[str, Any]: # type: ignore[valid-type] - return TypedDictModel.parse_obj(values).dict(exclude_unset=True) - - return typeddict_validator - - -class IfConfig: - def __init__(self, validator: AnyCallable, *config_attr_names: str, ignored_value: Any = False) -> None: - self.validator = validator - self.config_attr_names = config_attr_names - self.ignored_value = ignored_value - - def check(self, config: Type['BaseConfig']) -> bool: - return any(getattr(config, name) not in {None, self.ignored_value} for name in self.config_attr_names) - - -# order is important here, for example: bool is a subclass of int so has to come first, datetime before date same, -# IPv4Interface before IPv4Address, etc -_VALIDATORS: List[Tuple[Type[Any], List[Any]]] = [ - (IntEnum, [int_validator, enum_member_validator]), - (Enum, [enum_member_validator]), - ( - str, - [ - str_validator, - IfConfig(anystr_strip_whitespace, 'anystr_strip_whitespace'), - IfConfig(anystr_upper, 'anystr_upper'), - IfConfig(anystr_lower, 'anystr_lower'), - IfConfig(anystr_length_validator, 'min_anystr_length', 'max_anystr_length'), - ], - ), - ( - bytes, - [ - bytes_validator, - IfConfig(anystr_strip_whitespace, 'anystr_strip_whitespace'), - IfConfig(anystr_upper, 'anystr_upper'), - IfConfig(anystr_lower, 'anystr_lower'), - IfConfig(anystr_length_validator, 'min_anystr_length', 'max_anystr_length'), - ], - ), - (bool, [bool_validator]), - (int, [int_validator]), - (float, [float_validator, IfConfig(float_finite_validator, 'allow_inf_nan', ignored_value=True)]), - (Path, [path_validator]), - (datetime, [parse_datetime]), - (date, [parse_date]), - (time, [parse_time]), - (timedelta, [parse_duration]), - (OrderedDict, [ordered_dict_validator]), - (dict, [dict_validator]), - (list, [list_validator]), - (tuple, [tuple_validator]), - (set, [set_validator]), - (frozenset, [frozenset_validator]), - (deque, [deque_validator]), - (UUID, [uuid_validator]), - (Decimal, [decimal_validator]), - (IPv4Interface, [ip_v4_interface_validator]), - (IPv6Interface, [ip_v6_interface_validator]), - (IPv4Address, [ip_v4_address_validator]), - (IPv6Address, [ip_v6_address_validator]), - (IPv4Network, [ip_v4_network_validator]), - (IPv6Network, [ip_v6_network_validator]), -] - - -def find_validators( # noqa: C901 (ignore complexity) - type_: Type[Any], config: Type['BaseConfig'] -) -> Generator[AnyCallable, None, None]: - from pydantic.v1.dataclasses import is_builtin_dataclass, make_dataclass_validator - - if type_ is Any or type_ is object: - return - type_type = type_.__class__ - if type_type == ForwardRef or type_type == TypeVar: - return - - if is_none_type(type_): - yield none_validator - return - if type_ is Pattern or type_ is re.Pattern: - yield pattern_validator - return - if type_ is Hashable or type_ is CollectionsHashable: - yield hashable_validator - return - if is_callable_type(type_): - yield callable_validator - return - if is_literal_type(type_): - yield make_literal_validator(type_) - return - if is_builtin_dataclass(type_): - yield from make_dataclass_validator(type_, config) - return - if type_ is Enum: - yield enum_validator - return - if type_ is IntEnum: - yield int_enum_validator - return - if is_namedtuple(type_): - yield tuple_validator - yield make_namedtuple_validator(type_, config) - return - if is_typeddict(type_): - yield make_typeddict_validator(type_, config) - return - - class_ = get_class(type_) - if class_ is not None: - if class_ is not Any and isinstance(class_, type): - yield make_class_validator(class_) - else: - yield any_class_validator - return - - for val_type, validators in _VALIDATORS: - try: - if issubclass(type_, val_type): - for v in validators: - if isinstance(v, IfConfig): - if v.check(config): - yield v.validator - else: - yield v - return - except TypeError: - raise RuntimeError(f'error checking inheritance of {type_!r} (type: {display_as_type(type_)})') - - if config.arbitrary_types_allowed: - yield make_arbitrary_type_validator(type_) - else: - if hasattr(type_, '__pydantic_core_schema__'): - warn(f'Mixing V1 and V2 models is not supported. `{type_.__name__}` is a V2 model.', UserWarning) - raise RuntimeError(f'no validator found for {type_}, see `arbitrary_types_allowed` in Config') diff --git a/functions/python/data-processor/pydantic/v1/version.py b/functions/python/data-processor/pydantic/v1/version.py deleted file mode 100644 index 6ba1cb083..000000000 --- a/functions/python/data-processor/pydantic/v1/version.py +++ /dev/null @@ -1,38 +0,0 @@ -__all__ = 'compiled', 'VERSION', 'version_info' - -VERSION = '1.10.26' - -try: - import cython # type: ignore -except ImportError: - compiled: bool = False -else: # pragma: no cover - try: - compiled = cython.compiled - except AttributeError: - compiled = False - - -def version_info() -> str: - import platform - import sys - from importlib import import_module - from pathlib import Path - - optional_deps = [] - for p in ('devtools', 'dotenv', 'email-validator', 'typing-extensions'): - try: - import_module(p.replace('-', '_')) - except ImportError: - continue - optional_deps.append(p) - - info = { - 'pydantic version': VERSION, - 'pydantic compiled': compiled, - 'install path': Path(__file__).resolve().parent, - 'python version': sys.version, - 'platform': platform.platform(), - 'optional deps. installed': optional_deps, - } - return '\n'.join('{:>30} {}'.format(k + ':', str(v).replace('\n', ' ')) for k, v in info.items()) diff --git a/functions/python/data-processor/pydantic/validate_call_decorator.py b/functions/python/data-processor/pydantic/validate_call_decorator.py deleted file mode 100644 index 4bc899146..000000000 --- a/functions/python/data-processor/pydantic/validate_call_decorator.py +++ /dev/null @@ -1,116 +0,0 @@ -"""Decorator for validating function calls.""" - -from __future__ import annotations as _annotations - -import inspect -from functools import partial -from types import BuiltinFunctionType -from typing import TYPE_CHECKING, Any, Callable, TypeVar, cast, overload - -from ._internal import _generate_schema, _typing_extra, _validate_call -from .errors import PydanticUserError - -__all__ = ('validate_call',) - -if TYPE_CHECKING: - from .config import ConfigDict - - AnyCallableT = TypeVar('AnyCallableT', bound=Callable[..., Any]) - - -_INVALID_TYPE_ERROR_CODE = 'validate-call-type' - - -def _check_function_type(function: object) -> None: - """Check if the input function is a supported type for `validate_call`.""" - if isinstance(function, _generate_schema.VALIDATE_CALL_SUPPORTED_TYPES): - try: - _typing_extra.signature_no_eval(cast(_generate_schema.ValidateCallSupportedTypes, function)) - except (ValueError, TypeError): - raise PydanticUserError( - f"Input function `{function}` doesn't have a valid signature", code=_INVALID_TYPE_ERROR_CODE - ) - - if isinstance(function, partial): - try: - assert not isinstance(partial.func, partial), 'Partial of partial' - _check_function_type(function.func) - except PydanticUserError as e: - raise PydanticUserError( - f'Partial of `{function.func}` is invalid because the type of `{function.func}` is not supported by `validate_call`', - code=_INVALID_TYPE_ERROR_CODE, - ) from e - - return - - if isinstance(function, BuiltinFunctionType): - raise PydanticUserError(f'Input built-in function `{function}` is not supported', code=_INVALID_TYPE_ERROR_CODE) - if isinstance(function, (classmethod, staticmethod, property)): - name = type(function).__name__ - raise PydanticUserError( - f'The `@{name}` decorator should be applied after `@validate_call` (put `@{name}` on top)', - code=_INVALID_TYPE_ERROR_CODE, - ) - - if inspect.isclass(function): - raise PydanticUserError( - f'Unable to validate {function}: `validate_call` should be applied to functions, not classes (put `@validate_call` on top of `__init__` or `__new__` instead)', - code=_INVALID_TYPE_ERROR_CODE, - ) - if callable(function): - raise PydanticUserError( - f'Unable to validate {function}: `validate_call` should be applied to functions, not instances or other callables. Use `validate_call` explicitly on `__call__` instead.', - code=_INVALID_TYPE_ERROR_CODE, - ) - - raise PydanticUserError( - f'Unable to validate {function}: `validate_call` should be applied to one of the following: function, method, partial, or lambda', - code=_INVALID_TYPE_ERROR_CODE, - ) - - -@overload -def validate_call( - *, config: ConfigDict | None = None, validate_return: bool = False -) -> Callable[[AnyCallableT], AnyCallableT]: ... - - -@overload -def validate_call(func: AnyCallableT, /) -> AnyCallableT: ... - - -def validate_call( - func: AnyCallableT | None = None, - /, - *, - config: ConfigDict | None = None, - validate_return: bool = False, -) -> AnyCallableT | Callable[[AnyCallableT], AnyCallableT]: - """!!! abstract "Usage Documentation" - [Validation Decorator](../concepts/validation_decorator.md) - - Returns a decorated wrapper around the function that validates the arguments and, optionally, the return value. - - Usage may be either as a plain decorator `@validate_call` or with arguments `@validate_call(...)`. - - Args: - func: The function to be decorated. - config: The configuration dictionary. - validate_return: Whether to validate the return value. - - Returns: - The decorated function. - """ - parent_namespace = _typing_extra.parent_frame_namespace() - - def validate(function: AnyCallableT) -> AnyCallableT: - _check_function_type(function) - validate_call_wrapper = _validate_call.ValidateCallWrapper( - cast(_generate_schema.ValidateCallSupportedTypes, function), config, validate_return, parent_namespace - ) - return _validate_call.update_wrapper_attributes(function, validate_call_wrapper.__call__) # type: ignore - - if func is not None: - return validate(func) - else: - return validate diff --git a/functions/python/data-processor/pydantic/validators.py b/functions/python/data-processor/pydantic/validators.py deleted file mode 100644 index 7921b04f0..000000000 --- a/functions/python/data-processor/pydantic/validators.py +++ /dev/null @@ -1,5 +0,0 @@ -"""The `validators` module is a backport module from V1.""" - -from ._migration import getattr_migration - -__getattr__ = getattr_migration(__name__) diff --git a/functions/python/data-processor/pydantic/version.py b/functions/python/data-processor/pydantic/version.py deleted file mode 100644 index f17ae9c80..000000000 --- a/functions/python/data-processor/pydantic/version.py +++ /dev/null @@ -1,113 +0,0 @@ -"""The `version` module holds the version information for Pydantic.""" - -from __future__ import annotations as _annotations - -import sys - -from pydantic_core import __version__ as __pydantic_core_version__ - -__all__ = 'VERSION', 'version_info' - -VERSION = '2.13.4' -"""The version of Pydantic. - -This version specifier is guaranteed to be compliant with the [specification], -introduced by [PEP 440]. - -[specification]: https://packaging.python.org/en/latest/specifications/version-specifiers/ -[PEP 440]: https://peps.python.org/pep-0440/ -""" - -# Keep this in sync with the version constraint in the `pyproject.toml` dependencies: -_COMPATIBLE_PYDANTIC_CORE_VERSION = '2.46.4' - - -def version_short() -> str: - """Return the `major.minor` part of Pydantic version. - - It returns '2.1' if Pydantic version is '2.1.1'. - """ - return '.'.join(VERSION.split('.')[:2]) - - -def version_info() -> str: - """Return complete version information for Pydantic and its dependencies.""" - import importlib.metadata - import platform - from pathlib import Path - - import pydantic_core._pydantic_core as pdc - - from ._internal import _git as git - - # get data about packages that are closely related to pydantic, use pydantic or often conflict with pydantic - package_names = { - 'email-validator', - 'fastapi', - 'mypy', - 'pydantic-extra-types', - 'pydantic-settings', - 'pyright', - 'typing_extensions', - } - related_packages = [] - - for dist in importlib.metadata.distributions(): - name = dist.metadata['Name'] - if name in package_names: - related_packages.append(f'{name}-{dist.version}') - - pydantic_dir = Path(__file__).parents[1].resolve() - most_recent_commit = ( - git.git_revision(pydantic_dir) if git.is_git_repo(pydantic_dir) and git.have_git() else 'unknown' - ) - - info = { - 'pydantic version': VERSION, - 'pydantic-core version': __pydantic_core_version__, - 'pydantic-core build': getattr(pdc, 'build_info', None) or pdc.build_profile, # pyright: ignore[reportPrivateImportUsage] - 'python version': sys.version, - 'platform': platform.platform(), - 'related packages': ' '.join(related_packages), - 'commit': most_recent_commit, - } - return '\n'.join('{:>30} {}'.format(k + ':', str(v).replace('\n', ' ')) for k, v in info.items()) - - -def check_pydantic_core_version() -> bool: - """Check that the installed `pydantic-core` dependency is compatible.""" - return __pydantic_core_version__ == _COMPATIBLE_PYDANTIC_CORE_VERSION - - -def _ensure_pydantic_core_version() -> None: # pragma: no cover - if not check_pydantic_core_version(): - raise_error = True - # Do not raise the error if pydantic is installed in editable mode (i.e. in development): - if sys.version_info >= (3, 13): # origin property added in 3.13 - from importlib.metadata import distribution - - dist = distribution('pydantic') - if getattr(getattr(dist.origin, 'dir_info', None), 'editable', False): - raise_error = False - - if raise_error: - raise SystemError( - f'The installed pydantic-core version ({__pydantic_core_version__}) is incompatible ' - f'with the current pydantic version, which requires {_COMPATIBLE_PYDANTIC_CORE_VERSION}. ' - "If you encounter this error, make sure that you haven't upgraded pydantic-core manually." - ) - - -def parse_mypy_version(version: str) -> tuple[int, int, int]: - """Parse `mypy` string version to a 3-tuple of ints. - - It parses normal version like `1.11.0` and extra info followed by a `+` sign - like `1.11.0+dev.d6d9d8cd4f27c52edac1f537e236ec48a01e54cb.dirty`. - - Args: - version: The mypy version string. - - Returns: - A triple of ints, e.g. `(1, 11, 0)`. - """ - return tuple(map(int, version.partition('+')[0].split('.'))) # pyright: ignore[reportReturnType] diff --git a/functions/python/data-processor/pydantic/warnings.py b/functions/python/data-processor/pydantic/warnings.py deleted file mode 100644 index 02323b8da..000000000 --- a/functions/python/data-processor/pydantic/warnings.py +++ /dev/null @@ -1,122 +0,0 @@ -"""Pydantic-specific warnings.""" - -from __future__ import annotations as _annotations - -from .version import version_short - -__all__ = ( - 'PydanticDeprecatedSince20', - 'PydanticDeprecatedSince26', - 'PydanticDeprecatedSince29', - 'PydanticDeprecatedSince210', - 'PydanticDeprecatedSince211', - 'PydanticDeprecatedSince212', - 'PydanticDeprecationWarning', - 'PydanticExperimentalWarning', - 'ArbitraryTypeWarning', - 'UnsupportedFieldAttributeWarning', - 'TypedDictExtraConfigWarning', -) - - -class PydanticDeprecationWarning(DeprecationWarning): - """A Pydantic specific deprecation warning. - - This warning is raised when using deprecated functionality in Pydantic. It provides information on when the - deprecation was introduced and the expected version in which the corresponding functionality will be removed. - - Attributes: - message: Description of the warning. - since: Pydantic version in what the deprecation was introduced. - expected_removal: Pydantic version in what the corresponding functionality expected to be removed. - """ - - message: str - since: tuple[int, int] - expected_removal: tuple[int, int] - - def __init__( - self, message: str, *args: object, since: tuple[int, int], expected_removal: tuple[int, int] | None = None - ) -> None: - super().__init__(message, *args) - self.message = message.rstrip('.') - self.since = since - self.expected_removal = expected_removal if expected_removal is not None else (since[0] + 1, 0) - - def __str__(self) -> str: - message = ( - f'{self.message}. Deprecated in Pydantic V{self.since[0]}.{self.since[1]}' - f' to be removed in V{self.expected_removal[0]}.{self.expected_removal[1]}.' - ) - if self.since == (2, 0): - message += f' See Pydantic V2 Migration Guide at https://errors.pydantic.dev/{version_short()}/migration/' - return message - - -class PydanticDeprecatedSince20(PydanticDeprecationWarning): - """A specific `PydanticDeprecationWarning` subclass defining functionality deprecated since Pydantic 2.0.""" - - def __init__(self, message: str, *args: object) -> None: - super().__init__(message, *args, since=(2, 0), expected_removal=(3, 0)) - - -class PydanticDeprecatedSince26(PydanticDeprecationWarning): # pragma: no cover - """A specific `PydanticDeprecationWarning` subclass defining functionality deprecated since Pydantic 2.6.""" - - def __init__(self, message: str, *args: object) -> None: - super().__init__(message, *args, since=(2, 6), expected_removal=(3, 0)) - - -class PydanticDeprecatedSince29(PydanticDeprecationWarning): - """A specific `PydanticDeprecationWarning` subclass defining functionality deprecated since Pydantic 2.9.""" - - def __init__(self, message: str, *args: object) -> None: - super().__init__(message, *args, since=(2, 9), expected_removal=(3, 0)) - - -class PydanticDeprecatedSince210(PydanticDeprecationWarning): - """A specific `PydanticDeprecationWarning` subclass defining functionality deprecated since Pydantic 2.10.""" - - def __init__(self, message: str, *args: object) -> None: - super().__init__(message, *args, since=(2, 10), expected_removal=(3, 0)) - - -class PydanticDeprecatedSince211(PydanticDeprecationWarning): - """A specific `PydanticDeprecationWarning` subclass defining functionality deprecated since Pydantic 2.11.""" - - def __init__(self, message: str, *args: object) -> None: - super().__init__(message, *args, since=(2, 11), expected_removal=(3, 0)) - - -class PydanticDeprecatedSince212(PydanticDeprecationWarning): - """A specific `PydanticDeprecationWarning` subclass defining functionality deprecated since Pydantic 2.12.""" - - def __init__(self, message: str, *args: object) -> None: - super().__init__(message, *args, since=(2, 12), expected_removal=(3, 0)) - - -class GenericBeforeBaseModelWarning(Warning): - pass - - -class PydanticExperimentalWarning(Warning): - """A Pydantic specific experimental functionality warning. - - It is raised to warn users that the functionality may change or be removed in future versions of Pydantic. - """ - - -class CoreSchemaGenerationWarning(UserWarning): - """A warning raised during core schema generation.""" - - -class ArbitraryTypeWarning(CoreSchemaGenerationWarning): - """A warning raised when Pydantic fails to generate a core schema for an arbitrary type.""" - - -class UnsupportedFieldAttributeWarning(CoreSchemaGenerationWarning): - """A warning raised when a `Field()` attribute isn't supported in the context it is used.""" - - -class TypedDictExtraConfigWarning(CoreSchemaGenerationWarning): - """A warning raised when the [`extra`][pydantic.ConfigDict.extra] configuration is incompatible with the `closed` or `extra_items` specification.""" diff --git a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/INSTALLER b/functions/python/data-processor/pydantic_core-2.46.4.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/METADATA b/functions/python/data-processor/pydantic_core-2.46.4.dist-info/METADATA deleted file mode 100644 index 29068a781..000000000 --- a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/METADATA +++ /dev/null @@ -1,173 +0,0 @@ -Metadata-Version: 2.4 -Name: pydantic_core -Version: 2.46.4 -Classifier: Development Status :: 3 - Alpha -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Programming Language :: Python :: Implementation :: GraalPy -Classifier: Programming Language :: Rust -Classifier: Framework :: Pydantic -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: Information Technology -Classifier: Operating System :: POSIX :: Linux -Classifier: Operating System :: Microsoft :: Windows -Classifier: Operating System :: MacOS -Classifier: Typing :: Typed -Requires-Dist: typing-extensions>=4.14.1 -License-File: LICENSE -Summary: Core functionality for Pydantic validation and serialization -Home-Page: https://github.com/pydantic/pydantic -Author-email: Samuel Colvin , Adrian Garcia Badaracco <1755071+adriangb@users.noreply.github.com>, David Montague , David Hewitt , Sydney Runkle , Victorien Plot -License-Expression: MIT -Requires-Python: >=3.9 -Description-Content-Type: text/markdown; charset=UTF-8; variant=GFM -Project-URL: Funding, https://github.com/sponsors/samuelcolvin -Project-URL: Homepage, https://github.com/pydantic -Project-URL: Source, https://github.com/pydantic/pydantic/tree/main/pydantic-core - -# pydantic-core - -[![CI](https://github.com/pydantic/pydantic-core/workflows/ci/badge.svg?event=push)](https://github.com/pydantic/pydantic-core/actions?query=event%3Apush+branch%3Amain+workflow%3Aci) -[![Coverage](https://codecov.io/gh/pydantic/pydantic-core/branch/main/graph/badge.svg)](https://codecov.io/gh/pydantic/pydantic-core) -[![pypi](https://img.shields.io/pypi/v/pydantic-core.svg)](https://pypi.python.org/pypi/pydantic-core) -[![versions](https://img.shields.io/pypi/pyversions/pydantic-core.svg)](https://github.com/pydantic/pydantic-core) -[![license](https://img.shields.io/github/license/pydantic/pydantic-core.svg)](https://github.com/pydantic/pydantic-core/blob/main/LICENSE) - -This package provides the core functionality for [pydantic](https://docs.pydantic.dev) validation and serialization. - -Pydantic-core is currently around 17x faster than pydantic V1. -See [`tests/benchmarks/`](./tests/benchmarks/) for details. - -## Example of direct usage - -*NOTE: You should not need to use pydantic-core directly; instead, use pydantic, which in turn uses pydantic-core.* - -```py -from pydantic_core import SchemaValidator, ValidationError - - -v = SchemaValidator( - { - 'type': 'typed-dict', - 'fields': { - 'name': { - 'type': 'typed-dict-field', - 'schema': { - 'type': 'str', - }, - }, - 'age': { - 'type': 'typed-dict-field', - 'schema': { - 'type': 'int', - 'ge': 18, - }, - }, - 'is_developer': { - 'type': 'typed-dict-field', - 'schema': { - 'type': 'default', - 'schema': {'type': 'bool'}, - 'default': True, - }, - }, - }, - } -) - -r1 = v.validate_python({'name': 'Samuel', 'age': 35}) -assert r1 == {'name': 'Samuel', 'age': 35, 'is_developer': True} - -# pydantic-core can also validate JSON directly -r2 = v.validate_json('{"name": "Samuel", "age": 35}') -assert r1 == r2 - -try: - v.validate_python({'name': 'Samuel', 'age': 11}) -except ValidationError as e: - print(e) - """ - 1 validation error for model - age - Input should be greater than or equal to 18 - [type=greater_than_equal, context={ge: 18}, input_value=11, input_type=int] - """ -``` - -## Getting Started - -### Prerequisites - -You'll need: - -1. **[Rust](https://rustup.rs/)** - Rust stable (or nightly for coverage) -2. **[uv](https://docs.astral.sh/uv/getting-started/installation/)** - Fast Python package manager (will install Python 3.9+ automatically) -3. **[git](https://git-scm.com/)** - For version control -4. **[make](https://www.gnu.org/software/make/)** - For running development commands (or use `nmake` on Windows) - -### Quick Start - -```bash -# Clone the repository (or from your fork) -git clone git@github.com:pydantic/pydantic-core.git -cd pydantic-core - -# Install all dependencies using uv, setup pre-commit hooks, and build the development version -make install -``` - -Verify your installation by running: - -```bash -make -``` - -This runs a full development cycle: formatting, building, linting, and testing - -### Development Commands - -Run `make help` to see all available commands, or use these common ones: - -```bash -make build-dev # to build the package during development -make build-prod # to perform an optimised build for benchmarking -make test # to run the tests -make testcov # to run the tests and generate a coverage report -make lint # to run the linter -make format # to format python and rust code -make all # to run to run build-dev + format + lint + test -``` - -### Useful Resources - -* [`python/pydantic_core/_pydantic_core.pyi`](./python/pydantic_core/_pydantic_core.pyi) - Python API types -* [`python/pydantic_core/core_schema.py`](./python/pydantic_core/core_schema.py) - Core schema definitions -* [`tests/`](./tests) - Comprehensive usage examples - -## Profiling - -It's possible to profile the code using the [`flamegraph` utility from `flamegraph-rs`](https://github.com/flamegraph-rs/flamegraph). (Tested on Linux.) You can install this with `cargo install flamegraph`. - -Run `make build-profiling` to install a release build with debugging symbols included (needed for profiling). - -Once that is built, you can profile pytest benchmarks with (e.g.): - -```bash -flamegraph -- pytest tests/benchmarks/test_micro_benchmarks.py -k test_list_of_ints_core_py --benchmark-enable -``` - -The `flamegraph` command will produce an interactive SVG at `flamegraph.svg`. - -## Releasing - -TBC (needs to be integrated into `pydantic` repository release process). - diff --git a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/RECORD b/functions/python/data-processor/pydantic_core-2.46.4.dist-info/RECORD deleted file mode 100644 index aadb44f88..000000000 --- a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/RECORD +++ /dev/null @@ -1,13 +0,0 @@ -pydantic_core-2.46.4.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -pydantic_core-2.46.4.dist-info/METADATA,sha256=E1-osviUc-kZHHB4whZOPsbIknphwjecL93MB_EfO64,6573 -pydantic_core-2.46.4.dist-info/RECORD,, -pydantic_core-2.46.4.dist-info/WHEEL,sha256=yWybxgfsjm9L5AUC6WkdCoZOAE6STftckMNcF-q9VmU,105 -pydantic_core-2.46.4.dist-info/licenses/LICENSE,sha256=Kv3TDVS01itvSIprzBVG6E7FBh8T9CCcA9ASNIeDeVo,1080 -pydantic_core-2.46.4.dist-info/sboms/pydantic-core.cyclonedx.json,sha256=nYED6PJlqCgHq0UKYXi4T95RcBUPwwE1TR31JzOX-gk,125379 -pydantic_core/__init__.py,sha256=nK1ikrdSVK9gapcKrpv_blrp8LCAic1jrK-jkbYHlNI,5115 -pydantic_core/__pycache__/__init__.cpython-314.pyc,, -pydantic_core/__pycache__/core_schema.cpython-314.pyc,, -pydantic_core/_pydantic_core.cpython-314-darwin.so,sha256=YmmEKmPWRQRtn63tCSPDPSeJLw44bD5aUjTHU1YV8xk,4169712 -pydantic_core/_pydantic_core.pyi,sha256=H_lG8iKiv3Bpfd3rixY7xzs67gE-s0eiAqJaSbnF8Ss,45932 -pydantic_core/core_schema.py,sha256=1PM31yf_kGpm_H7GLnNFRkIPAQGPL9zh6rf2SxohVxg,155574 -pydantic_core/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 diff --git a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/WHEEL b/functions/python/data-processor/pydantic_core-2.46.4.dist-info/WHEEL deleted file mode 100644 index eac7f3b25..000000000 --- a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: maturin (1.13.1) -Root-Is-Purelib: false -Tag: cp314-cp314-macosx_11_0_arm64 diff --git a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/licenses/LICENSE b/functions/python/data-processor/pydantic_core-2.46.4.dist-info/licenses/LICENSE deleted file mode 100644 index 0716871ca..000000000 --- a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2022 Samuel Colvin - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/sboms/pydantic-core.cyclonedx.json b/functions/python/data-processor/pydantic_core-2.46.4.dist-info/sboms/pydantic-core.cyclonedx.json deleted file mode 100644 index 324b563f8..000000000 --- a/functions/python/data-processor/pydantic_core-2.46.4.dist-info/sboms/pydantic-core.cyclonedx.json +++ /dev/null @@ -1,3961 +0,0 @@ -{ - "bomFormat": "CycloneDX", - "specVersion": "1.5", - "version": 1, - "serialNumber": "urn:uuid:ec9b5c39-09d0-4f3a-8c03-1dfa04347daf", - "metadata": { - "timestamp": "2026-05-06T08:30:57.343865000Z", - "tools": [ - { - "vendor": "CycloneDX", - "name": "cargo-cyclonedx", - "version": "0.5.9" - } - ], - "component": { - "type": "library", - "bom-ref": "path+file:///Users/runner/work/pydantic/pydantic/pydantic-core#2.46.4", - "name": "pydantic-core", - "version": "2.46.4", - "scope": "required", - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/pydantic-core@2.46.4?download_url=file://.", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pydantic/pydantic" - }, - { - "type": "vcs", - "url": "https://github.com/pydantic/pydantic.git" - } - ], - "components": [ - { - "type": "library", - "bom-ref": "path+file:///Users/runner/work/pydantic/pydantic/pydantic-core#2.46.4 bin-target-0", - "name": "_pydantic_core", - "version": "2.46.4", - "purl": "pkg:cargo/pydantic-core@2.46.4?download_url=file://.#src/lib.rs" - } - ] - }, - "properties": [ - { - "name": "cdx:rustc:sbom:target:all_targets", - "value": "true" - } - ] - }, - "components": [ - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#ahash@0.8.12", - "author": "Tom Kaitchuck ", - "name": "ahash", - "version": "0.8.12", - "description": "A non-cryptographic hash function using AES-NI for high performance", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/ahash@0.8.12", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/ahash" - }, - { - "type": "vcs", - "url": "https://github.com/tkaitchuck/ahash" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.3", - "author": "Andrew Gallant ", - "name": "aho-corasick", - "version": "1.1.3", - "description": "Fast multiple substring searching.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" - } - ], - "licenses": [ - { - "expression": "Unlicense OR MIT" - } - ], - "purl": "pkg:cargo/aho-corasick@1.1.3", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/BurntSushi/aho-corasick" - }, - { - "type": "vcs", - "url": "https://github.com/BurntSushi/aho-corasick" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#allocator-api2@0.2.21", - "author": "Zakarum ", - "name": "allocator-api2", - "version": "0.2.21", - "description": "Mirror of Rust's allocator API", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/allocator-api2@0.2.21", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/allocator-api2" - }, - { - "type": "website", - "url": "https://github.com/zakarumych/allocator-api2" - }, - { - "type": "vcs", - "url": "https://github.com/zakarumych/allocator-api2" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.3.0", - "author": "Josh Stone ", - "name": "autocfg", - "version": "1.3.0", - "description": "Automatic cfg for Rust compiler features", - "scope": "excluded", - "hashes": [ - { - "alg": "SHA-256", - "content": "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/autocfg@1.3.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/autocfg/" - }, - { - "type": "vcs", - "url": "https://github.com/cuviper/autocfg" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", - "author": "Marshall Pierce ", - "name": "base64", - "version": "0.22.1", - "description": "encodes and decodes base64 as bytes or utf8", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/base64@0.22.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/base64" - }, - { - "type": "vcs", - "url": "https://github.com/marshallpierce/rust-base64" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.9.1", - "author": "The Rust Project Developers", - "name": "bitflags", - "version": "2.9.1", - "description": "A macro to generate structures which behave like bitflags. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/bitflags@2.9.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/bitflags" - }, - { - "type": "website", - "url": "https://github.com/bitflags/bitflags" - }, - { - "type": "vcs", - "url": "https://github.com/bitflags/bitflags" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bitvec@1.0.1", - "name": "bitvec", - "version": "1.0.1", - "description": "Addresses memory by bits, for packed collections and bitfields", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1bc2832c24239b0141d5674bb9174f9d68a8b5b3f2753311927c172ca46f7e9c" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/bitvec@1.0.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/bitvec/latest/bitvec" - }, - { - "type": "website", - "url": "https://bitvecto-rs.github.io/bitvec" - }, - { - "type": "vcs", - "url": "https://github.com/bitvecto-rs/bitvec" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#bumpalo@3.19.0", - "author": "Nick Fitzgerald ", - "name": "bumpalo", - "version": "3.19.0", - "description": "A fast bump allocation arena for Rust.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/bumpalo@3.19.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/bumpalo" - }, - { - "type": "vcs", - "url": "https://github.com/fitzgen/bumpalo" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cc@1.0.101", - "author": "Alex Crichton ", - "name": "cc", - "version": "1.0.101", - "description": "A build-time dependency for Cargo build scripts to assist in invoking the native C compiler to compile native C code into a static archive to be linked into Rust code. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "ac367972e516d45567c7eafc73d24e1c193dcf200a8d94e9db7b3d38b349572d" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/cc@1.0.101", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/cc" - }, - { - "type": "website", - "url": "https://github.com/rust-lang/cc-rs" - }, - { - "type": "vcs", - "url": "https://github.com/rust-lang/cc-rs" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.0", - "author": "Alex Crichton ", - "name": "cfg-if", - "version": "1.0.0", - "description": "A macro to ergonomically define an item depending on a large number of #[cfg] parameters. Structured like an if-else chain, the first matching branch is the item that gets emitted. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/cfg-if@1.0.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/cfg-if" - }, - { - "type": "website", - "url": "https://github.com/alexcrichton/cfg-if" - }, - { - "type": "vcs", - "url": "https://github.com/alexcrichton/cfg-if" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "author": "Jane Lusby ", - "name": "displaydoc", - "version": "0.2.5", - "description": "A derive macro for implementing the display Trait via a doc comment and string interpolation ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/displaydoc@0.2.5", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/displaydoc" - }, - { - "type": "website", - "url": "https://github.com/yaahc/displaydoc" - }, - { - "type": "vcs", - "url": "https://github.com/yaahc/displaydoc" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#enum_dispatch@0.3.13", - "author": "Anton Lazarev ", - "name": "enum_dispatch", - "version": "0.3.13", - "description": "Near drop-in replacement for dynamic-dispatched method calls with up to 10x the speed", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "aa18ce2bc66555b3218614519ac839ddb759a7d6720732f979ef8d13be147ecd" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/enum_dispatch@0.3.13", - "externalReferences": [ - { - "type": "vcs", - "url": "https://gitlab.com/antonok/enum_dispatch" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#equivalent@1.0.2", - "name": "equivalent", - "version": "1.0.2", - "description": "Traits for key comparison in maps.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/equivalent@1.0.2", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/indexmap-rs/equivalent" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#foldhash@0.2.0", - "author": "Orson Peters ", - "name": "foldhash", - "version": "0.2.0", - "description": "A fast, non-cryptographic, minimally DoS-resistant hashing algorithm.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" - } - ], - "licenses": [ - { - "expression": "Zlib" - } - ], - "purl": "pkg:cargo/foldhash@0.2.0", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/orlp/foldhash" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#form_urlencoded@1.2.2", - "author": "The rust-url developers", - "name": "form_urlencoded", - "version": "1.2.2", - "description": "Parser and serializer for the application/x-www-form-urlencoded syntax, as used by HTML forms.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/form_urlencoded@1.2.2", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/servo/rust-url" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#funty@2.0.0", - "author": "myrrlyn ", - "name": "funty", - "version": "2.0.0", - "description": "Trait generalization over the primitive types", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "e6d5a32815ae3f33302d95fdcb2ce17862f8c65363dcfd29360480ba1001fc9c" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/funty@2.0.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/funty" - }, - { - "type": "vcs", - "url": "https://github.com/myrrlyn/funty" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.3.3", - "author": "The Rand Project Developers", - "name": "getrandom", - "version": "0.3.3", - "description": "A small cross-platform library for retrieving random data from system source", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/getrandom@0.3.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/getrandom" - }, - { - "type": "vcs", - "url": "https://github.com/rust-random/getrandom" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#hashbrown@0.16.1", - "author": "Amanieu d'Antras ", - "name": "hashbrown", - "version": "0.16.1", - "description": "A Rust port of Google's SwissTable hash map", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/hashbrown@0.16.1", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/rust-lang/hashbrown" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0", - "name": "heck", - "version": "0.5.0", - "description": "heck is a case conversion library.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/heck@0.5.0", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/withoutboats/heck" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#hex@0.4.3", - "author": "KokaKiwi ", - "name": "hex", - "version": "0.4.3", - "description": "Encoding and decoding data into/from hexadecimal representation.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/hex@0.4.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/hex/" - }, - { - "type": "vcs", - "url": "https://github.com/KokaKiwi/rust-hex" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_collections@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_collections", - "version": "1.5.0", - "description": "Collection of API for use in ICU libraries.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "db2fa452206ebee18c4b5c2274dbf1de17008e874b4dc4f0aea9d01ca79e4526" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_collections@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locid@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_locid", - "version": "1.5.0", - "description": "API for managing Unicode Language and Locale Identifiers", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "13acbb8371917fc971be86fc8057c41a64b521c184808a698c02acc242dbf637" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_locid@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locid_transform@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_locid_transform", - "version": "1.5.0", - "description": "API for Unicode Language and Locale Identifiers canonicalization", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "01d11ac35de8e40fdeda00d9e1e9d92525f3f9d887cdd7aa81d727596788b54e" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_locid_transform@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locid_transform_data@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_locid_transform_data", - "version": "1.5.0", - "description": "Data for the icu_locid_transform crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "fdc8ff3388f852bede6b579ad4e978ab004f139284d7b28715f773507b946f6e" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_locid_transform_data@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_normalizer", - "version": "1.5.0", - "description": "API for normalizing text into Unicode Normalization Forms", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "19ce3e0da2ec68599d193c93d088142efd7f9c5d6fc9b803774855747dc6a84f" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_normalizer@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer_data@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_normalizer_data", - "version": "1.5.0", - "description": "Data for the icu_normalizer crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "f8cafbf7aa791e9b22bec55a167906f9e1215fd475cd22adfcf660e03e989516" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_normalizer_data@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties@1.5.1", - "author": "The ICU4X Project Developers", - "name": "icu_properties", - "version": "1.5.1", - "description": "Definitions for Unicode properties", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "93d6020766cfc6302c15dbbc9c8778c37e62c14427cb7f6e601d849e092aeef5" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_properties@1.5.1", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties_data@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_properties_data", - "version": "1.5.0", - "description": "Data for the icu_properties crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "67a8effbc3dd3e4ba1afa8ad918d5684b8868b3b26500753effea8d2eed19569" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_properties_data@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_provider@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_provider", - "version": "1.5.0", - "description": "Trait and struct definitions for the ICU data provider", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "6ed421c8a8ef78d3e2dbc98a973be2f3770cb42b606e3ab18d6237c4dfde68d9" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_provider@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#icu_provider_macros@1.5.0", - "author": "The ICU4X Project Developers", - "name": "icu_provider_macros", - "version": "1.5.0", - "description": "Proc macros for ICU data providers", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1ec89e9337638ecdc08744df490b221a7399bf8d164eb52a665454e60e075ad6" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/icu_provider_macros@1.5.0", - "externalReferences": [ - { - "type": "website", - "url": "https://icu4x.unicode.org" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", - "author": "The rust-url developers", - "name": "idna", - "version": "1.1.0", - "description": "IDNA (Internationalizing Domain Names in Applications) and Punycode.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/idna@1.1.0", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/servo/rust-url/" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#idna_adapter@1.2.0", - "author": "The rust-url developers", - "name": "idna_adapter", - "version": "1.2.0", - "description": "Back end adapter for idna", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "daca1df1c957320b2cf139ac61e7bd64fed304c5040df000a745aa1de3b4ef71" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/idna_adapter@1.2.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/idna_adapter/latest/idna_adapter/" - }, - { - "type": "website", - "url": "https://docs.rs/crate/idna_adapter/latest" - }, - { - "type": "vcs", - "url": "https://github.com/hsivonen/idna_adapter" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.11", - "author": "David Tolnay ", - "name": "itoa", - "version": "1.0.11", - "description": "Fast integer primitive to string conversion", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/itoa@1.0.11", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/itoa" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/itoa" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#jiter@0.14.0", - "author": "Samuel Colvin ", - "name": "jiter", - "version": "0.14.0", - "description": "Fast Iterable JSON parser", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "b6f3b5d3f84b36f4ad09fd1da896d23d9852a1aa86556578dd0289f43dce311d" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/jiter@0.14.0", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pydantic/jiter/" - }, - { - "type": "vcs", - "url": "https://github.com/pydantic/jiter/" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#js-sys@0.3.77", - "author": "The wasm-bindgen Developers", - "name": "js-sys", - "version": "0.3.77", - "description": "Bindings for all JS global objects and functions in all JS environments like Node.js and browsers, built on `#[wasm_bindgen]` using the `wasm-bindgen` crate. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/js-sys@0.3.77", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/js-sys" - }, - { - "type": "website", - "url": "https://rustwasm.github.io/wasm-bindgen/" - }, - { - "type": "vcs", - "url": "https://github.com/rustwasm/wasm-bindgen/tree/master/crates/js-sys" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-float@1.0.5", - "author": "Alex Huszagh ", - "name": "lexical-parse-float", - "version": "1.0.5", - "description": "Efficient parsing of floats from strings.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/lexical-parse-float@1.0.5", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/Alexhuszagh/rust-lexical" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-integer@1.0.5", - "author": "Alex Huszagh ", - "name": "lexical-parse-integer", - "version": "1.0.5", - "description": "Efficient parsing of integers from strings.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/lexical-parse-integer@1.0.5", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/Alexhuszagh/rust-lexical" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lexical-util@1.0.6", - "author": "Alex Huszagh ", - "name": "lexical-util", - "version": "1.0.6", - "description": "Shared utilities for lexical creates.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/lexical-util@1.0.6", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/Alexhuszagh/rust-lexical" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.185", - "author": "The Rust Project Developers", - "name": "libc", - "version": "0.2.185", - "description": "Raw FFI bindings to platform libraries like libc.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/libc@0.2.185", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/rust-lang/libc" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#litemap@0.7.3", - "author": "The ICU4X Project Developers", - "name": "litemap", - "version": "0.7.3", - "description": "A key-value Map implementation based on a flat, sorted Vec.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "643cb0b8d4fcc284004d5fd0d67ccf61dfffadb7f75e1e71bc420f4688a3a704" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/litemap@0.7.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/litemap" - }, - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#log@0.4.27", - "author": "The Rust Project Developers", - "name": "log", - "version": "0.4.27", - "description": "A lightweight logging facade for Rust ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/log@0.4.27", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/log" - }, - { - "type": "vcs", - "url": "https://github.com/rust-lang/log" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#lru@0.16.3", - "author": "Jerome Froelich ", - "name": "lru", - "version": "0.16.3", - "description": "A LRU cache implementation", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "a1dc47f592c06f33f8e3aea9591776ec7c9f9e4124778ff8a3c3b87159f7e593" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/lru@0.16.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/lru/" - }, - { - "type": "website", - "url": "https://github.com/jeromefroe/lru-rs" - }, - { - "type": "vcs", - "url": "https://github.com/jeromefroe/lru-rs.git" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4", - "author": "Andrew Gallant , bluss", - "name": "memchr", - "version": "2.7.4", - "description": "Provides extremely fast (uses SIMD on x86_64, aarch64 and wasm32) routines for 1, 2 or 3 byte search and single substring search. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" - } - ], - "licenses": [ - { - "expression": "Unlicense OR MIT" - } - ], - "purl": "pkg:cargo/memchr@2.7.4", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/memchr/" - }, - { - "type": "website", - "url": "https://github.com/BurntSushi/memchr" - }, - { - "type": "vcs", - "url": "https://github.com/BurntSushi/memchr" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#num-bigint@0.4.6", - "author": "The Rust Project Developers", - "name": "num-bigint", - "version": "0.4.6", - "description": "Big integer implementation for Rust", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/num-bigint@0.4.6", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/num-bigint" - }, - { - "type": "website", - "url": "https://github.com/rust-num/num-bigint" - }, - { - "type": "vcs", - "url": "https://github.com/rust-num/num-bigint" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#num-integer@0.1.46", - "author": "The Rust Project Developers", - "name": "num-integer", - "version": "0.1.46", - "description": "Integer traits and functions", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/num-integer@0.1.46", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/num-integer" - }, - { - "type": "website", - "url": "https://github.com/rust-num/num-integer" - }, - { - "type": "vcs", - "url": "https://github.com/rust-num/num-integer" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", - "author": "The Rust Project Developers", - "name": "num-traits", - "version": "0.2.19", - "description": "Numeric traits for generic mathematics", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/num-traits@0.2.19", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/num-traits" - }, - { - "type": "website", - "url": "https://github.com/rust-num/num-traits" - }, - { - "type": "vcs", - "url": "https://github.com/rust-num/num-traits" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3", - "author": "Aleksey Kladov ", - "name": "once_cell", - "version": "1.21.3", - "description": "Single assignment cells and lazy values.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/once_cell@1.21.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/once_cell" - }, - { - "type": "vcs", - "url": "https://github.com/matklad/once_cell" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", - "author": "The rust-url developers", - "name": "percent-encoding", - "version": "2.3.2", - "description": "Percent encoding and decoding", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/percent-encoding@2.3.2", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/servo/rust-url/" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#portable-atomic@1.6.0", - "name": "portable-atomic", - "version": "1.6.0", - "description": "Portable atomic types including support for 128-bit atomics, atomic float, etc. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "7170ef9988bc169ba16dd36a7fa041e5c4cbeb6a35b76d4c03daded371eae7c0" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/portable-atomic@1.6.0", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/taiki-e/portable-atomic" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "author": "David Tolnay , Alex Crichton ", - "name": "proc-macro2", - "version": "1.0.86", - "description": "A substitute implementation of the compiler's `proc_macro` API to decouple token-based libraries from the procedural macro use case.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "5e719e8df665df0d1c8fbfd238015744736151d4445ec0836b8e628aae103b77" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/proc-macro2@1.0.86", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/proc-macro2" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/proc-macro2" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3", - "author": "PyO3 Project and Contributors ", - "name": "pyo3-build-config", - "version": "0.28.3", - "description": "Build configuration for the PyO3 ecosystem", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "e368e7ddfdeb98c9bca7f8383be1648fd84ab466bf2bc015e94008db6d35611e" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/pyo3-build-config@0.28.3", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pyo3/pyo3" - }, - { - "type": "vcs", - "url": "https://github.com/pyo3/pyo3" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-ffi@0.28.3", - "author": "PyO3 Project and Contributors ", - "name": "pyo3-ffi", - "version": "0.28.3", - "description": "Python-API bindings for the PyO3 ecosystem", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "7f29e10af80b1f7ccaf7f69eace800a03ecd13e883acfacc1e5d0988605f651e" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/pyo3-ffi@0.28.3", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pyo3/pyo3" - }, - { - "type": "other", - "url": "python" - }, - { - "type": "vcs", - "url": "https://github.com/pyo3/pyo3" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros-backend@0.28.3", - "author": "PyO3 Project and Contributors ", - "name": "pyo3-macros-backend", - "version": "0.28.3", - "description": "Code generation for PyO3 package", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "c4cdc218d835738f81c2338f822078af45b4afdf8b2e33cbb5916f108b813acb" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/pyo3-macros-backend@0.28.3", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pyo3/pyo3" - }, - { - "type": "vcs", - "url": "https://github.com/pyo3/pyo3" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros@0.28.3", - "author": "PyO3 Project and Contributors ", - "name": "pyo3-macros", - "version": "0.28.3", - "description": "Proc macros for PyO3 package", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "df6e520eff47c45997d2fc7dd8214b25dd1310918bbb2642156ef66a67f29813" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/pyo3-macros@0.28.3", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pyo3/pyo3" - }, - { - "type": "vcs", - "url": "https://github.com/pyo3/pyo3" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.28.3", - "author": "PyO3 Project and Contributors ", - "name": "pyo3", - "version": "0.28.3", - "description": "Bindings to Python interpreter", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "91fd8e38a3b50ed1167fb981cd6fd60147e091784c427b8f7183a7ee32c31c12" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/pyo3@0.28.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/crate/pyo3/" - }, - { - "type": "website", - "url": "https://github.com/pyo3/pyo3" - }, - { - "type": "vcs", - "url": "https://github.com/pyo3/pyo3" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#python3-dll-a@0.2.14", - "author": "Sergey Kvachonok , messense , Adam Reichold ", - "name": "python3-dll-a", - "version": "0.2.14", - "description": "Standalone python3(y)(t).dll import library generator", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "d381ef313ae70b4da5f95f8a4de773c6aa5cd28f73adec4b4a31df70b66780d8" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/python3-dll-a@0.2.14", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/PyO3/python3-dll-a" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "author": "David Tolnay ", - "name": "quote", - "version": "1.0.44", - "description": "Quasi-quoting macro quote!(...)", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "21b2ebcf727b7760c461f091f9f0f539b77b8e87f2fd88131e7f1b433b3cece4" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/quote@1.0.44", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/quote/" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/quote" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#r-efi@5.2.0", - "name": "r-efi", - "version": "5.2.0", - "description": "UEFI Reference Specification Protocol Constants and Definitions", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0 OR LGPL-2.1-or-later" - } - ], - "purl": "pkg:cargo/r-efi@5.2.0", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/r-efi/r-efi/wiki" - }, - { - "type": "vcs", - "url": "https://github.com/r-efi/r-efi" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#radium@0.7.0", - "author": "Nika Layzell , myrrlyn ", - "name": "radium", - "version": "0.7.0", - "description": "Portable interfaces for maybe-atomic types", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "dc33ff2d4973d518d823d61aa239014831e521c75da58e3df4840d3f47749d09" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/radium@0.7.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/radium" - }, - { - "type": "website", - "url": "https://github.com/bitvecto-rs/radium" - }, - { - "type": "vcs", - "url": "https://github.com/bitvecto-rs/radium" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.13", - "author": "The Rust Project Developers, Andrew Gallant ", - "name": "regex-automata", - "version": "0.4.13", - "description": "Automata construction and matching using regular expressions.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/regex-automata@0.4.13", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/regex-automata" - }, - { - "type": "website", - "url": "https://github.com/rust-lang/regex/tree/master/regex-automata" - }, - { - "type": "vcs", - "url": "https://github.com/rust-lang/regex" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.5", - "author": "The Rust Project Developers, Andrew Gallant ", - "name": "regex-syntax", - "version": "0.8.5", - "description": "A regular expression parser.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/regex-syntax@0.8.5", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/regex-syntax" - }, - { - "type": "vcs", - "url": "https://github.com/rust-lang/regex/tree/master/regex-syntax" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#regex@1.12.3", - "author": "The Rust Project Developers, Andrew Gallant ", - "name": "regex", - "version": "1.12.3", - "description": "An implementation of regular expressions for Rust. This implementation uses finite automata and guarantees linear time matching on all inputs. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/regex@1.12.3", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/regex" - }, - { - "type": "website", - "url": "https://github.com/rust-lang/regex" - }, - { - "type": "vcs", - "url": "https://github.com/rust-lang/regex" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#rustversion@1.0.17", - "author": "David Tolnay ", - "name": "rustversion", - "version": "1.0.17", - "description": "Conditional compilation according to rustc compiler version", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/rustversion@1.0.17", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/rustversion" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/rustversion" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", - "author": "Erick Tryzelaar , David Tolnay ", - "name": "serde", - "version": "1.0.228", - "description": "A generic serialization/deserialization framework", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/serde@1.0.228", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/serde" - }, - { - "type": "website", - "url": "https://serde.rs" - }, - { - "type": "vcs", - "url": "https://github.com/serde-rs/serde" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", - "author": "Erick Tryzelaar , David Tolnay ", - "name": "serde_core", - "version": "1.0.228", - "description": "Serde traits only, with no support for derive -- use the `serde` crate instead", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/serde_core@1.0.228", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/serde_core" - }, - { - "type": "website", - "url": "https://serde.rs" - }, - { - "type": "vcs", - "url": "https://github.com/serde-rs/serde" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228", - "author": "Erick Tryzelaar , David Tolnay ", - "name": "serde_derive", - "version": "1.0.228", - "description": "Macros 1.1 implementation of #[derive(Serialize, Deserialize)]", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/serde_derive@1.0.228", - "externalReferences": [ - { - "type": "documentation", - "url": "https://serde.rs/derive.html" - }, - { - "type": "website", - "url": "https://serde.rs" - }, - { - "type": "vcs", - "url": "https://github.com/serde-rs/serde" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", - "author": "Erick Tryzelaar , David Tolnay ", - "name": "serde_json", - "version": "1.0.149", - "description": "A JSON serialization file format", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/serde_json@1.0.149", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/serde_json" - }, - { - "type": "vcs", - "url": "https://github.com/serde-rs/json" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", - "author": "The Servo Project Developers", - "name": "smallvec", - "version": "1.15.1", - "description": "'Small vector' optimization: store up to a small number of items on the stack", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/smallvec@1.15.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/smallvec/" - }, - { - "type": "vcs", - "url": "https://github.com/servo/rust-smallvec" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#speedate@0.17.0", - "author": "Samuel Colvin ", - "name": "speedate", - "version": "0.17.0", - "description": "Fast and simple datetime, date, time and duration parsing", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "aba069c070b5e213f2a094deb7e5ed50ecb092be36102a4f4042e8d2056d060e" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/speedate@0.17.0", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/pydantic/speedate/" - }, - { - "type": "vcs", - "url": "https://github.com/pydantic/speedate/" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.0", - "author": "Robert Grosse ", - "name": "stable_deref_trait", - "version": "1.2.0", - "description": "An unsafe marker trait for types like Box and Rc that dereference to a stable address even when moved, and hence can be used with libraries such as owning_ref and rental. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/stable_deref_trait@1.2.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/stable_deref_trait/1.2.0/stable_deref_trait" - }, - { - "type": "vcs", - "url": "https://github.com/storyyeller/stable_deref_trait" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0", - "author": "Nikolai Vazquez", - "name": "static_assertions", - "version": "1.1.0", - "description": "Compile-time assertions to ensure that invariants are met.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/static_assertions@1.1.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/static_assertions/" - }, - { - "type": "website", - "url": "https://github.com/nvzqz/static-assertions-rs" - }, - { - "type": "vcs", - "url": "https://github.com/nvzqz/static-assertions-rs" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#strum@0.27.2", - "author": "Peter Glotfelty ", - "name": "strum", - "version": "0.27.2", - "description": "Helpful macros for working with enums and strings", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/strum@0.27.2", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/strum" - }, - { - "type": "website", - "url": "https://github.com/Peternator7/strum" - }, - { - "type": "vcs", - "url": "https://github.com/Peternator7/strum" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#strum_macros@0.27.2", - "author": "Peter Glotfelty ", - "name": "strum_macros", - "version": "0.27.2", - "description": "Helpful macros for working with enums and strings", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/strum_macros@0.27.2", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/strum" - }, - { - "type": "website", - "url": "https://github.com/Peternator7/strum" - }, - { - "type": "vcs", - "url": "https://github.com/Peternator7/strum" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82", - "author": "David Tolnay ", - "name": "syn", - "version": "2.0.82", - "description": "Parser for Rust source code", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "83540f837a8afc019423a8edb95b52a8effe46957ee402287f4292fae35be021" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/syn@2.0.82", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/syn" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/syn" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.1", - "author": "Nika Layzell ", - "name": "synstructure", - "version": "0.13.1", - "description": "Helper methods and macros for custom derives", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "c8af7666ab7b6390ab78131fb5b0fce11d6b7a6951602017c35fa82800708971" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/synstructure@0.13.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/synstructure" - }, - { - "type": "vcs", - "url": "https://github.com/mystor/synstructure" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tap@1.0.1", - "author": "Elliott Linder , myrrlyn ", - "name": "tap", - "version": "1.0.1", - "description": "Generic extensions for tapping values in Rust", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/tap@1.0.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/tap" - }, - { - "type": "website", - "url": "https://github.com/myrrlyn/tap" - }, - { - "type": "vcs", - "url": "https://github.com/myrrlyn/tap" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#target-lexicon@0.13.4", - "author": "Dan Gohman ", - "name": "target-lexicon", - "version": "0.13.4", - "description": "LLVM target triple types", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "b1dd07eb858a2067e2f3c7155d54e929265c264e6f37efe3ee7a8d1b5a1dd0ba" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 WITH LLVM-exception" - } - ], - "purl": "pkg:cargo/target-lexicon@0.13.4", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/target-lexicon/" - }, - { - "type": "vcs", - "url": "https://github.com/bytecodealliance/target-lexicon" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.7.6", - "author": "The ICU4X Project Developers", - "name": "tinystr", - "version": "0.7.6", - "description": "A small ASCII-only bounded length string representation.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "9117f5d4db391c1cf6927e7bea3db74b9a1c1add8f7eda9ffd5364f40f57b82f" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/tinystr@0.7.6", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.12", - "author": "David Tolnay ", - "name": "unicode-ident", - "version": "1.0.12", - "description": "Determine whether characters have the XID_Start or XID_Continue properties according to Unicode Standard Annex #31", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" - } - ], - "licenses": [ - { - "expression": "(MIT OR Apache-2.0) AND Unicode-DFS-2016" - } - ], - "purl": "pkg:cargo/unicode-ident@1.0.12", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/unicode-ident" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/unicode-ident" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", - "author": "The rust-url developers", - "name": "url", - "version": "2.5.8", - "description": "URL library for Rust, based on the WHATWG URL Standard", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "ff67a8a4397373c3ef660812acab3268222035010ab8680ec4215f38ba3d0eed" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/url@2.5.8", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/url" - }, - { - "type": "vcs", - "url": "https://github.com/servo/rust-url" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#utf16_iter@1.0.5", - "author": "Henri Sivonen ", - "name": "utf16_iter", - "version": "1.0.5", - "description": "Iterator by char over potentially-invalid UTF-16 in &[u16]", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/utf16_iter@1.0.5", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/utf16_iter/" - }, - { - "type": "website", - "url": "https://docs.rs/utf16_iter/" - }, - { - "type": "vcs", - "url": "https://github.com/hsivonen/utf16_iter" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4", - "author": "Henri Sivonen ", - "name": "utf8_iter", - "version": "1.0.4", - "description": "Iterator by char over potentially-invalid UTF-8 in &[u8]", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/utf8_iter@1.0.4", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/utf8_iter/" - }, - { - "type": "website", - "url": "https://docs.rs/utf8_iter/" - }, - { - "type": "vcs", - "url": "https://github.com/hsivonen/utf8_iter" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#uuid@1.23.0", - "author": "Ashley Mannix, Dylan DPC, Hunar Roop Kahlon", - "name": "uuid", - "version": "1.23.0", - "description": "A library to generate and parse UUIDs.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "5ac8b6f42ead25368cf5b098aeb3dc8a1a2c05a3eee8a9a1a68c640edbfc79d9" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/uuid@1.23.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/uuid" - }, - { - "type": "website", - "url": "https://github.com/uuid-rs/uuid" - }, - { - "type": "vcs", - "url": "https://github.com/uuid-rs/uuid" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5", - "author": "Sergio Benitez ", - "name": "version_check", - "version": "0.9.5", - "description": "Tiny crate to check the version of the installed/running rustc.", - "scope": "excluded", - "hashes": [ - { - "alg": "SHA-256", - "content": "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/version_check@0.9.5", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/version_check/" - }, - { - "type": "vcs", - "url": "https://github.com/SergioBenitez/version_check" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wasi@0.14.2+wasi-0.2.4", - "author": "The Cranelift Project Developers", - "name": "wasi", - "version": "0.14.2+wasi-0.2.4", - "description": "WASI API bindings for Rust", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/wasi@0.14.2+wasi-0.2.4", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wasi" - }, - { - "type": "vcs", - "url": "https://github.com/bytecodealliance/wasi-rs" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-backend@0.2.100", - "author": "The wasm-bindgen Developers", - "name": "wasm-bindgen-backend", - "version": "0.2.100", - "description": "Backend code generation of the wasm-bindgen tool ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/wasm-bindgen-backend@0.2.100", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wasm-bindgen-backend" - }, - { - "type": "website", - "url": "https://rustwasm.github.io/wasm-bindgen/" - }, - { - "type": "vcs", - "url": "https://github.com/rustwasm/wasm-bindgen/tree/master/crates/backend" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-macro-support@0.2.100", - "author": "The wasm-bindgen Developers", - "name": "wasm-bindgen-macro-support", - "version": "0.2.100", - "description": "The part of the implementation of the `#[wasm_bindgen]` attribute that is not in the shared backend crate ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/wasm-bindgen-macro-support@0.2.100", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wasm-bindgen" - }, - { - "type": "website", - "url": "https://rustwasm.github.io/wasm-bindgen/" - }, - { - "type": "vcs", - "url": "https://github.com/rustwasm/wasm-bindgen/tree/master/crates/macro-support" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-macro@0.2.100", - "author": "The wasm-bindgen Developers", - "name": "wasm-bindgen-macro", - "version": "0.2.100", - "description": "Definition of the `#[wasm_bindgen]` attribute, an internal dependency ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/wasm-bindgen-macro@0.2.100", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wasm-bindgen" - }, - { - "type": "website", - "url": "https://rustwasm.github.io/wasm-bindgen/" - }, - { - "type": "vcs", - "url": "https://github.com/rustwasm/wasm-bindgen/tree/master/crates/macro" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-shared@0.2.100", - "author": "The wasm-bindgen Developers", - "name": "wasm-bindgen-shared", - "version": "0.2.100", - "description": "Shared support between wasm-bindgen and wasm-bindgen cli, an internal dependency. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/wasm-bindgen-shared@0.2.100", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wasm-bindgen-shared" - }, - { - "type": "website", - "url": "https://rustwasm.github.io/wasm-bindgen/" - }, - { - "type": "other", - "url": "wasm_bindgen" - }, - { - "type": "vcs", - "url": "https://github.com/rustwasm/wasm-bindgen/tree/master/crates/shared" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen@0.2.100", - "author": "The wasm-bindgen Developers", - "name": "wasm-bindgen", - "version": "0.2.100", - "description": "Easy support for interacting between JS and Rust. ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" - } - ], - "licenses": [ - { - "expression": "MIT OR Apache-2.0" - } - ], - "purl": "pkg:cargo/wasm-bindgen@0.2.100", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wasm-bindgen" - }, - { - "type": "website", - "url": "https://rustwasm.github.io/" - }, - { - "type": "vcs", - "url": "https://github.com/rustwasm/wasm-bindgen" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wit-bindgen-rt@0.39.0", - "name": "wit-bindgen-rt", - "version": "0.39.0", - "description": "Runtime support for the `wit-bindgen` crate ", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/wit-bindgen-rt@0.39.0", - "externalReferences": [ - { - "type": "website", - "url": "https://github.com/bytecodealliance/wit-bindgen" - }, - { - "type": "vcs", - "url": "https://github.com/bytecodealliance/wit-bindgen" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#write16@1.0.0", - "name": "write16", - "version": "1.0.0", - "description": "A UTF-16 analog of the Write trait", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "d1890f4022759daae28ed4fe62859b1236caebfc61ede2f63ed4e695f3f6d936" - } - ], - "licenses": [ - { - "expression": "Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/write16@1.0.0", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/write16/" - }, - { - "type": "website", - "url": "https://docs.rs/write16/" - }, - { - "type": "vcs", - "url": "https://github.com/hsivonen/write16" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#writeable@0.5.5", - "author": "The ICU4X Project Developers", - "name": "writeable", - "version": "0.5.5", - "description": "A more efficient alternative to fmt::Display", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "1e9df38ee2d2c3c5948ea468a8406ff0db0b29ae1ffde1bcf20ef305bcc95c51" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/writeable@0.5.5", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#wyz@0.5.1", - "author": "myrrlyn ", - "name": "wyz", - "version": "0.5.1", - "description": "myrrlyn’s utility collection", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "05f360fc0b24296329c78fda852a1e9ae82de9cf7b27dae4b7f62f118f77b9ed" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/wyz@0.5.1", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/wyz" - }, - { - "type": "website", - "url": "https://myrrlyn.net/crates/wyz" - }, - { - "type": "vcs", - "url": "https://github.com/myrrlyn/wyz" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#yoke-derive@0.7.4", - "author": "Manish Goregaokar ", - "name": "yoke-derive", - "version": "0.7.4", - "description": "Custom derive for the yoke crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "28cc31741b18cb6f1d5ff12f5b7523e3d6eb0852bbbad19d73905511d9849b95" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/yoke-derive@0.7.4", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#yoke@0.7.4", - "author": "Manish Goregaokar ", - "name": "yoke", - "version": "0.7.4", - "description": "Abstraction allowing borrowed data to be carried along with the backing data it borrows from", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "6c5b1314b079b0930c31e3af543d8ee1757b1951ae1e1565ec704403a7240ca5" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/yoke@0.7.4", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerocopy-derive@0.8.25", - "author": "Joshua Liebow-Feeser , Jack Wrenn ", - "name": "zerocopy-derive", - "version": "0.8.25", - "description": "Custom derive for traits from the zerocopy crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" - } - ], - "licenses": [ - { - "expression": "BSD-2-Clause OR Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/zerocopy-derive@0.8.25", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/google/zerocopy" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerocopy@0.8.25", - "author": "Joshua Liebow-Feeser , Jack Wrenn ", - "name": "zerocopy", - "version": "0.8.25", - "description": "Zerocopy makes zero-cost memory manipulation effortless. We write \"unsafe\" so you don't have to.", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" - } - ], - "licenses": [ - { - "expression": "BSD-2-Clause OR Apache-2.0 OR MIT" - } - ], - "purl": "pkg:cargo/zerocopy@0.8.25", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/google/zerocopy" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom-derive@0.1.4", - "author": "Manish Goregaokar ", - "name": "zerofrom-derive", - "version": "0.1.4", - "description": "Custom derive for the zerofrom crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "0ea7b4a3637ea8669cedf0f1fd5c286a17f3de97b8dd5a70a6c167a1730e63a5" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/zerofrom-derive@0.1.4", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.4", - "author": "Manish Goregaokar ", - "name": "zerofrom", - "version": "0.1.4", - "description": "ZeroFrom trait for constructing", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "91ec111ce797d0e0784a1116d0ddcdbea84322cd79e5d5ad173daeba4f93ab55" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/zerofrom@0.1.4", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec-derive@0.10.3", - "author": "Manish Goregaokar ", - "name": "zerovec-derive", - "version": "0.10.3", - "description": "Custom derive for the zerovec crate", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "6eafa6dfb17584ea3e2bd6e76e0cc15ad7af12b09abdd1ca55961bed9b1063c6" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/zerovec-derive@0.10.3", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4", - "author": "The ICU4X Project Developers", - "name": "zerovec", - "version": "0.10.4", - "description": "Zero-copy vector backed by a byte array", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "aa2b893d79df23bfb12d5461018d408ea19dfafe76c2c7ef6d4eba614f8ff079" - } - ], - "licenses": [ - { - "expression": "Unicode-3.0" - } - ], - "purl": "pkg:cargo/zerovec@0.10.4", - "externalReferences": [ - { - "type": "vcs", - "url": "https://github.com/unicode-org/icu4x" - } - ] - }, - { - "type": "library", - "bom-ref": "registry+https://github.com/rust-lang/crates.io-index#zmij@1.0.6", - "author": "David Tolnay ", - "name": "zmij", - "version": "1.0.6", - "description": "A double-to-string conversion algorithm based on Schubfach and yy", - "scope": "required", - "hashes": [ - { - "alg": "SHA-256", - "content": "aac060176f7020d62c3bcc1cdbcec619d54f48b07ad1963a3f80ce7a0c17755f" - } - ], - "licenses": [ - { - "expression": "MIT" - } - ], - "purl": "pkg:cargo/zmij@1.0.6", - "externalReferences": [ - { - "type": "documentation", - "url": "https://docs.rs/zmij" - }, - { - "type": "vcs", - "url": "https://github.com/dtolnay/zmij" - } - ] - } - ], - "dependencies": [ - { - "ref": "path+file:///Users/runner/work/pydantic/pydantic/pydantic-core#2.46.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#ahash@0.8.12", - "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1", - "registry+https://github.com/rust-lang/crates.io-index#enum_dispatch@0.3.13", - "registry+https://github.com/rust-lang/crates.io-index#hashbrown@0.16.1", - "registry+https://github.com/rust-lang/crates.io-index#hex@0.4.3", - "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", - "registry+https://github.com/rust-lang/crates.io-index#jiter@0.14.0", - "registry+https://github.com/rust-lang/crates.io-index#lru@0.16.3", - "registry+https://github.com/rust-lang/crates.io-index#num-bigint@0.4.6", - "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", - "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", - "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#regex@1.12.3", - "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", - "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", - "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", - "registry+https://github.com/rust-lang/crates.io-index#speedate@0.17.0", - "registry+https://github.com/rust-lang/crates.io-index#strum@0.27.2", - "registry+https://github.com/rust-lang/crates.io-index#strum_macros@0.27.2", - "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", - "registry+https://github.com/rust-lang/crates.io-index#uuid@1.23.0", - "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#ahash@0.8.12", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.0", - "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.3.3", - "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3", - "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5", - "registry+https://github.com/rust-lang/crates.io-index#zerocopy@0.8.25" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#allocator-api2@0.2.21" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.3.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#base64@0.22.1" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.9.1" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#bitvec@1.0.1", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#funty@2.0.0", - "registry+https://github.com/rust-lang/crates.io-index#radium@0.7.0", - "registry+https://github.com/rust-lang/crates.io-index#tap@1.0.1", - "registry+https://github.com/rust-lang/crates.io-index#wyz@0.5.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#bumpalo@3.19.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#cc@1.0.101" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#enum_dispatch@0.3.13", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3", - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#equivalent@1.0.2" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#foldhash@0.2.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#form_urlencoded@1.2.2", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#funty@2.0.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#getrandom@0.3.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.0", - "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.185", - "registry+https://github.com/rust-lang/crates.io-index#r-efi@5.2.0", - "registry+https://github.com/rust-lang/crates.io-index#wasi@0.14.2+wasi-0.2.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#hashbrown@0.16.1", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#allocator-api2@0.2.21", - "registry+https://github.com/rust-lang/crates.io-index#equivalent@1.0.2", - "registry+https://github.com/rust-lang/crates.io-index#foldhash@0.2.0" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#hex@0.4.3" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_collections@1.5.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#yoke@0.7.4", - "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.4", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locid@1.5.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#litemap@0.7.3", - "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.7.6", - "registry+https://github.com/rust-lang/crates.io-index#writeable@0.5.5", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locid_transform@1.5.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#icu_locid@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_locid_transform_data@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_provider@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.7.6", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_locid_transform_data@1.5.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer@1.5.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#icu_collections@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer_data@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_properties@1.5.1", - "registry+https://github.com/rust-lang/crates.io-index#icu_provider@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", - "registry+https://github.com/rust-lang/crates.io-index#utf16_iter@1.0.5", - "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4", - "registry+https://github.com/rust-lang/crates.io-index#write16@1.0.0", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer_data@1.5.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties@1.5.1", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#icu_collections@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_locid_transform@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_properties_data@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_provider@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.7.6", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_properties_data@1.5.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_provider@1.5.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#icu_locid@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_provider_macros@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.0", - "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.7.6", - "registry+https://github.com/rust-lang/crates.io-index#writeable@0.5.5", - "registry+https://github.com/rust-lang/crates.io-index#yoke@0.7.4", - "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.4", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#icu_provider_macros@1.5.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#idna_adapter@1.2.0", - "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1", - "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#idna_adapter@1.2.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#icu_normalizer@1.5.0", - "registry+https://github.com/rust-lang/crates.io-index#icu_properties@1.5.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.11" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#jiter@0.14.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#ahash@0.8.12", - "registry+https://github.com/rust-lang/crates.io-index#bitvec@1.0.1", - "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-float@1.0.5", - "registry+https://github.com/rust-lang/crates.io-index#num-bigint@0.4.6", - "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", - "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#js-sys@0.3.77", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen@0.2.100" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-float@1.0.5", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-integer@1.0.5", - "registry+https://github.com/rust-lang/crates.io-index#lexical-util@1.0.6", - "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-integer@1.0.5", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#lexical-util@1.0.6", - "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#lexical-util@1.0.6", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.185" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#litemap@0.7.3" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#log@0.4.27" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#lru@0.16.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#hashbrown@0.16.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#num-bigint@0.4.6", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#num-integer@0.1.46", - "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#num-integer@0.1.46", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#autocfg@1.3.0" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#portable-atomic@1.6.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.12" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#python3-dll-a@0.2.14", - "registry+https://github.com/rust-lang/crates.io-index#target-lexicon@0.13.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-ffi@0.28.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.185", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros-backend@0.28.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0", - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros@0.28.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros-backend@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#pyo3@0.28.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#libc@0.2.185", - "registry+https://github.com/rust-lang/crates.io-index#num-bigint@0.4.6", - "registry+https://github.com/rust-lang/crates.io-index#num-traits@0.2.19", - "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3", - "registry+https://github.com/rust-lang/crates.io-index#portable-atomic@1.6.0", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-build-config@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-ffi@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#pyo3-macros@0.28.3", - "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#python3-dll-a@0.2.14", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#cc@1.0.101" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#r-efi@5.2.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#radium@0.7.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.13", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.3", - "registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4", - "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.5" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.5" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#regex@1.12.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#aho-corasick@1.1.3", - "registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4", - "registry+https://github.com/rust-lang/crates.io-index#regex-automata@0.4.13", - "registry+https://github.com/rust-lang/crates.io-index#regex-syntax@0.8.5" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#rustversion@1.0.17" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", - "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_derive@1.0.228", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#serde_json@1.0.149", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#itoa@1.0.11", - "registry+https://github.com/rust-lang/crates.io-index#memchr@2.7.4", - "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", - "registry+https://github.com/rust-lang/crates.io-index#serde_core@1.0.228", - "registry+https://github.com/rust-lang/crates.io-index#zmij@1.0.6" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#smallvec@1.15.1" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#speedate@0.17.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#lexical-parse-float@1.0.5", - "registry+https://github.com/rust-lang/crates.io-index#strum@0.27.2", - "registry+https://github.com/rust-lang/crates.io-index#strum_macros@0.27.2" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#static_assertions@1.1.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#strum@0.27.2", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#strum_macros@0.27.2" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#strum_macros@0.27.2", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#heck@0.5.0", - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.12" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.1", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#tap@1.0.1" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#target-lexicon@0.13.4" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#tinystr@0.7.6", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#displaydoc@0.2.5", - "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.12" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#url@2.5.8", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#form_urlencoded@1.2.2", - "registry+https://github.com/rust-lang/crates.io-index#idna@1.1.0", - "registry+https://github.com/rust-lang/crates.io-index#percent-encoding@2.3.2", - "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#utf16_iter@1.0.5" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#utf8_iter@1.0.4" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#uuid@1.23.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#js-sys@0.3.77", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen@0.2.100" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#version_check@0.9.5" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wasi@0.14.2+wasi-0.2.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#wit-bindgen-rt@0.39.0" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-backend@0.2.100", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#bumpalo@3.19.0", - "registry+https://github.com/rust-lang/crates.io-index#log@0.4.27", - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-shared@0.2.100" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-macro-support@0.2.100", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-backend@0.2.100", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-shared@0.2.100" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-macro@0.2.100", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-macro-support@0.2.100" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-shared@0.2.100", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#unicode-ident@1.0.12" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen@0.2.100", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#cfg-if@1.0.0", - "registry+https://github.com/rust-lang/crates.io-index#once_cell@1.21.3", - "registry+https://github.com/rust-lang/crates.io-index#rustversion@1.0.17", - "registry+https://github.com/rust-lang/crates.io-index#wasm-bindgen-macro@0.2.100" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wit-bindgen-rt@0.39.0", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#bitflags@2.9.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#write16@1.0.0" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#writeable@0.5.5" - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#wyz@0.5.1", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#tap@1.0.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#yoke-derive@0.7.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82", - "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#yoke@0.7.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#serde@1.0.228", - "registry+https://github.com/rust-lang/crates.io-index#stable_deref_trait@1.2.0", - "registry+https://github.com/rust-lang/crates.io-index#yoke-derive@0.7.4", - "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zerocopy-derive@0.8.25", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zerocopy@0.8.25", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#zerocopy-derive@0.8.25" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom-derive@0.1.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82", - "registry+https://github.com/rust-lang/crates.io-index#synstructure@0.13.1" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#zerofrom-derive@0.1.4" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec-derive@0.10.3", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#proc-macro2@1.0.86", - "registry+https://github.com/rust-lang/crates.io-index#quote@1.0.44", - "registry+https://github.com/rust-lang/crates.io-index#syn@2.0.82" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zerovec@0.10.4", - "dependsOn": [ - "registry+https://github.com/rust-lang/crates.io-index#yoke@0.7.4", - "registry+https://github.com/rust-lang/crates.io-index#zerofrom@0.1.4", - "registry+https://github.com/rust-lang/crates.io-index#zerovec-derive@0.10.3" - ] - }, - { - "ref": "registry+https://github.com/rust-lang/crates.io-index#zmij@1.0.6" - } - ] -} \ No newline at end of file diff --git a/functions/python/data-processor/pydantic_core/__init__.py b/functions/python/data-processor/pydantic_core/__init__.py deleted file mode 100644 index d5facd161..000000000 --- a/functions/python/data-processor/pydantic_core/__init__.py +++ /dev/null @@ -1,171 +0,0 @@ -from __future__ import annotations - -import sys as _sys -from typing import Any as _Any - -from typing_extensions import Sentinel - -from ._pydantic_core import ( - ArgsKwargs, - MultiHostUrl, - PydanticCustomError, - PydanticKnownError, - PydanticOmit, - PydanticSerializationError, - PydanticSerializationUnexpectedValue, - PydanticUndefined, - PydanticUndefinedType, - PydanticUseDefault, - SchemaError, - SchemaSerializer, - SchemaValidator, - Some, - TzInfo, - Url, - ValidationError, - __version__, - from_json, - to_json, - to_jsonable_python, -) -from .core_schema import CoreConfig, CoreSchema, CoreSchemaType, ErrorType - -if _sys.version_info < (3, 11): - from typing_extensions import NotRequired as _NotRequired -else: - from typing import NotRequired as _NotRequired - -if _sys.version_info < (3, 12): - from typing_extensions import TypedDict as _TypedDict -else: - from typing import TypedDict as _TypedDict - -__all__ = [ - '__version__', - 'UNSET', - 'CoreConfig', - 'CoreSchema', - 'CoreSchemaType', - 'SchemaValidator', - 'SchemaSerializer', - 'Some', - 'Url', - 'MultiHostUrl', - 'ArgsKwargs', - 'PydanticUndefined', - 'PydanticUndefinedType', - 'SchemaError', - 'ErrorDetails', - 'InitErrorDetails', - 'ValidationError', - 'PydanticCustomError', - 'PydanticKnownError', - 'PydanticOmit', - 'PydanticUseDefault', - 'PydanticSerializationError', - 'PydanticSerializationUnexpectedValue', - 'TzInfo', - 'to_json', - 'from_json', - 'to_jsonable_python', -] - - -class ErrorDetails(_TypedDict): - type: str - """ - The type of error that occurred, this is an identifier designed for - programmatic use that will change rarely or never. - - `type` is unique for each error message, and can hence be used as an identifier to build custom error messages. - """ - loc: tuple[int | str, ...] - """Tuple of strings and ints identifying where in the schema the error occurred.""" - msg: str - """A human readable error message.""" - input: _Any - """The input data at this `loc` that caused the error.""" - ctx: _NotRequired[dict[str, _Any]] - """ - Values which are required to render the error message, and could hence be useful in rendering custom error messages. - Also useful for passing custom error data forward. - """ - url: _NotRequired[str] - """ - The documentation URL giving information about the error. No URL is available if - a [`PydanticCustomError`][pydantic_core.PydanticCustomError] is used. - """ - - -class InitErrorDetails(_TypedDict): - type: str | PydanticCustomError - """The type of error that occurred, this should be a "slug" identifier that changes rarely or never.""" - loc: _NotRequired[tuple[int | str, ...]] - """Tuple of strings and ints identifying where in the schema the error occurred.""" - input: _Any - """The input data at this `loc` that caused the error.""" - ctx: _NotRequired[dict[str, _Any]] - """ - Values which are required to render the error message, and could hence be useful in rendering custom error messages. - Also useful for passing custom error data forward. - """ - - -class ErrorTypeInfo(_TypedDict): - """ - Gives information about errors. - """ - - type: ErrorType - """The type of error that occurred, this should be a "slug" identifier that changes rarely or never.""" - message_template_python: str - """String template to render a human readable error message from using context, when the input is Python.""" - example_message_python: str - """Example of a human readable error message, when the input is Python.""" - message_template_json: _NotRequired[str] - """String template to render a human readable error message from using context, when the input is JSON data.""" - example_message_json: _NotRequired[str] - """Example of a human readable error message, when the input is JSON data.""" - example_context: dict[str, _Any] | None - """Example of context values.""" - - -class MultiHostHost(_TypedDict): - """ - A host part of a multi-host URL. - """ - - username: str | None - """The username part of this host, or `None`.""" - password: str | None - """The password part of this host, or `None`.""" - host: str | None - """The host part of this host, or `None`.""" - port: int | None - """The port part of this host, or `None`.""" - - -MISSING = Sentinel('MISSING') -"""A singleton indicating a field value was not provided during validation. - -This singleton can be used a default value, as an alternative to `None` when it has -an explicit meaning. During serialization, any field with `MISSING` as a value is excluded -from the output. - -Example: - ```python - from pydantic import BaseModel - - from pydantic_core import MISSING - - - class Configuration(BaseModel): - timeout: int | None | MISSING = MISSING - - - # configuration defaults, stored somewhere else: - defaults = {'timeout': 200} - - conf = Configuration.model_validate({...}) - timeout = conf.timeout if timeout.timeout is not MISSING else defaults['timeout'] -""" diff --git a/functions/python/data-processor/pydantic_core/_pydantic_core.cpython-314-darwin.so b/functions/python/data-processor/pydantic_core/_pydantic_core.cpython-314-darwin.so deleted file mode 100755 index 3aa6ca6f7..000000000 Binary files a/functions/python/data-processor/pydantic_core/_pydantic_core.cpython-314-darwin.so and /dev/null differ diff --git a/functions/python/data-processor/pydantic_core/_pydantic_core.pyi b/functions/python/data-processor/pydantic_core/_pydantic_core.pyi deleted file mode 100644 index 122c86faa..000000000 --- a/functions/python/data-processor/pydantic_core/_pydantic_core.pyi +++ /dev/null @@ -1,1056 +0,0 @@ -import datetime -from collections.abc import Mapping -from typing import Any, Callable, Generic, Literal, TypeVar, final - -from _typeshed import SupportsAllComparisons -from typing_extensions import LiteralString, Self, TypeAlias - -from pydantic_core import ErrorDetails, ErrorTypeInfo, InitErrorDetails, MultiHostHost -from pydantic_core.core_schema import CoreConfig, CoreSchema, ErrorType, ExtraBehavior - -__all__ = [ - '__version__', - 'build_profile', - 'build_info', - '_recursion_limit', - 'ArgsKwargs', - 'SchemaValidator', - 'SchemaSerializer', - 'Url', - 'MultiHostUrl', - 'SchemaError', - 'ValidationError', - 'PydanticCustomError', - 'PydanticKnownError', - 'PydanticOmit', - 'PydanticUseDefault', - 'PydanticSerializationError', - 'PydanticSerializationUnexpectedValue', - 'PydanticUndefined', - 'PydanticUndefinedType', - 'Some', - 'to_json', - 'from_json', - 'to_jsonable_python', - 'list_all_errors', - 'TzInfo', -] -__version__: str -build_profile: str -build_info: str -_recursion_limit: int - -_T = TypeVar('_T', default=Any, covariant=True) - -_StringInput: TypeAlias = 'dict[str, _StringInput]' - -@final -class Some(Generic[_T]): - """ - Similar to Rust's [`Option::Some`](https://doc.rust-lang.org/std/option/enum.Option.html) type, this - identifies a value as being present, and provides a way to access it. - - Generally used in a union with `None` to different between "some value which could be None" and no value. - """ - - __match_args__ = ('value',) - - @property - def value(self) -> _T: - """ - Returns the value wrapped by `Some`. - """ - @classmethod - def __class_getitem__(cls, item: Any, /) -> type[Self]: ... - -@final -class SchemaValidator: - """ - `SchemaValidator` is the Python wrapper for `pydantic-core`'s Rust validation logic, internally it owns one - `CombinedValidator` which may in turn own more `CombinedValidator`s which make up the full schema validator. - """ - - # note: pyo3 currently supports __new__, but not __init__, though we include __init__ stubs - # and docstrings here (and in the following classes) for documentation purposes - - def __init__(self, schema: CoreSchema, config: CoreConfig | None = None, _use_prebuilt: bool = True) -> None: - """Initializes the `SchemaValidator`. - - Arguments: - schema: The `CoreSchema` to use for validation. - config: Optionally a [`CoreConfig`][pydantic_core.core_schema.CoreConfig] to configure validation. - _use_prebuilt: Whether to use pre-built validators (False during rebuilds to avoid stale references). - """ - - def __new__(cls, schema: CoreSchema, config: CoreConfig | None = None, _use_prebuilt: bool = True) -> Self: ... - @property - def title(self) -> str: - """ - The title of the schema, as used in the heading of [`ValidationError.__str__()`][pydantic_core.ValidationError]. - """ - def validate_python( - self, - input: Any, - *, - strict: bool | None = None, - extra: ExtraBehavior | None = None, - from_attributes: bool | None = None, - context: Any | None = None, - self_instance: Any | None = None, - allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> Any: - """ - Validate a Python object against the schema and return the validated object. - - Arguments: - input: The Python object to validate. - strict: Whether to validate the object in strict mode. - If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. - extra: Whether to ignore, allow, or forbid extra data during model validation. - If `None`, the value of [`CoreConfig.extra_fields_behavior`][pydantic_core.core_schema.CoreConfig] is used. - from_attributes: Whether to validate objects as inputs to models by extracting attributes. - If `None`, the value of [`CoreConfig.from_attributes`][pydantic_core.core_schema.CoreConfig] is used. - context: The context to use for validation, this is passed to functional validators as - [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. - self_instance: An instance of a model set attributes on from validation, this is used when running - validation from the `__init__` method of a model. - allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences - and mappings are ignored. - `'trailing-strings'` means any final unfinished JSON string is included in the result. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Raises: - ValidationError: If validation fails. - Exception: Other error types maybe raised if internal errors occur. - - Returns: - The validated object. - """ - def isinstance_python( - self, - input: Any, - *, - strict: bool | None = None, - extra: ExtraBehavior | None = None, - from_attributes: bool | None = None, - context: Any | None = None, - self_instance: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> bool: - """ - Similar to [`validate_python()`][pydantic_core.SchemaValidator.validate_python] but returns a boolean. - - Arguments match `validate_python()`. This method will not raise `ValidationError`s but will raise internal - errors. - - Returns: - `True` if validation succeeds, `False` if validation fails. - """ - def validate_json( - self, - input: str | bytes | bytearray, - *, - strict: bool | None = None, - extra: ExtraBehavior | None = None, - context: Any | None = None, - self_instance: Any | None = None, - allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> Any: - """ - Validate JSON data directly against the schema and return the validated Python object. - - This method should be significantly faster than `validate_python(json.loads(json_data))` as it avoids the - need to create intermediate Python objects - - It also handles constructing the correct Python type even in strict mode, where - `validate_python(json.loads(json_data))` would fail validation. - - Arguments: - input: The JSON data to validate. - strict: Whether to validate the object in strict mode. - If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. - extra: Whether to ignore, allow, or forbid extra data during model validation. - If `None`, the value of [`CoreConfig.extra_fields_behavior`][pydantic_core.core_schema.CoreConfig] is used. - context: The context to use for validation, this is passed to functional validators as - [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. - self_instance: An instance of a model set attributes on from validation. - allow_partial: Whether to allow partial validation; if `True` incomplete JSON will be parsed successfully - and errors in the last element of sequences and mappings are ignored. - `'trailing-strings'` means any final unfinished JSON string is included in the result. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Raises: - ValidationError: If validation fails or if the JSON data is invalid. - Exception: Other error types maybe raised if internal errors occur. - - Returns: - The validated Python object. - """ - def validate_strings( - self, - input: _StringInput, - *, - strict: bool | None = None, - extra: ExtraBehavior | None = None, - context: Any | None = None, - allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> Any: - """ - Validate a string against the schema and return the validated Python object. - - This is similar to `validate_json` but applies to scenarios where the input will be a string but not - JSON data, e.g. URL fragments, query parameters, etc. - - Arguments: - input: The input as a string, or bytes/bytearray if `strict=False`. - strict: Whether to validate the object in strict mode. - If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. - extra: Whether to ignore, allow, or forbid extra data during model validation. - If `None`, the value of [`CoreConfig.extra_fields_behavior`][pydantic_core.core_schema.CoreConfig] is used. - context: The context to use for validation, this is passed to functional validators as - [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. - allow_partial: Whether to allow partial validation; if `True` errors in the last element of sequences - and mappings are ignored. - `'trailing-strings'` means any final unfinished JSON string is included in the result. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Raises: - ValidationError: If validation fails or if the JSON data is invalid. - Exception: Other error types maybe raised if internal errors occur. - - Returns: - The validated Python object. - """ - def validate_assignment( - self, - obj: Any, - field_name: str, - field_value: Any, - *, - strict: bool | None = None, - extra: ExtraBehavior | None = None, - from_attributes: bool | None = None, - context: Any | None = None, - by_alias: bool | None = None, - by_name: bool | None = None, - ) -> dict[str, Any] | tuple[dict[str, Any], dict[str, Any] | None, set[str]]: - """ - Validate an assignment to a field on a model. - - Arguments: - obj: The model instance being assigned to. - field_name: The name of the field to validate assignment for. - field_value: The value to assign to the field. - strict: Whether to validate the object in strict mode. - If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. - extra: Whether to ignore, allow, or forbid extra data during model validation. - If `None`, the value of [`CoreConfig.extra_fields_behavior`][pydantic_core.core_schema.CoreConfig] is used. - from_attributes: Whether to validate objects as inputs to models by extracting attributes. - If `None`, the value of [`CoreConfig.from_attributes`][pydantic_core.core_schema.CoreConfig] is used. - context: The context to use for validation, this is passed to functional validators as - [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. - by_alias: Whether to use the field's alias when validating against the provided input data. - by_name: Whether to use the field's name when validating against the provided input data. - - Raises: - ValidationError: If validation fails. - Exception: Other error types maybe raised if internal errors occur. - - Returns: - Either the model dict or a tuple of `(model_data, model_extra, fields_set)` - """ - def get_default_value(self, *, strict: bool | None = None, context: Any = None) -> Some | None: - """ - Get the default value for the schema, including running default value validation. - - Arguments: - strict: Whether to validate the default value in strict mode. - If `None`, the value of [`CoreConfig.strict`][pydantic_core.core_schema.CoreConfig] is used. - context: The context to use for validation, this is passed to functional validators as - [`info.context`][pydantic_core.core_schema.ValidationInfo.context]. - - Raises: - ValidationError: If validation fails. - Exception: Other error types maybe raised if internal errors occur. - - Returns: - `None` if the schema has no default value, otherwise a [`Some`][pydantic_core.Some] containing the default. - """ - -# In reality, `bool` should be replaced by `Literal[True]` but mypy fails to correctly apply bidirectional type inference -# (e.g. when using `{'a': {'b': True}}`). -_IncEx: TypeAlias = set[int] | set[str] | Mapping[int, _IncEx | bool] | Mapping[str, _IncEx | bool] - -@final -class SchemaSerializer: - """ - `SchemaSerializer` is the Python wrapper for `pydantic-core`'s Rust serialization logic, internally it owns one - `CombinedSerializer` which may in turn own more `CombinedSerializer`s which make up the full schema serializer. - """ - - def __init__(self, schema: CoreSchema, config: CoreConfig | None = None, _use_prebuilt: bool = True) -> None: - """Initializes the `SchemaSerializer`. - - Arguments: - schema: The `CoreSchema` to use for serialization. - config: Optionally a [`CoreConfig`][pydantic_core.core_schema.CoreConfig] to to configure serialization. - _use_prebuilt: Whether to use pre-built validators (False during rebuilds to avoid stale references). - """ - - def __new__(cls, schema: CoreSchema, config: CoreConfig | None = None, _use_prebuilt: bool = True) -> Self: ... - def to_python( - self, - value: Any, - *, - mode: str | None = None, - include: _IncEx | None = None, - exclude: _IncEx | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - context: Any | None = None, - ) -> Any: - """ - Serialize/marshal a Python object to a Python object including transforming and filtering data. - - Arguments: - value: The Python object to serialize. - mode: The serialization mode to use, either `'python'` or `'json'`, defaults to `'python'`. In JSON mode, - all values are converted to JSON compatible types, e.g. `None`, `int`, `float`, `str`, `list`, `dict`. - include: A set of fields to include, if `None` all fields are included. - exclude: A set of fields to exclude, if `None` no fields are excluded. - by_alias: Whether to use the alias names of fields. - exclude_unset: Whether to exclude fields that are not set, - e.g. are not included in `__pydantic_fields_set__`. - exclude_defaults: Whether to exclude fields that are equal to their default value. - exclude_none: Whether to exclude fields that have a value of `None`. - exclude_computed_fields: Whether to exclude computed fields. - round_trip: Whether to enable serialization and validation round-trip support. - warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered, - if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - context: The context to use for serialization, this is passed to functional serializers as - [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. - - Raises: - PydanticSerializationError: If serialization fails and no `fallback` function is provided. - - Returns: - The serialized Python object. - """ - def to_json( - self, - value: Any, - *, - indent: int | None = None, - ensure_ascii: bool = False, - include: _IncEx | None = None, - exclude: _IncEx | None = None, - by_alias: bool | None = None, - exclude_unset: bool = False, - exclude_defaults: bool = False, - exclude_none: bool = False, - exclude_computed_fields: bool = False, - round_trip: bool = False, - warnings: bool | Literal['none', 'warn', 'error'] = True, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - context: Any | None = None, - ) -> bytes: - """ - Serialize a Python object to JSON including transforming and filtering data. - - Arguments: - value: The Python object to serialize. - indent: If `None`, the JSON will be compact, otherwise it will be pretty-printed with the indent provided. - ensure_ascii: If `True`, the output is guaranteed to have all incoming non-ASCII characters escaped. - If `False` (the default), these characters will be output as-is. - include: A set of fields to include, if `None` all fields are included. - exclude: A set of fields to exclude, if `None` no fields are excluded. - by_alias: Whether to use the alias names of fields. - exclude_unset: Whether to exclude fields that are not set, - e.g. are not included in `__pydantic_fields_set__`. - exclude_defaults: Whether to exclude fields that are equal to their default value. - exclude_none: Whether to exclude fields that have a value of `None`. - exclude_computed_fields: Whether to exclude computed fields. - round_trip: Whether to enable serialization and validation round-trip support. - warnings: How to handle invalid fields. False/"none" ignores them, True/"warn" logs errors, - "error" raises a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError]. - fallback: A function to call when an unknown value is encountered, - if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - context: The context to use for serialization, this is passed to functional serializers as - [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. - - Raises: - PydanticSerializationError: If serialization fails and no `fallback` function is provided. - - Returns: - JSON bytes. - """ - -def to_json( - value: Any, - *, - indent: int | None = None, - ensure_ascii: bool = False, - include: _IncEx | None = None, - exclude: _IncEx | None = None, - # Note: In Pydantic 2.11, the default value of `by_alias` on `SchemaSerializer` was changed from `True` to `None`, - # to be consistent with the Pydantic "dump" methods. However, the default of `True` was kept here for - # backwards compatibility. In Pydantic V3, `by_alias` is expected to default to `True` everywhere: - by_alias: bool = True, - exclude_none: bool = False, - round_trip: bool = False, - timedelta_mode: Literal['iso8601', 'float'] = 'iso8601', - temporal_mode: Literal['iso8601', 'seconds', 'milliseconds'] = 'iso8601', - bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', - inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', - serialize_unknown: bool = False, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - context: Any | None = None, -) -> bytes: - """ - Serialize a Python object to JSON including transforming and filtering data. - - This is effectively a standalone version of [`SchemaSerializer.to_json`][pydantic_core.SchemaSerializer.to_json]. - - Arguments: - value: The Python object to serialize. - indent: If `None`, the JSON will be compact, otherwise it will be pretty-printed with the indent provided. - ensure_ascii: If `True`, the output is guaranteed to have all incoming non-ASCII characters escaped. - If `False` (the default), these characters will be output as-is. - include: A set of fields to include, if `None` all fields are included. - exclude: A set of fields to exclude, if `None` no fields are excluded. - by_alias: Whether to use the alias names of fields. - exclude_none: Whether to exclude fields that have a value of `None`. - round_trip: Whether to enable serialization and validation round-trip support. - timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`. - temporal_mode: How to serialize datetime-like objects (`datetime`, `date`, `time`), either `'iso8601'`, `'seconds'`, or `'milliseconds'`. - `iso8601` returns an ISO 8601 string; `seconds` returns the Unix timestamp in seconds as a float; `milliseconds` returns the Unix timestamp in milliseconds as a float. - - bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. - inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. - serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails - `""` will be used. - fallback: A function to call when an unknown value is encountered, - if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - context: The context to use for serialization, this is passed to functional serializers as - [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. - - Raises: - PydanticSerializationError: If serialization fails and no `fallback` function is provided. - - Returns: - JSON bytes. - """ - -def from_json( - data: str | bytes | bytearray, - *, - allow_inf_nan: bool = True, - cache_strings: bool | Literal['all', 'keys', 'none'] = True, - allow_partial: bool | Literal['off', 'on', 'trailing-strings'] = False, -) -> Any: - """ - Deserialize JSON data to a Python object. - - This is effectively a faster version of `json.loads()`, with some extra functionality. - - Arguments: - data: The JSON data to deserialize. - allow_inf_nan: Whether to allow `Infinity`, `-Infinity` and `NaN` values as `json.loads()` does by default. - cache_strings: Whether to cache strings to avoid constructing new Python objects, - this should have a significant impact on performance while increasing memory usage slightly, - `all/True` means cache all strings, `keys` means cache only dict keys, `none/False` means no caching. - allow_partial: Whether to allow partial deserialization, if `True` JSON data is returned if the end of the - input is reached before the full object is deserialized, e.g. `["aa", "bb", "c` would return `['aa', 'bb']`. - `'trailing-strings'` means any final unfinished JSON string is included in the result. - - Raises: - ValueError: If deserialization fails. - - Returns: - The deserialized Python object. - """ - -def to_jsonable_python( - value: Any, - *, - include: _IncEx | None = None, - exclude: _IncEx | None = None, - # Note: In Pydantic 2.11, the default value of `by_alias` on `SchemaSerializer` was changed from `True` to `None`, - # to be consistent with the Pydantic "dump" methods. However, the default of `True` was kept here for - # backwards compatibility. In Pydantic V3, `by_alias` is expected to default to `True` everywhere: - by_alias: bool = True, - exclude_none: bool = False, - round_trip: bool = False, - timedelta_mode: Literal['iso8601', 'float'] = 'iso8601', - temporal_mode: Literal['iso8601', 'seconds', 'milliseconds'] = 'iso8601', - bytes_mode: Literal['utf8', 'base64', 'hex'] = 'utf8', - inf_nan_mode: Literal['null', 'constants', 'strings'] = 'constants', - serialize_unknown: bool = False, - fallback: Callable[[Any], Any] | None = None, - serialize_as_any: bool = False, - polymorphic_serialization: bool | None = None, - context: Any | None = None, -) -> Any: - """ - Serialize/marshal a Python object to a JSON-serializable Python object including transforming and filtering data. - - This is effectively a standalone version of - [`SchemaSerializer.to_python(mode='json')`][pydantic_core.SchemaSerializer.to_python]. - - Args: - value: The Python object to serialize. - include: A set of fields to include, if `None` all fields are included. - exclude: A set of fields to exclude, if `None` no fields are excluded. - by_alias: Whether to use the alias names of fields. - exclude_none: Whether to exclude fields that have a value of `None`. - round_trip: Whether to enable serialization and validation round-trip support. - timedelta_mode: How to serialize `timedelta` objects, either `'iso8601'` or `'float'`. - temporal_mode: How to serialize datetime-like objects (`datetime`, `date`, `time`), either `'iso8601'`, `'seconds'`, or `'milliseconds'`. - `iso8601` returns an ISO 8601 string; `seconds` returns the Unix timestamp in seconds as a float; `milliseconds` returns the Unix timestamp in milliseconds as a float. - - bytes_mode: How to serialize `bytes` objects, either `'utf8'`, `'base64'`, or `'hex'`. - inf_nan_mode: How to serialize `Infinity`, `-Infinity` and `NaN` values, either `'null'`, `'constants'`, or `'strings'`. - serialize_unknown: Attempt to serialize unknown types, `str(value)` will be used, if that fails - `""` will be used. - fallback: A function to call when an unknown value is encountered, - if `None` a [`PydanticSerializationError`][pydantic_core.PydanticSerializationError] error is raised. - serialize_as_any: Whether to serialize fields with duck-typing serialization behavior. - polymorphic_serialization: Whether to use model and dataclass polymorphic serialization for this call. - context: The context to use for serialization, this is passed to functional serializers as - [`info.context`][pydantic_core.core_schema.SerializationInfo.context]. - - Raises: - PydanticSerializationError: If serialization fails and no `fallback` function is provided. - - Returns: - The serialized Python object. - """ - -class Url(SupportsAllComparisons): - """ - A URL type, internal logic uses the [url rust crate](https://docs.rs/url/latest/url/) originally developed - by Mozilla. - """ - - def __init__(self, url: str) -> None: ... - def __new__(cls, url: str) -> Self: ... - @property - def scheme(self) -> str: ... - @property - def username(self) -> str | None: ... - @property - def password(self) -> str | None: ... - @property - def host(self) -> str | None: ... - def unicode_host(self) -> str | None: ... - @property - def port(self) -> int | None: ... - @property - def path(self) -> str | None: ... - @property - def query(self) -> str | None: ... - def query_params(self) -> list[tuple[str, str]]: ... - @property - def fragment(self) -> str | None: ... - def unicode_string(self) -> str: ... - def __repr__(self) -> str: ... - def __str__(self) -> str: ... - def __deepcopy__(self, memo: dict) -> str: ... - @classmethod - def build( - cls, - *, - scheme: str, - username: str | None = None, - password: str | None = None, - host: str, - port: int | None = None, - path: str | None = None, - query: str | None = None, - fragment: str | None = None, - ) -> Self: ... - -class MultiHostUrl(SupportsAllComparisons): - """ - A URL type with support for multiple hosts, as used by some databases for DSNs, e.g. `https://foo.com,bar.com/path`. - - Internal URL logic uses the [url rust crate](https://docs.rs/url/latest/url/) originally developed - by Mozilla. - """ - - def __init__(self, url: str) -> None: ... - def __new__(cls, url: str) -> Self: ... - @property - def scheme(self) -> str: ... - @property - def path(self) -> str | None: ... - @property - def query(self) -> str | None: ... - def query_params(self) -> list[tuple[str, str]]: ... - @property - def fragment(self) -> str | None: ... - def hosts(self) -> list[MultiHostHost]: ... - def unicode_string(self) -> str: ... - def __repr__(self) -> str: ... - def __str__(self) -> str: ... - def __deepcopy__(self, memo: dict) -> Self: ... - @classmethod - def build( - cls, - *, - scheme: str, - hosts: list[MultiHostHost] | None = None, - username: str | None = None, - password: str | None = None, - host: str | None = None, - port: int | None = None, - path: str | None = None, - query: str | None = None, - fragment: str | None = None, - ) -> Self: ... - -@final -class SchemaError(Exception): - """ - Information about errors that occur while building a [`SchemaValidator`][pydantic_core.SchemaValidator] - or [`SchemaSerializer`][pydantic_core.SchemaSerializer]. - """ - - def error_count(self) -> int: - """ - Returns: - The number of errors in the schema. - """ - def errors(self) -> list[ErrorDetails]: - """ - Returns: - A list of [`ErrorDetails`][pydantic_core.ErrorDetails] for each error in the schema. - """ - -class ValidationError(ValueError): - """ - `ValidationError` is the exception raised by `pydantic-core` when validation fails, it contains a list of errors - which detail why validation failed. - """ - @classmethod - def from_exception_data( - cls, - title: str, - line_errors: list[InitErrorDetails], - input_type: Literal['python', 'json'] = 'python', - hide_input: bool = False, - ) -> Self: - """ - Python constructor for a Validation Error. - - Arguments: - title: The title of the error, as used in the heading of `str(validation_error)` - line_errors: A list of [`InitErrorDetails`][pydantic_core.InitErrorDetails] which contain information - about errors that occurred during validation. - input_type: Whether the error is for a Python object or JSON. - hide_input: Whether to hide the input value in the error message. - """ - @property - def title(self) -> str: - """ - The title of the error, as used in the heading of `str(validation_error)`. - """ - def error_count(self) -> int: - """ - Returns: - The number of errors in the validation error. - """ - def errors( - self, *, include_url: bool = True, include_context: bool = True, include_input: bool = True - ) -> list[ErrorDetails]: - """ - Details about each error in the validation error. - - Args: - include_url: Whether to include a URL to documentation on the error each error. - include_context: Whether to include the context of each error. - include_input: Whether to include the input value of each error. - - Returns: - A list of [`ErrorDetails`][pydantic_core.ErrorDetails] for each error in the validation error. - """ - def json( - self, - *, - indent: int | None = None, - include_url: bool = True, - include_context: bool = True, - include_input: bool = True, - ) -> str: - """ - Same as [`errors()`][pydantic_core.ValidationError.errors] but returns a JSON string. - - Args: - indent: The number of spaces to indent the JSON by, or `None` for no indentation - compact JSON. - include_url: Whether to include a URL to documentation on the error each error. - include_context: Whether to include the context of each error. - include_input: Whether to include the input value of each error. - - Returns: - a JSON string. - """ - - def __repr__(self) -> str: - """ - A string representation of the validation error. - - Whether or not documentation URLs are included in the repr is controlled by the - environment variable `PYDANTIC_ERRORS_INCLUDE_URL` being set to `1` or - `true`; by default, URLs are shown. - - Due to implementation details, this environment variable can only be set once, - before the first validation error is created. - """ - -class PydanticCustomError(ValueError): - """A custom exception providing flexible error handling for Pydantic validators. - - You can raise this error in custom validators when you'd like flexibility in regards to the error type, message, and context. - - Example: - ```py - from pydantic_core import PydanticCustomError - - def custom_validator(v) -> None: - if v <= 10: - raise PydanticCustomError('custom_value_error', 'Value must be greater than {value}', {'value': 10, 'extra_context': 'extra_data'}) - return v - ``` - - Arguments: - error_type: The error type. - message_template: The message template. - context: The data to inject into the message template. - """ - - def __init__( - self, error_type: LiteralString, message_template: LiteralString, context: dict[str, Any] | None = None, / - ) -> None: ... - @property - def context(self) -> dict[str, Any] | None: - """Values which are required to render the error message, and could hence be useful in passing error data forward.""" - - @property - def type(self) -> str: - """The error type associated with the error. For consistency with Pydantic, this is typically a snake_case string.""" - - @property - def message_template(self) -> str: - """The message template associated with the error. This is a string that can be formatted with context variables in `{curly_braces}`.""" - - def message(self) -> str: - """The formatted message associated with the error. This presents as the message template with context variables appropriately injected.""" - -@final -class PydanticKnownError(ValueError): - """A helper class for raising exceptions that mimic Pydantic's built-in exceptions, with more flexibility in regards to context. - - Unlike [`PydanticCustomError`][pydantic_core.PydanticCustomError], the `error_type` argument must be a known `ErrorType`. - - Example: - ```py - from pydantic_core import PydanticKnownError - - def custom_validator(v) -> None: - if v <= 10: - raise PydanticKnownError('greater_than', {'gt': 10}) - return v - ``` - - Arguments: - error_type: The error type. - context: The data to inject into the message template. - """ - - def __init__(self, error_type: ErrorType, context: dict[str, Any] | None = None, /) -> None: ... - @property - def context(self) -> dict[str, Any] | None: - """Values which are required to render the error message, and could hence be useful in passing error data forward.""" - - @property - def type(self) -> ErrorType: - """The type of the error.""" - - @property - def message_template(self) -> str: - """The message template associated with the provided error type. This is a string that can be formatted with context variables in `{curly_braces}`.""" - - def message(self) -> str: - """The formatted message associated with the error. This presents as the message template with context variables appropriately injected.""" - -@final -class PydanticOmit(Exception): - """An exception to signal that a field should be omitted from a generated result. - - This could span from omitting a field from a JSON Schema to omitting a field from a serialized result. - Upcoming: more robust support for using PydanticOmit in custom serializers is still in development. - Right now, this is primarily used in the JSON Schema generation process. - - Example: - ```py - from typing import Callable - - from pydantic_core import PydanticOmit - - from pydantic import BaseModel - from pydantic.json_schema import GenerateJsonSchema, JsonSchemaValue - - - class MyGenerateJsonSchema(GenerateJsonSchema): - def handle_invalid_for_json_schema(self, schema, error_info) -> JsonSchemaValue: - raise PydanticOmit - - - class Predicate(BaseModel): - name: str = 'no-op' - func: Callable = lambda x: x - - - instance_example = Predicate() - - validation_schema = instance_example.model_json_schema(schema_generator=MyGenerateJsonSchema, mode='validation') - print(validation_schema) - ''' - {'properties': {'name': {'default': 'no-op', 'title': 'Name', 'type': 'string'}}, 'title': 'Predicate', 'type': 'object'} - ''' - ``` - - For a more in depth example / explanation, see the [customizing JSON schema](../concepts/json_schema.md#customizing-the-json-schema-generation-process) docs. - """ - - def __new__(cls) -> Self: ... - -@final -class PydanticUseDefault(Exception): - """An exception to signal that standard validation either failed or should be skipped, and the default value should be used instead. - - This warning can be raised in custom validation functions to redirect the flow of validation. - - Example: - ```py - from pydantic_core import PydanticUseDefault - from datetime import datetime - from pydantic import BaseModel, field_validator - - - class Event(BaseModel): - name: str = 'meeting' - time: datetime - - @field_validator('name', mode='plain') - def name_must_be_present(cls, v) -> str: - if not v or not isinstance(v, str): - raise PydanticUseDefault() - return v - - - event1 = Event(name='party', time=datetime(2024, 1, 1, 12, 0, 0)) - print(repr(event1)) - # > Event(name='party', time=datetime.datetime(2024, 1, 1, 12, 0)) - event2 = Event(time=datetime(2024, 1, 1, 12, 0, 0)) - print(repr(event2)) - # > Event(name='meeting', time=datetime.datetime(2024, 1, 1, 12, 0)) - ``` - - For an additional example, see the [validating partial json data](../concepts/json.md#partial-json-parsing) section of the Pydantic documentation. - """ - - def __new__(cls) -> Self: ... - -@final -class PydanticSerializationError(ValueError): - """An error raised when an issue occurs during serialization. - - In custom serializers, this error can be used to indicate that serialization has failed. - - Arguments: - message: The message associated with the error. - """ - - def __init__(self, message: str, /) -> None: ... - -@final -class PydanticSerializationUnexpectedValue(ValueError): - """An error raised when an unexpected value is encountered during serialization. - - This error is often caught and coerced into a warning, as `pydantic-core` generally makes a best attempt - at serializing values, in contrast with validation where errors are eagerly raised. - - Example: - ```py - from pydantic import BaseModel, field_serializer - from pydantic_core import PydanticSerializationUnexpectedValue - - class BasicPoint(BaseModel): - x: int - y: int - - @field_serializer('*') - def serialize(self, v): - if not isinstance(v, int): - raise PydanticSerializationUnexpectedValue(f'Expected type `int`, got {type(v)} with value {v}') - return v - - point = BasicPoint(x=1, y=2) - # some sort of mutation - point.x = 'a' - - print(point.model_dump()) - ''' - UserWarning: Pydantic serializer warnings: - PydanticSerializationUnexpectedValue(Expected type `int`, got with value a) - return self.__pydantic_serializer__.to_python( - {'x': 'a', 'y': 2} - ''' - ``` - - This is often used internally in `pydantic-core` when unexpected types are encountered during serialization, - but it can also be used by users in custom serializers, as seen above. - - Arguments: - message: The message associated with the unexpected value. - """ - - def __init__(self, message: str, /) -> None: ... - -@final -class ArgsKwargs: - """A construct used to store arguments and keyword arguments for a function call. - - This data structure is generally used to store information for core schemas associated with functions (like in an arguments schema). - This data structure is also currently used for some validation against dataclasses. - - Example: - ```py - from pydantic.dataclasses import dataclass - from pydantic import model_validator - - - @dataclass - class Model: - a: int - b: int - - @model_validator(mode="before") - @classmethod - def no_op_validator(cls, values): - print(values) - return values - - Model(1, b=2) - #> ArgsKwargs((1,), {"b": 2}) - - Model(1, 2) - #> ArgsKwargs((1, 2), {}) - - Model(a=1, b=2) - #> ArgsKwargs((), {"a": 1, "b": 2}) - ``` - """ - - def __init__(self, args: tuple[Any, ...], kwargs: dict[str, Any] | None = None) -> None: - """Initializes the `ArgsKwargs`. - - Arguments: - args: The arguments (inherently ordered) for a function call. - kwargs: The keyword arguments for a function call - """ - - def __new__(cls, args: tuple[Any, ...], kwargs: dict[str, Any] | None = None) -> Self: ... - @property - def args(self) -> tuple[Any, ...]: - """The arguments (inherently ordered) for a function call.""" - - @property - def kwargs(self) -> dict[str, Any] | None: - """The keyword arguments for a function call.""" - -@final -class PydanticUndefinedType: - """A type used as a sentinel for undefined values.""" - - def __copy__(self) -> Self: ... - def __deepcopy__(self, memo: Any) -> Self: ... - -PydanticUndefined: PydanticUndefinedType - -def list_all_errors() -> list[ErrorTypeInfo]: - """ - Get information about all built-in errors. - - Returns: - A list of `ErrorTypeInfo` typed dicts. - """ -@final -class TzInfo(datetime.tzinfo): - """An `pydantic-core` implementation of the abstract [`datetime.tzinfo`][] class.""" - - def __init__(self, seconds: float = 0.0) -> None: - """Initializes the `TzInfo`. - - Arguments: - seconds: The offset from UTC in seconds. Defaults to 0.0 (UTC). - """ - - def __new__(cls, seconds: float = 0.0) -> Self: ... - - # Docstrings for attributes sourced from the abstract base class, [`datetime.tzinfo`](https://docs.python.org/3/library/datetime.html#datetime.tzinfo). - - def tzname(self, dt: datetime.datetime | None) -> str | None: - """Return the time zone name corresponding to the [`datetime`][datetime.datetime] object _dt_, as a string. - - For more info, see [`tzinfo.tzname`][datetime.tzinfo.tzname]. - """ - - def utcoffset(self, dt: datetime.datetime | None) -> datetime.timedelta | None: - """Return offset of local time from UTC, as a [`timedelta`][datetime.timedelta] object that is positive east of UTC. If local time is west of UTC, this should be negative. - - More info can be found at [`tzinfo.utcoffset`][datetime.tzinfo.utcoffset]. - """ - - def dst(self, dt: datetime.datetime | None) -> datetime.timedelta | None: - """Return the daylight saving time (DST) adjustment, as a [`timedelta`][datetime.timedelta] object or `None` if DST information isn’t known. - - More info can be found at[`tzinfo.dst`][datetime.tzinfo.dst].""" - - def fromutc(self, dt: datetime.datetime) -> datetime.datetime: - """Adjust the date and time data associated datetime object _dt_, returning an equivalent datetime in self’s local time. - - More info can be found at [`tzinfo.fromutc`][datetime.tzinfo.fromutc].""" - - def __deepcopy__(self, _memo: dict[Any, Any]) -> TzInfo: ... diff --git a/functions/python/data-processor/pydantic_core/core_schema.py b/functions/python/data-processor/pydantic_core/core_schema.py deleted file mode 100644 index 6ff202c15..000000000 --- a/functions/python/data-processor/pydantic_core/core_schema.py +++ /dev/null @@ -1,4461 +0,0 @@ -""" -This module contains definitions to build schemas which `pydantic_core` can -validate and serialize. -""" - -from __future__ import annotations as _annotations - -import sys -import warnings -from collections.abc import Generator, Hashable, Mapping -from datetime import date, datetime, time, timedelta -from decimal import Decimal -from re import Pattern -from typing import TYPE_CHECKING, Any, Callable, Literal, Union - -from typing_extensions import TypeVar, deprecated - -if sys.version_info < (3, 12): - from typing_extensions import TypedDict -else: - from typing import TypedDict - -if sys.version_info < (3, 11): - from typing_extensions import Protocol, Required, TypeAlias -else: - from typing import Protocol, Required, TypeAlias - -if TYPE_CHECKING: - from pydantic_core import PydanticUndefined -else: - # The initial build of pydantic_core requires PydanticUndefined to generate - # the core schema; so we need to conditionally skip it. mypy doesn't like - # this at all, hence the TYPE_CHECKING branch above. - try: - from pydantic_core import PydanticUndefined - except ImportError: - PydanticUndefined = object() - - -ExtraBehavior = Literal['allow', 'forbid', 'ignore'] - - -class CoreConfig(TypedDict, total=False): - """ - Base class for schema configuration options. - - Attributes: - title: The name of the configuration. - strict: Whether the configuration should strictly adhere to specified rules. - extra_fields_behavior: The behavior for handling extra fields. - typed_dict_total: Whether the TypedDict should be considered total. Default is `True`. - from_attributes: Whether to use attributes for models, dataclasses, and tagged union keys. - loc_by_alias: Whether to use the used alias (or first alias for "field required" errors) instead of - `field_names` to construct error `loc`s. Default is `True`. - revalidate_instances: Whether instances of models and dataclasses should re-validate. Default is 'never'. - validate_default: Whether to validate default values during validation. Default is `False`. - str_max_length: The maximum length for string fields. - str_min_length: The minimum length for string fields. - str_strip_whitespace: Whether to strip whitespace from string fields. - str_to_lower: Whether to convert string fields to lowercase. - str_to_upper: Whether to convert string fields to uppercase. - allow_inf_nan: Whether to allow infinity and NaN values for float fields. Default is `True`. - ser_json_timedelta: The serialization option for `timedelta` values. Default is 'iso8601'. - Note that if ser_json_temporal is set, then this param will be ignored. - ser_json_temporal: The serialization option for datetime like values. Default is 'iso8601'. - The types this covers are datetime, date, time and timedelta. - If this is set, it will take precedence over ser_json_timedelta - ser_json_bytes: The serialization option for `bytes` values. Default is 'utf8'. - ser_json_inf_nan: The serialization option for infinity and NaN values - in float fields. Default is 'null'. - val_json_bytes: The validation option for `bytes` values, complementing ser_json_bytes. Default is 'utf8'. - hide_input_in_errors: Whether to hide input data from `ValidationError` representation. - validation_error_cause: Whether to add user-python excs to the __cause__ of a ValidationError. - Requires exceptiongroup backport pre Python 3.11. - coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode). - regex_engine: The regex engine to use for regex pattern validation. Default is 'rust-regex'. See `StringSchema`. - cache_strings: Whether to cache strings. Default is `True`, `True` or `'all'` is required to cache strings - during general validation since validators don't know if they're in a key or a value. - validate_by_alias: Whether to use the field's alias when validating against the provided input data. Default is `True`. - validate_by_name: Whether to use the field's name when validating against the provided input data. Default is `False`. Replacement for `populate_by_name`. - serialize_by_alias: Whether to serialize by alias. Default is `False`, expected to change to `True` in V3. - polymorphic_serialization: Whether to enable polymorphic serialization for models and dataclasses. Default is `False`. - url_preserve_empty_path: Whether to preserve empty URL paths when validating values for a URL type. Defaults to `False`. - """ - - title: str - strict: bool - # settings related to typed dicts, model fields, dataclass fields - extra_fields_behavior: ExtraBehavior - typed_dict_total: bool # default: True - # used for models, dataclasses, and tagged union keys - from_attributes: bool - # whether to use the used alias (or first alias for "field required" errors) instead of field_names - # to construct error `loc`s, default True - loc_by_alias: bool - # whether instances of models and dataclasses (including subclass instances) should re-validate, default 'never' - revalidate_instances: Literal['always', 'never', 'subclass-instances'] - # whether to validate default values during validation, default False - validate_default: bool - # used on typed-dicts and arguments - # fields related to string fields only - str_max_length: int - str_min_length: int - str_strip_whitespace: bool - str_to_lower: bool - str_to_upper: bool - # fields related to float fields only - allow_inf_nan: bool # default: True - # the config options are used to customise serialization to JSON - ser_json_timedelta: Literal['iso8601', 'float'] # default: 'iso8601' - ser_json_temporal: Literal['iso8601', 'seconds', 'milliseconds'] # default: 'iso8601' - ser_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' - ser_json_inf_nan: Literal['null', 'constants', 'strings'] # default: 'null' - val_json_bytes: Literal['utf8', 'base64', 'hex'] # default: 'utf8' - # used to hide input data from ValidationError repr - hide_input_in_errors: bool - validation_error_cause: bool # default: False - coerce_numbers_to_str: bool # default: False - regex_engine: Literal['rust-regex', 'python-re'] # default: 'rust-regex' - cache_strings: Union[bool, Literal['all', 'keys', 'none']] # default: 'True' - validate_by_alias: bool # default: True - validate_by_name: bool # default: False - serialize_by_alias: bool # default: False - polymorphic_serialization: bool # default: False - url_preserve_empty_path: bool # default: False - - -IncExCall: TypeAlias = 'set[int | str] | dict[int | str, IncExCall] | None' - -ContextT = TypeVar('ContextT', covariant=True, default='Any | None') - - -class SerializationInfo(Protocol[ContextT]): - """Extra data used during serialization.""" - - @property - def include(self) -> IncExCall: - """The `include` argument set during serialization.""" - ... - - @property - def exclude(self) -> IncExCall: - """The `exclude` argument set during serialization.""" - ... - - @property - def context(self) -> ContextT: - """The current serialization context.""" - ... - - @property - def mode(self) -> Literal['python', 'json'] | str: - """The serialization mode set during serialization.""" - ... - - @property - def by_alias(self) -> bool: - """The `by_alias` argument set during serialization.""" - ... - - @property - def exclude_unset(self) -> bool: - """The `exclude_unset` argument set during serialization.""" - ... - - @property - def exclude_defaults(self) -> bool: - """The `exclude_defaults` argument set during serialization.""" - ... - - @property - def exclude_none(self) -> bool: - """The `exclude_none` argument set during serialization.""" - ... - - @property - def exclude_computed_fields(self) -> bool: - """The `exclude_computed_fields` argument set during serialization.""" - ... - - @property - def serialize_as_any(self) -> bool: - """The `serialize_as_any` argument set during serialization.""" - ... - - @property - def polymorphic_serialization(self) -> bool | None: - """The `polymorphic_serialization` argument set during serialization, if any.""" - ... - - @property - def round_trip(self) -> bool: - """The `round_trip` argument set during serialization.""" - ... - - def mode_is_json(self) -> bool: ... - - def __str__(self) -> str: ... - - def __repr__(self) -> str: ... - - -class FieldSerializationInfo(SerializationInfo[ContextT], Protocol): - """Extra data used during field serialization.""" - - @property - def field_name(self) -> str: - """The name of the current field being serialized.""" - ... - - -class ValidationInfo(Protocol[ContextT]): - """Extra data used during validation.""" - - @property - def context(self) -> ContextT: - """The current validation context.""" - ... - - @property - def config(self) -> CoreConfig | None: - """The CoreConfig that applies to this validation.""" - ... - - @property - def mode(self) -> Literal['python', 'json']: - """The type of input data we are currently validating.""" - ... - - @property - def data(self) -> dict[str, Any]: - """The data being validated for this model.""" - ... - - @property - def field_name(self) -> str | None: - """ - The name of the current field being validated if this validator is - attached to a model field. - """ - ... - - -ExpectedSerializationTypes = Literal[ - 'none', - 'int', - 'bool', - 'float', - 'str', - 'bytes', - 'bytearray', - 'list', - 'tuple', - 'set', - 'frozenset', - 'generator', - 'dict', - 'datetime', - 'date', - 'time', - 'timedelta', - 'url', - 'multi-host-url', - 'json', - 'uuid', - 'any', -] - - -class SimpleSerSchema(TypedDict, total=False): - type: Required[ExpectedSerializationTypes] - - -def simple_ser_schema(type: ExpectedSerializationTypes) -> SimpleSerSchema: - """ - Returns a schema for serialization with a custom type. - - Args: - type: The type to use for serialization - """ - return SimpleSerSchema(type=type) - - -# (input_value: Any, /) -> Any -GeneralPlainNoInfoSerializerFunction = Callable[[Any], Any] -# (input_value: Any, info: FieldSerializationInfo, /) -> Any -GeneralPlainInfoSerializerFunction = Callable[[Any, SerializationInfo[Any]], Any] -# (model: Any, input_value: Any, /) -> Any -FieldPlainNoInfoSerializerFunction = Callable[[Any, Any], Any] -# (model: Any, input_value: Any, info: FieldSerializationInfo, /) -> Any -FieldPlainInfoSerializerFunction = Callable[[Any, Any, FieldSerializationInfo[Any]], Any] -SerializerFunction = Union[ - GeneralPlainNoInfoSerializerFunction, - GeneralPlainInfoSerializerFunction, - FieldPlainNoInfoSerializerFunction, - FieldPlainInfoSerializerFunction, -] - -WhenUsed = Literal['always', 'unless-none', 'json', 'json-unless-none'] -""" -Values have the following meanings: - -* `'always'` means always use -* `'unless-none'` means use unless the value is `None` -* `'json'` means use when serializing to JSON -* `'json-unless-none'` means use when serializing to JSON and the value is not `None` -""" - - -class PlainSerializerFunctionSerSchema(TypedDict, total=False): - type: Required[Literal['function-plain']] - function: Required[SerializerFunction] - is_field_serializer: bool # default False - info_arg: bool # default False - return_schema: CoreSchema # if omitted, AnySchema is used - when_used: WhenUsed # default: 'always' - - -def plain_serializer_function_ser_schema( - function: SerializerFunction, - *, - is_field_serializer: bool | None = None, - info_arg: bool | None = None, - return_schema: CoreSchema | None = None, - when_used: WhenUsed = 'always', -) -> PlainSerializerFunctionSerSchema: - """ - Returns a schema for serialization with a function, can be either a "general" or "field" function. - - Args: - function: The function to use for serialization - is_field_serializer: Whether the serializer is for a field, e.g. takes `model` as the first argument, - and `info` includes `field_name` - info_arg: Whether the function takes an `info` argument - return_schema: Schema to use for serializing return value - when_used: When the function should be called - """ - if when_used == 'always': - # just to avoid extra elements in schema, and to use the actual default defined in rust - when_used = None # type: ignore - return _dict_not_none( - type='function-plain', - function=function, - is_field_serializer=is_field_serializer, - info_arg=info_arg, - return_schema=return_schema, - when_used=when_used, - ) - - -class SerializerFunctionWrapHandler(Protocol): # pragma: no cover - def __call__(self, input_value: Any, index_key: int | str | None = None, /) -> Any: ... - - -# (input_value: Any, serializer: SerializerFunctionWrapHandler, /) -> Any -GeneralWrapNoInfoSerializerFunction = Callable[[Any, SerializerFunctionWrapHandler], Any] -# (input_value: Any, serializer: SerializerFunctionWrapHandler, info: SerializationInfo, /) -> Any -GeneralWrapInfoSerializerFunction = Callable[[Any, SerializerFunctionWrapHandler, SerializationInfo[Any]], Any] -# (model: Any, input_value: Any, serializer: SerializerFunctionWrapHandler, /) -> Any -FieldWrapNoInfoSerializerFunction = Callable[[Any, Any, SerializerFunctionWrapHandler], Any] -# (model: Any, input_value: Any, serializer: SerializerFunctionWrapHandler, info: FieldSerializationInfo, /) -> Any -FieldWrapInfoSerializerFunction = Callable[[Any, Any, SerializerFunctionWrapHandler, FieldSerializationInfo[Any]], Any] -WrapSerializerFunction = Union[ - GeneralWrapNoInfoSerializerFunction, - GeneralWrapInfoSerializerFunction, - FieldWrapNoInfoSerializerFunction, - FieldWrapInfoSerializerFunction, -] - - -class WrapSerializerFunctionSerSchema(TypedDict, total=False): - type: Required[Literal['function-wrap']] - function: Required[WrapSerializerFunction] - is_field_serializer: bool # default False - info_arg: bool # default False - schema: CoreSchema # if omitted, the schema on which this serializer is defined is used - return_schema: CoreSchema # if omitted, AnySchema is used - when_used: WhenUsed # default: 'always' - - -def wrap_serializer_function_ser_schema( - function: WrapSerializerFunction, - *, - is_field_serializer: bool | None = None, - info_arg: bool | None = None, - schema: CoreSchema | None = None, - return_schema: CoreSchema | None = None, - when_used: WhenUsed = 'always', -) -> WrapSerializerFunctionSerSchema: - """ - Returns a schema for serialization with a wrap function, can be either a "general" or "field" function. - - Args: - function: The function to use for serialization - is_field_serializer: Whether the serializer is for a field, e.g. takes `model` as the first argument, - and `info` includes `field_name` - info_arg: Whether the function takes an `info` argument - schema: The schema to use for the inner serialization - return_schema: Schema to use for serializing return value - when_used: When the function should be called - """ - if when_used == 'always': - # just to avoid extra elements in schema, and to use the actual default defined in rust - when_used = None # type: ignore - return _dict_not_none( - type='function-wrap', - function=function, - is_field_serializer=is_field_serializer, - info_arg=info_arg, - schema=schema, - return_schema=return_schema, - when_used=when_used, - ) - - -class FormatSerSchema(TypedDict, total=False): - type: Required[Literal['format']] - formatting_string: Required[str] - when_used: WhenUsed # default: 'json-unless-none' - - -def format_ser_schema(formatting_string: str, *, when_used: WhenUsed = 'json-unless-none') -> FormatSerSchema: - """ - Returns a schema for serialization using python's `format` method. - - Args: - formatting_string: String defining the format to use - when_used: Same meaning as for [general_function_plain_ser_schema], but with a different default - """ - if when_used == 'json-unless-none': - # just to avoid extra elements in schema, and to use the actual default defined in rust - when_used = None # type: ignore - return _dict_not_none(type='format', formatting_string=formatting_string, when_used=when_used) - - -class ToStringSerSchema(TypedDict, total=False): - type: Required[Literal['to-string']] - when_used: WhenUsed # default: 'json-unless-none' - - -def to_string_ser_schema(*, when_used: WhenUsed = 'json-unless-none') -> ToStringSerSchema: - """ - Returns a schema for serialization using python's `str()` / `__str__` method. - - Args: - when_used: Same meaning as for [general_function_plain_ser_schema], but with a different default - """ - s = dict(type='to-string') - if when_used != 'json-unless-none': - # just to avoid extra elements in schema, and to use the actual default defined in rust - s['when_used'] = when_used - return s # type: ignore - - -class ModelSerSchema(TypedDict, total=False): - type: Required[Literal['model']] - cls: Required[type[Any]] - schema: Required[CoreSchema] - - -def model_ser_schema(cls: type[Any], schema: CoreSchema) -> ModelSerSchema: - """ - Returns a schema for serialization using a model. - - Args: - cls: The expected class type, used to generate warnings if the wrong type is passed - schema: Internal schema to use to serialize the model dict - """ - return ModelSerSchema(type='model', cls=cls, schema=schema) - - -SerSchema = Union[ - SimpleSerSchema, - PlainSerializerFunctionSerSchema, - WrapSerializerFunctionSerSchema, - FormatSerSchema, - ToStringSerSchema, - ModelSerSchema, -] - - -class InvalidSchema(TypedDict, total=False): - type: Required[Literal['invalid']] - ref: str - metadata: dict[str, Any] - # note, we never plan to use this, but include it for type checking purposes to match - # all other CoreSchema union members - serialization: SerSchema - - -def invalid_schema(ref: str | None = None, metadata: dict[str, Any] | None = None) -> InvalidSchema: - """ - Returns an invalid schema, used to indicate that a schema is invalid. - - Args: - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - """ - - return _dict_not_none(type='invalid', ref=ref, metadata=metadata) - - -class ComputedField(TypedDict, total=False): - type: Required[Literal['computed-field']] - property_name: Required[str] - return_schema: Required[CoreSchema] - alias: str - serialization_exclude_if: Callable[[Any], bool] - metadata: dict[str, Any] - - -def computed_field( - property_name: str, - return_schema: CoreSchema, - *, - alias: str | None = None, - serialization_exclude_if: Callable[[Any], bool] | None = None, - metadata: dict[str, Any] | None = None, -) -> ComputedField: - """ - ComputedFields are properties of a model or dataclass that are included in serialization. - - Args: - property_name: The name of the property on the model or dataclass - return_schema: The schema used for the type returned by the computed field - alias: The name to use in the serialized output - metadata: Any other information you want to include with the schema, not used by pydantic-core - """ - return _dict_not_none( - type='computed-field', - property_name=property_name, - return_schema=return_schema, - alias=alias, - serialization_exclude_if=serialization_exclude_if, - metadata=metadata, - ) - - -class AnySchema(TypedDict, total=False): - type: Required[Literal['any']] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def any_schema( - *, ref: str | None = None, metadata: dict[str, Any] | None = None, serialization: SerSchema | None = None -) -> AnySchema: - """ - Returns a schema that matches any value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.any_schema() - v = SchemaValidator(schema) - assert v.validate_python(1) == 1 - ``` - - Args: - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='any', ref=ref, metadata=metadata, serialization=serialization) - - -class NoneSchema(TypedDict, total=False): - type: Required[Literal['none']] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def none_schema( - *, ref: str | None = None, metadata: dict[str, Any] | None = None, serialization: SerSchema | None = None -) -> NoneSchema: - """ - Returns a schema that matches a None value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.none_schema() - v = SchemaValidator(schema) - assert v.validate_python(None) is None - ``` - - Args: - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='none', ref=ref, metadata=metadata, serialization=serialization) - - -class BoolSchema(TypedDict, total=False): - type: Required[Literal['bool']] - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def bool_schema( - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> BoolSchema: - """ - Returns a schema that matches a bool value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.bool_schema() - v = SchemaValidator(schema) - assert v.validate_python('True') is True - ``` - - Args: - strict: Whether the value should be a bool or a value that can be converted to a bool - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='bool', strict=strict, ref=ref, metadata=metadata, serialization=serialization) - - -class IntSchema(TypedDict, total=False): - type: Required[Literal['int']] - multiple_of: int - le: int - ge: int - lt: int - gt: int - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def int_schema( - *, - multiple_of: int | None = None, - le: int | None = None, - ge: int | None = None, - lt: int | None = None, - gt: int | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> IntSchema: - """ - Returns a schema that matches a int value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.int_schema(multiple_of=2, le=6, ge=2) - v = SchemaValidator(schema) - assert v.validate_python('4') == 4 - ``` - - Args: - multiple_of: The value must be a multiple of this number - le: The value must be less than or equal to this number - ge: The value must be greater than or equal to this number - lt: The value must be strictly less than this number - gt: The value must be strictly greater than this number - strict: Whether the value should be a int or a value that can be converted to a int - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='int', - multiple_of=multiple_of, - le=le, - ge=ge, - lt=lt, - gt=gt, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class FloatSchema(TypedDict, total=False): - type: Required[Literal['float']] - allow_inf_nan: bool # whether 'NaN', '+inf', '-inf' should be forbidden. default: True - multiple_of: float - le: float - ge: float - lt: float - gt: float - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def float_schema( - *, - allow_inf_nan: bool | None = None, - multiple_of: float | None = None, - le: float | None = None, - ge: float | None = None, - lt: float | None = None, - gt: float | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> FloatSchema: - """ - Returns a schema that matches a float value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.float_schema(le=0.8, ge=0.2) - v = SchemaValidator(schema) - assert v.validate_python('0.5') == 0.5 - ``` - - Args: - allow_inf_nan: Whether to allow inf and nan values - multiple_of: The value must be a multiple of this number - le: The value must be less than or equal to this number - ge: The value must be greater than or equal to this number - lt: The value must be strictly less than this number - gt: The value must be strictly greater than this number - strict: Whether the value should be a float or a value that can be converted to a float - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='float', - allow_inf_nan=allow_inf_nan, - multiple_of=multiple_of, - le=le, - ge=ge, - lt=lt, - gt=gt, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class DecimalSchema(TypedDict, total=False): - type: Required[Literal['decimal']] - allow_inf_nan: bool # whether 'NaN', '+inf', '-inf' should be forbidden. default: False - multiple_of: Decimal - le: Decimal - ge: Decimal - lt: Decimal - gt: Decimal - max_digits: int - decimal_places: int - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def decimal_schema( - *, - allow_inf_nan: bool | None = None, - multiple_of: Decimal | None = None, - le: Decimal | None = None, - ge: Decimal | None = None, - lt: Decimal | None = None, - gt: Decimal | None = None, - max_digits: int | None = None, - decimal_places: int | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> DecimalSchema: - """ - Returns a schema that matches a decimal value, e.g.: - - ```py - from decimal import Decimal - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.decimal_schema(le=0.8, ge=0.2) - v = SchemaValidator(schema) - assert v.validate_python('0.5') == Decimal('0.5') - ``` - - Args: - allow_inf_nan: Whether to allow inf and nan values - multiple_of: The value must be a multiple of this number - le: The value must be less than or equal to this number - ge: The value must be greater than or equal to this number - lt: The value must be strictly less than this number - gt: The value must be strictly greater than this number - max_digits: The maximum number of decimal digits allowed - decimal_places: The maximum number of decimal places allowed - strict: Whether the value should be a float or a value that can be converted to a float - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='decimal', - gt=gt, - ge=ge, - lt=lt, - le=le, - max_digits=max_digits, - decimal_places=decimal_places, - multiple_of=multiple_of, - allow_inf_nan=allow_inf_nan, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class ComplexSchema(TypedDict, total=False): - type: Required[Literal['complex']] - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def complex_schema( - *, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> ComplexSchema: - """ - Returns a schema that matches a complex value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.complex_schema() - v = SchemaValidator(schema) - assert v.validate_python('1+2j') == complex(1, 2) - assert v.validate_python(complex(1, 2)) == complex(1, 2) - ``` - - Args: - strict: Whether the value should be a complex object instance or a value that can be converted to a complex object - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='complex', - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class StringSchema(TypedDict, total=False): - type: Required[Literal['str']] - pattern: Union[str, Pattern[str]] - max_length: int - min_length: int - strip_whitespace: bool - to_lower: bool - to_upper: bool - regex_engine: Literal['rust-regex', 'python-re'] # default: 'rust-regex' - strict: bool - coerce_numbers_to_str: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def str_schema( - *, - pattern: str | Pattern[str] | None = None, - max_length: int | None = None, - min_length: int | None = None, - strip_whitespace: bool | None = None, - to_lower: bool | None = None, - to_upper: bool | None = None, - regex_engine: Literal['rust-regex', 'python-re'] | None = None, - strict: bool | None = None, - coerce_numbers_to_str: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> StringSchema: - """ - Returns a schema that matches a string value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.str_schema(max_length=10, min_length=2) - v = SchemaValidator(schema) - assert v.validate_python('hello') == 'hello' - ``` - - Args: - pattern: A regex pattern that the value must match - max_length: The value must be at most this length - min_length: The value must be at least this length - strip_whitespace: Whether to strip whitespace from the value - to_lower: Whether to convert the value to lowercase - to_upper: Whether to convert the value to uppercase - regex_engine: The regex engine to use for pattern validation. Default is 'rust-regex'. - - `rust-regex` uses the [`regex`](https://docs.rs/regex) Rust - crate, which is non-backtracking and therefore more DDoS - resistant, but does not support all regex features. - - `python-re` use the [`re`](https://docs.python.org/3/library/re.html) module, - which supports all regex features, but may be slower. - strict: Whether the value should be a string or a value that can be converted to a string - coerce_numbers_to_str: Whether to enable coercion of any `Number` type to `str` (not applicable in `strict` mode). - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='str', - pattern=pattern, - max_length=max_length, - min_length=min_length, - strip_whitespace=strip_whitespace, - to_lower=to_lower, - to_upper=to_upper, - regex_engine=regex_engine, - strict=strict, - coerce_numbers_to_str=coerce_numbers_to_str, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class BytesSchema(TypedDict, total=False): - type: Required[Literal['bytes']] - max_length: int - min_length: int - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def bytes_schema( - *, - max_length: int | None = None, - min_length: int | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> BytesSchema: - """ - Returns a schema that matches a bytes value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.bytes_schema(max_length=10, min_length=2) - v = SchemaValidator(schema) - assert v.validate_python(b'hello') == b'hello' - ``` - - Args: - max_length: The value must be at most this length - min_length: The value must be at least this length - strict: Whether the value should be a bytes or a value that can be converted to a bytes - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='bytes', - max_length=max_length, - min_length=min_length, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class DateSchema(TypedDict, total=False): - type: Required[Literal['date']] - strict: bool - le: date - ge: date - lt: date - gt: date - now_op: Literal['past', 'future'] - # defaults to current local utc offset from `time.localtime().tm_gmtoff` - # value is restricted to -86_400 < offset < 86_400: - now_utc_offset: int - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def date_schema( - *, - strict: bool | None = None, - le: date | None = None, - ge: date | None = None, - lt: date | None = None, - gt: date | None = None, - now_op: Literal['past', 'future'] | None = None, - now_utc_offset: int | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> DateSchema: - """ - Returns a schema that matches a date value, e.g.: - - ```py - from datetime import date - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.date_schema(le=date(2020, 1, 1), ge=date(2019, 1, 1)) - v = SchemaValidator(schema) - assert v.validate_python(date(2019, 6, 1)) == date(2019, 6, 1) - ``` - - Args: - strict: Whether the value should be a date or a value that can be converted to a date - le: The value must be less than or equal to this date - ge: The value must be greater than or equal to this date - lt: The value must be strictly less than this date - gt: The value must be strictly greater than this date - now_op: The value must be in the past or future relative to the current date - now_utc_offset: The value must be in the past or future relative to the current date with this utc offset - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='date', - strict=strict, - le=le, - ge=ge, - lt=lt, - gt=gt, - now_op=now_op, - now_utc_offset=now_utc_offset, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class TimeSchema(TypedDict, total=False): - type: Required[Literal['time']] - strict: bool - le: time - ge: time - lt: time - gt: time - tz_constraint: Union[Literal['aware', 'naive'], int] - microseconds_precision: Literal['truncate', 'error'] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def time_schema( - *, - strict: bool | None = None, - le: time | None = None, - ge: time | None = None, - lt: time | None = None, - gt: time | None = None, - tz_constraint: Literal['aware', 'naive'] | int | None = None, - microseconds_precision: Literal['truncate', 'error'] = 'truncate', - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> TimeSchema: - """ - Returns a schema that matches a time value, e.g.: - - ```py - from datetime import time - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.time_schema(le=time(12, 0, 0), ge=time(6, 0, 0)) - v = SchemaValidator(schema) - assert v.validate_python(time(9, 0, 0)) == time(9, 0, 0) - ``` - - Args: - strict: Whether the value should be a time or a value that can be converted to a time - le: The value must be less than or equal to this time - ge: The value must be greater than or equal to this time - lt: The value must be strictly less than this time - gt: The value must be strictly greater than this time - tz_constraint: The value must be timezone aware or naive, or an int to indicate required tz offset - microseconds_precision: The behavior when seconds have more than 6 digits or microseconds is too large - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='time', - strict=strict, - le=le, - ge=ge, - lt=lt, - gt=gt, - tz_constraint=tz_constraint, - microseconds_precision=microseconds_precision, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class DatetimeSchema(TypedDict, total=False): - type: Required[Literal['datetime']] - strict: bool - le: datetime - ge: datetime - lt: datetime - gt: datetime - now_op: Literal['past', 'future'] - tz_constraint: Union[Literal['aware', 'naive'], int] - # defaults to current local utc offset from `time.localtime().tm_gmtoff` - # value is restricted to -86_400 < offset < 86_400 by bounds in generate_self_schema.py - now_utc_offset: int - microseconds_precision: Literal['truncate', 'error'] # default: 'truncate' - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def datetime_schema( - *, - strict: bool | None = None, - le: datetime | None = None, - ge: datetime | None = None, - lt: datetime | None = None, - gt: datetime | None = None, - now_op: Literal['past', 'future'] | None = None, - tz_constraint: Literal['aware', 'naive'] | int | None = None, - now_utc_offset: int | None = None, - microseconds_precision: Literal['truncate', 'error'] = 'truncate', - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> DatetimeSchema: - """ - Returns a schema that matches a datetime value, e.g.: - - ```py - from datetime import datetime - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.datetime_schema() - v = SchemaValidator(schema) - now = datetime.now() - assert v.validate_python(str(now)) == now - ``` - - Args: - strict: Whether the value should be a datetime or a value that can be converted to a datetime - le: The value must be less than or equal to this datetime - ge: The value must be greater than or equal to this datetime - lt: The value must be strictly less than this datetime - gt: The value must be strictly greater than this datetime - now_op: The value must be in the past or future relative to the current datetime - tz_constraint: The value must be timezone aware or naive, or an int to indicate required tz offset - TODO: use of a tzinfo where offset changes based on the datetime is not yet supported - now_utc_offset: The value must be in the past or future relative to the current datetime with this utc offset - microseconds_precision: The behavior when seconds have more than 6 digits or microseconds is too large - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='datetime', - strict=strict, - le=le, - ge=ge, - lt=lt, - gt=gt, - now_op=now_op, - tz_constraint=tz_constraint, - now_utc_offset=now_utc_offset, - microseconds_precision=microseconds_precision, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class TimedeltaSchema(TypedDict, total=False): - type: Required[Literal['timedelta']] - strict: bool - le: timedelta - ge: timedelta - lt: timedelta - gt: timedelta - microseconds_precision: Literal['truncate', 'error'] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def timedelta_schema( - *, - strict: bool | None = None, - le: timedelta | None = None, - ge: timedelta | None = None, - lt: timedelta | None = None, - gt: timedelta | None = None, - microseconds_precision: Literal['truncate', 'error'] = 'truncate', - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> TimedeltaSchema: - """ - Returns a schema that matches a timedelta value, e.g.: - - ```py - from datetime import timedelta - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.timedelta_schema(le=timedelta(days=1), ge=timedelta(days=0)) - v = SchemaValidator(schema) - assert v.validate_python(timedelta(hours=12)) == timedelta(hours=12) - ``` - - Args: - strict: Whether the value should be a timedelta or a value that can be converted to a timedelta - le: The value must be less than or equal to this timedelta - ge: The value must be greater than or equal to this timedelta - lt: The value must be strictly less than this timedelta - gt: The value must be strictly greater than this timedelta - microseconds_precision: The behavior when seconds have more than 6 digits or microseconds is too large - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='timedelta', - strict=strict, - le=le, - ge=ge, - lt=lt, - gt=gt, - microseconds_precision=microseconds_precision, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class LiteralSchema(TypedDict, total=False): - type: Required[Literal['literal']] - expected: Required[list[Any]] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def literal_schema( - expected: list[Any], - *, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> LiteralSchema: - """ - Returns a schema that matches a literal value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.literal_schema(['hello', 'world']) - v = SchemaValidator(schema) - assert v.validate_python('hello') == 'hello' - ``` - - Args: - expected: The value must be one of these values - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='literal', expected=expected, ref=ref, metadata=metadata, serialization=serialization) - - -class EnumSchema(TypedDict, total=False): - type: Required[Literal['enum']] - cls: Required[Any] - members: Required[list[Any]] - sub_type: Literal['str', 'int', 'float'] - missing: Callable[[Any], Any] - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def enum_schema( - cls: Any, - members: list[Any], - *, - sub_type: Literal['str', 'int', 'float'] | None = None, - missing: Callable[[Any], Any] | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> EnumSchema: - """ - Returns a schema that matches an enum value, e.g.: - - ```py - from enum import Enum - from pydantic_core import SchemaValidator, core_schema - - class Color(Enum): - RED = 1 - GREEN = 2 - BLUE = 3 - - schema = core_schema.enum_schema(Color, list(Color.__members__.values())) - v = SchemaValidator(schema) - assert v.validate_python(2) is Color.GREEN - ``` - - Args: - cls: The enum class - members: The members of the enum, generally `list(MyEnum.__members__.values())` - sub_type: The type of the enum, either 'str' or 'int' or None for plain enums - missing: A function to use when the value is not found in the enum, from `_missing_` - strict: Whether to use strict mode, defaults to False - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='enum', - cls=cls, - members=members, - sub_type=sub_type, - missing=missing, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class MissingSentinelSchema(TypedDict, total=False): - type: Required[Literal['missing-sentinel']] - metadata: dict[str, Any] - serialization: SerSchema - - -def missing_sentinel_schema( - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> MissingSentinelSchema: - """Returns a schema for the `MISSING` sentinel.""" - - return _dict_not_none( - type='missing-sentinel', - metadata=metadata, - serialization=serialization, - ) - - -# must match input/parse_json.rs::JsonType::try_from -JsonType = Literal['null', 'bool', 'int', 'float', 'str', 'list', 'dict'] - - -class IsInstanceSchema(TypedDict, total=False): - type: Required[Literal['is-instance']] - cls: Required[Any] - cls_repr: str - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def is_instance_schema( - cls: Any, - *, - cls_repr: str | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> IsInstanceSchema: - """ - Returns a schema that checks if a value is an instance of a class, equivalent to python's `isinstance` method, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - class A: - pass - - schema = core_schema.is_instance_schema(cls=A) - v = SchemaValidator(schema) - v.validate_python(A()) - ``` - - Args: - cls: The value must be an instance of this class - cls_repr: If provided this string is used in the validator name instead of `repr(cls)` - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='is-instance', cls=cls, cls_repr=cls_repr, ref=ref, metadata=metadata, serialization=serialization - ) - - -class IsSubclassSchema(TypedDict, total=False): - type: Required[Literal['is-subclass']] - cls: Required[type[Any]] - cls_repr: str - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def is_subclass_schema( - cls: type[Any], - *, - cls_repr: str | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> IsInstanceSchema: - """ - Returns a schema that checks if a value is a subtype of a class, equivalent to python's `issubclass` method, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - class A: - pass - - class B(A): - pass - - schema = core_schema.is_subclass_schema(cls=A) - v = SchemaValidator(schema) - v.validate_python(B) - ``` - - Args: - cls: The value must be a subclass of this class - cls_repr: If provided this string is used in the validator name instead of `repr(cls)` - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='is-subclass', cls=cls, cls_repr=cls_repr, ref=ref, metadata=metadata, serialization=serialization - ) - - -class CallableSchema(TypedDict, total=False): - type: Required[Literal['callable']] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def callable_schema( - *, ref: str | None = None, metadata: dict[str, Any] | None = None, serialization: SerSchema | None = None -) -> CallableSchema: - """ - Returns a schema that checks if a value is callable, equivalent to python's `callable` method, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.callable_schema() - v = SchemaValidator(schema) - v.validate_python(min) - ``` - - Args: - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='callable', ref=ref, metadata=metadata, serialization=serialization) - - -class UuidSchema(TypedDict, total=False): - type: Required[Literal['uuid']] - version: Literal[1, 3, 4, 5, 7] - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def uuid_schema( - *, - version: Literal[1, 3, 4, 5, 6, 7, 8] | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> UuidSchema: - return _dict_not_none( - type='uuid', version=version, strict=strict, ref=ref, metadata=metadata, serialization=serialization - ) - - -class IncExSeqSerSchema(TypedDict, total=False): - type: Required[Literal['include-exclude-sequence']] - include: set[int] - exclude: set[int] - - -def filter_seq_schema(*, include: set[int] | None = None, exclude: set[int] | None = None) -> IncExSeqSerSchema: - return _dict_not_none(type='include-exclude-sequence', include=include, exclude=exclude) - - -IncExSeqOrElseSerSchema = Union[IncExSeqSerSchema, SerSchema] - - -class ListSchema(TypedDict, total=False): - type: Required[Literal['list']] - items_schema: CoreSchema - min_length: int - max_length: int - fail_fast: bool - strict: bool - ref: str - metadata: dict[str, Any] - serialization: IncExSeqOrElseSerSchema - - -def list_schema( - items_schema: CoreSchema | None = None, - *, - min_length: int | None = None, - max_length: int | None = None, - fail_fast: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: IncExSeqOrElseSerSchema | None = None, -) -> ListSchema: - """ - Returns a schema that matches a list value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.list_schema(core_schema.int_schema(), min_length=0, max_length=10) - v = SchemaValidator(schema) - assert v.validate_python(['4']) == [4] - ``` - - Args: - items_schema: The value must be a list of items that match this schema - min_length: The value must be a list with at least this many items - max_length: The value must be a list with at most this many items - fail_fast: Stop validation on the first error - strict: The value must be a list with exactly this many items - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='list', - items_schema=items_schema, - min_length=min_length, - max_length=max_length, - fail_fast=fail_fast, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -# @deprecated('tuple_positional_schema is deprecated. Use pydantic_core.core_schema.tuple_schema instead.') -def tuple_positional_schema( - items_schema: list[CoreSchema], - *, - extras_schema: CoreSchema | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: IncExSeqOrElseSerSchema | None = None, -) -> TupleSchema: - """ - Returns a schema that matches a tuple of schemas, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.tuple_positional_schema( - [core_schema.int_schema(), core_schema.str_schema()] - ) - v = SchemaValidator(schema) - assert v.validate_python((1, 'hello')) == (1, 'hello') - ``` - - Args: - items_schema: The value must be a tuple with items that match these schemas - extras_schema: The value must be a tuple with items that match this schema - This was inspired by JSON schema's `prefixItems` and `items` fields. - In python's `typing.Tuple`, you can't specify a type for "extra" items -- they must all be the same type - if the length is variable. So this field won't be set from a `typing.Tuple` annotation on a pydantic model. - strict: The value must be a tuple with exactly this many items - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - if extras_schema is not None: - variadic_item_index = len(items_schema) - items_schema = items_schema + [extras_schema] - else: - variadic_item_index = None - return tuple_schema( - items_schema=items_schema, - variadic_item_index=variadic_item_index, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -# @deprecated('tuple_variable_schema is deprecated. Use pydantic_core.core_schema.tuple_schema instead.') -def tuple_variable_schema( - items_schema: CoreSchema | None = None, - *, - min_length: int | None = None, - max_length: int | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: IncExSeqOrElseSerSchema | None = None, -) -> TupleSchema: - """ - Returns a schema that matches a tuple of a given schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.tuple_variable_schema( - items_schema=core_schema.int_schema(), min_length=0, max_length=10 - ) - v = SchemaValidator(schema) - assert v.validate_python(('1', 2, 3)) == (1, 2, 3) - ``` - - Args: - items_schema: The value must be a tuple with items that match this schema - min_length: The value must be a tuple with at least this many items - max_length: The value must be a tuple with at most this many items - strict: The value must be a tuple with exactly this many items - ref: Optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return tuple_schema( - items_schema=[items_schema or any_schema()], - variadic_item_index=0, - min_length=min_length, - max_length=max_length, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class TupleSchema(TypedDict, total=False): - type: Required[Literal['tuple']] - items_schema: Required[list[CoreSchema]] - variadic_item_index: int - min_length: int - max_length: int - fail_fast: bool - strict: bool - ref: str - metadata: dict[str, Any] - serialization: IncExSeqOrElseSerSchema - - -def tuple_schema( - items_schema: list[CoreSchema], - *, - variadic_item_index: int | None = None, - min_length: int | None = None, - max_length: int | None = None, - fail_fast: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: IncExSeqOrElseSerSchema | None = None, -) -> TupleSchema: - """ - Returns a schema that matches a tuple of schemas, with an optional variadic item, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.tuple_schema( - [core_schema.int_schema(), core_schema.str_schema(), core_schema.float_schema()], - variadic_item_index=1, - ) - v = SchemaValidator(schema) - assert v.validate_python((1, 'hello', 'world', 1.5)) == (1, 'hello', 'world', 1.5) - ``` - - Args: - items_schema: The value must be a tuple with items that match these schemas - variadic_item_index: The index of the schema in `items_schema` to be treated as variadic (following PEP 646) - min_length: The value must be a tuple with at least this many items - max_length: The value must be a tuple with at most this many items - fail_fast: Stop validation on the first error - strict: The value must be a tuple with exactly this many items - ref: Optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='tuple', - items_schema=items_schema, - variadic_item_index=variadic_item_index, - min_length=min_length, - max_length=max_length, - fail_fast=fail_fast, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class SetSchema(TypedDict, total=False): - type: Required[Literal['set']] - items_schema: CoreSchema - min_length: int - max_length: int - fail_fast: bool - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def set_schema( - items_schema: CoreSchema | None = None, - *, - min_length: int | None = None, - max_length: int | None = None, - fail_fast: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> SetSchema: - """ - Returns a schema that matches a set of a given schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.set_schema( - items_schema=core_schema.int_schema(), min_length=0, max_length=10 - ) - v = SchemaValidator(schema) - assert v.validate_python({1, '2', 3}) == {1, 2, 3} - ``` - - Args: - items_schema: The value must be a set with items that match this schema - min_length: The value must be a set with at least this many items - max_length: The value must be a set with at most this many items - fail_fast: Stop validation on the first error - strict: The value must be a set with exactly this many items - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='set', - items_schema=items_schema, - min_length=min_length, - max_length=max_length, - fail_fast=fail_fast, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class FrozenSetSchema(TypedDict, total=False): - type: Required[Literal['frozenset']] - items_schema: CoreSchema - min_length: int - max_length: int - fail_fast: bool - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def frozenset_schema( - items_schema: CoreSchema | None = None, - *, - min_length: int | None = None, - max_length: int | None = None, - fail_fast: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> FrozenSetSchema: - """ - Returns a schema that matches a frozenset of a given schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.frozenset_schema( - items_schema=core_schema.int_schema(), min_length=0, max_length=10 - ) - v = SchemaValidator(schema) - assert v.validate_python(frozenset(range(3))) == frozenset({0, 1, 2}) - ``` - - Args: - items_schema: The value must be a frozenset with items that match this schema - min_length: The value must be a frozenset with at least this many items - max_length: The value must be a frozenset with at most this many items - fail_fast: Stop validation on the first error - strict: The value must be a frozenset with exactly this many items - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='frozenset', - items_schema=items_schema, - min_length=min_length, - max_length=max_length, - fail_fast=fail_fast, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class GeneratorSchema(TypedDict, total=False): - type: Required[Literal['generator']] - items_schema: CoreSchema - min_length: int - max_length: int - ref: str - metadata: dict[str, Any] - serialization: IncExSeqOrElseSerSchema - - -def generator_schema( - items_schema: CoreSchema | None = None, - *, - min_length: int | None = None, - max_length: int | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: IncExSeqOrElseSerSchema | None = None, -) -> GeneratorSchema: - """ - Returns a schema that matches a generator value, e.g.: - - ```py - from typing import Iterator - from pydantic_core import SchemaValidator, core_schema - - def gen() -> Iterator[int]: - yield 1 - - schema = core_schema.generator_schema(items_schema=core_schema.int_schema()) - v = SchemaValidator(schema) - v.validate_python(gen()) - ``` - - Unlike other types, validated generators do not raise ValidationErrors eagerly, - but instead will raise a ValidationError when a violating value is actually read from the generator. - This is to ensure that "validated" generators retain the benefit of lazy evaluation. - - Args: - items_schema: The value must be a generator with items that match this schema - min_length: The value must be a generator that yields at least this many items - max_length: The value must be a generator that yields at most this many items - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='generator', - items_schema=items_schema, - min_length=min_length, - max_length=max_length, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -IncExDict = set[Union[int, str]] - - -class IncExDictSerSchema(TypedDict, total=False): - type: Required[Literal['include-exclude-dict']] - include: IncExDict - exclude: IncExDict - - -def filter_dict_schema(*, include: IncExDict | None = None, exclude: IncExDict | None = None) -> IncExDictSerSchema: - return _dict_not_none(type='include-exclude-dict', include=include, exclude=exclude) - - -IncExDictOrElseSerSchema = Union[IncExDictSerSchema, SerSchema] - - -class DictSchema(TypedDict, total=False): - type: Required[Literal['dict']] - keys_schema: CoreSchema # default: AnySchema - values_schema: CoreSchema # default: AnySchema - min_length: int - max_length: int - fail_fast: bool - strict: bool - ref: str - metadata: dict[str, Any] - serialization: IncExDictOrElseSerSchema - - -def dict_schema( - keys_schema: CoreSchema | None = None, - values_schema: CoreSchema | None = None, - *, - min_length: int | None = None, - max_length: int | None = None, - fail_fast: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> DictSchema: - """ - Returns a schema that matches a dict value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.dict_schema( - keys_schema=core_schema.str_schema(), values_schema=core_schema.int_schema() - ) - v = SchemaValidator(schema) - assert v.validate_python({'a': '1', 'b': 2}) == {'a': 1, 'b': 2} - ``` - - Args: - keys_schema: The value must be a dict with keys that match this schema - values_schema: The value must be a dict with values that match this schema - min_length: The value must be a dict with at least this many items - max_length: The value must be a dict with at most this many items - fail_fast: Stop validation on the first error - strict: Whether the keys and values should be validated with strict mode - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='dict', - keys_schema=keys_schema, - values_schema=values_schema, - min_length=min_length, - max_length=max_length, - fail_fast=fail_fast, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -# (input_value: Any, /) -> Any -NoInfoValidatorFunction = Callable[[Any], Any] - - -class NoInfoValidatorFunctionSchema(TypedDict): - type: Literal['no-info'] - function: NoInfoValidatorFunction - - -# (input_value: Any, info: ValidationInfo, /) -> Any -WithInfoValidatorFunction = Callable[[Any, ValidationInfo[Any]], Any] - - -class WithInfoValidatorFunctionSchema(TypedDict, total=False): - type: Required[Literal['with-info']] - function: Required[WithInfoValidatorFunction] - field_name: str # deprecated - - -ValidationFunction = Union[NoInfoValidatorFunctionSchema, WithInfoValidatorFunctionSchema] - - -class _ValidatorFunctionSchema(TypedDict, total=False): - function: Required[ValidationFunction] - schema: Required[CoreSchema] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -class BeforeValidatorFunctionSchema(_ValidatorFunctionSchema, total=False): - type: Required[Literal['function-before']] - json_schema_input_schema: CoreSchema - - -def no_info_before_validator_function( - function: NoInfoValidatorFunction, - schema: CoreSchema, - *, - ref: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> BeforeValidatorFunctionSchema: - """ - Returns a schema that calls a validator function before validating, no `info` argument is provided, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: bytes) -> str: - return v.decode() + 'world' - - func_schema = core_schema.no_info_before_validator_function( - function=fn, schema=core_schema.str_schema() - ) - schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) - - v = SchemaValidator(schema) - assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} - ``` - - Args: - function: The validator function to call - schema: The schema to validate the output of the validator function - ref: optional unique identifier of the schema, used to reference the schema in other places - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='function-before', - function={'type': 'no-info', 'function': function}, - schema=schema, - ref=ref, - json_schema_input_schema=json_schema_input_schema, - metadata=metadata, - serialization=serialization, - ) - - -def with_info_before_validator_function( - function: WithInfoValidatorFunction, - schema: CoreSchema, - *, - field_name: str | None = None, - ref: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> BeforeValidatorFunctionSchema: - """ - Returns a schema that calls a validator function before validation, the function is called with - an `info` argument, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: bytes, info: core_schema.ValidationInfo) -> str: - assert info.data is not None - assert info.field_name is not None - return v.decode() + 'world' - - func_schema = core_schema.with_info_before_validator_function( - function=fn, schema=core_schema.str_schema() - ) - schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) - - v = SchemaValidator(schema) - assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} - ``` - - Args: - function: The validator function to call - field_name: The name of the field this validator is applied to, if any (deprecated) - schema: The schema to validate the output of the validator function - ref: optional unique identifier of the schema, used to reference the schema in other places - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - if field_name is not None: - warnings.warn( - 'The `field_name` argument on `with_info_before_validator_function` is deprecated, it will be passed to the function through `ValidationState` instead.', - DeprecationWarning, - stacklevel=2, - ) - - return _dict_not_none( - type='function-before', - function=_dict_not_none(type='with-info', function=function, field_name=field_name), - schema=schema, - ref=ref, - json_schema_input_schema=json_schema_input_schema, - metadata=metadata, - serialization=serialization, - ) - - -class AfterValidatorFunctionSchema(_ValidatorFunctionSchema, total=False): - type: Required[Literal['function-after']] - - -def no_info_after_validator_function( - function: NoInfoValidatorFunction, - schema: CoreSchema, - *, - ref: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> AfterValidatorFunctionSchema: - """ - Returns a schema that calls a validator function after validating, no `info` argument is provided, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: str) -> str: - return v + 'world' - - func_schema = core_schema.no_info_after_validator_function(fn, core_schema.str_schema()) - schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) - - v = SchemaValidator(schema) - assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} - ``` - - Args: - function: The validator function to call after the schema is validated - schema: The schema to validate before the validator function - ref: optional unique identifier of the schema, used to reference the schema in other places - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='function-after', - function={'type': 'no-info', 'function': function}, - schema=schema, - ref=ref, - json_schema_input_schema=json_schema_input_schema, - metadata=metadata, - serialization=serialization, - ) - - -def with_info_after_validator_function( - function: WithInfoValidatorFunction, - schema: CoreSchema, - *, - field_name: str | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> AfterValidatorFunctionSchema: - """ - Returns a schema that calls a validator function after validation, the function is called with - an `info` argument, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: str, info: core_schema.ValidationInfo) -> str: - assert info.data is not None - assert info.field_name is not None - return v + 'world' - - func_schema = core_schema.with_info_after_validator_function( - function=fn, schema=core_schema.str_schema() - ) - schema = core_schema.typed_dict_schema({'a': core_schema.typed_dict_field(func_schema)}) - - v = SchemaValidator(schema) - assert v.validate_python({'a': b'hello '}) == {'a': 'hello world'} - ``` - - Args: - function: The validator function to call after the schema is validated - schema: The schema to validate before the validator function - field_name: The name of the field this validator is applied to, if any (deprecated) - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - if field_name is not None: - warnings.warn( - 'The `field_name` argument on `with_info_after_validator_function` is deprecated, it will be passed to the function through `ValidationState` instead.', - DeprecationWarning, - stacklevel=2, - ) - - return _dict_not_none( - type='function-after', - function=_dict_not_none(type='with-info', function=function, field_name=field_name), - schema=schema, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class ValidatorFunctionWrapHandler(Protocol): - def __call__(self, input_value: Any, outer_location: str | int | None = None, /) -> Any: # pragma: no cover - ... - - -# (input_value: Any, validator: ValidatorFunctionWrapHandler, /) -> Any -NoInfoWrapValidatorFunction = Callable[[Any, ValidatorFunctionWrapHandler], Any] - - -class NoInfoWrapValidatorFunctionSchema(TypedDict): - type: Literal['no-info'] - function: NoInfoWrapValidatorFunction - - -# (input_value: Any, validator: ValidatorFunctionWrapHandler, info: ValidationInfo, /) -> Any -WithInfoWrapValidatorFunction = Callable[[Any, ValidatorFunctionWrapHandler, ValidationInfo[Any]], Any] - - -class WithInfoWrapValidatorFunctionSchema(TypedDict, total=False): - type: Required[Literal['with-info']] - function: Required[WithInfoWrapValidatorFunction] - field_name: str # deprecated - - -WrapValidatorFunction = Union[NoInfoWrapValidatorFunctionSchema, WithInfoWrapValidatorFunctionSchema] - - -class WrapValidatorFunctionSchema(TypedDict, total=False): - type: Required[Literal['function-wrap']] - function: Required[WrapValidatorFunction] - schema: Required[CoreSchema] - ref: str - json_schema_input_schema: CoreSchema - metadata: dict[str, Any] - serialization: SerSchema - - -def no_info_wrap_validator_function( - function: NoInfoWrapValidatorFunction, - schema: CoreSchema, - *, - ref: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> WrapValidatorFunctionSchema: - """ - Returns a schema which calls a function with a `validator` callable argument which can - optionally be used to call inner validation with the function logic, this is much like the - "onion" implementation of middleware in many popular web frameworks, no `info` argument is passed, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn( - v: str, - validator: core_schema.ValidatorFunctionWrapHandler, - ) -> str: - return validator(input_value=v) + 'world' - - schema = core_schema.no_info_wrap_validator_function( - function=fn, schema=core_schema.str_schema() - ) - v = SchemaValidator(schema) - assert v.validate_python('hello ') == 'hello world' - ``` - - Args: - function: The validator function to call - schema: The schema to validate the output of the validator function - ref: optional unique identifier of the schema, used to reference the schema in other places - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='function-wrap', - function={'type': 'no-info', 'function': function}, - schema=schema, - json_schema_input_schema=json_schema_input_schema, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -def with_info_wrap_validator_function( - function: WithInfoWrapValidatorFunction, - schema: CoreSchema, - *, - field_name: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> WrapValidatorFunctionSchema: - """ - Returns a schema which calls a function with a `validator` callable argument which can - optionally be used to call inner validation with the function logic, this is much like the - "onion" implementation of middleware in many popular web frameworks, an `info` argument is also passed, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn( - v: str, - validator: core_schema.ValidatorFunctionWrapHandler, - info: core_schema.ValidationInfo, - ) -> str: - return validator(input_value=v) + 'world' - - schema = core_schema.with_info_wrap_validator_function( - function=fn, schema=core_schema.str_schema() - ) - v = SchemaValidator(schema) - assert v.validate_python('hello ') == 'hello world' - ``` - - Args: - function: The validator function to call - schema: The schema to validate the output of the validator function - field_name: The name of the field this validator is applied to, if any (deprecated) - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - if field_name is not None: - warnings.warn( - 'The `field_name` argument on `with_info_wrap_validator_function` is deprecated, it will be passed to the function through `ValidationState` instead.', - DeprecationWarning, - stacklevel=2, - ) - - return _dict_not_none( - type='function-wrap', - function=_dict_not_none(type='with-info', function=function, field_name=field_name), - schema=schema, - json_schema_input_schema=json_schema_input_schema, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class PlainValidatorFunctionSchema(TypedDict, total=False): - type: Required[Literal['function-plain']] - function: Required[ValidationFunction] - ref: str - json_schema_input_schema: CoreSchema - metadata: dict[str, Any] - serialization: SerSchema - - -def no_info_plain_validator_function( - function: NoInfoValidatorFunction, - *, - ref: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> PlainValidatorFunctionSchema: - """ - Returns a schema that uses the provided function for validation, no `info` argument is passed, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: str) -> str: - assert 'hello' in v - return v + 'world' - - schema = core_schema.no_info_plain_validator_function(function=fn) - v = SchemaValidator(schema) - assert v.validate_python('hello ') == 'hello world' - ``` - - Args: - function: The validator function to call - ref: optional unique identifier of the schema, used to reference the schema in other places - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='function-plain', - function={'type': 'no-info', 'function': function}, - ref=ref, - json_schema_input_schema=json_schema_input_schema, - metadata=metadata, - serialization=serialization, - ) - - -def with_info_plain_validator_function( - function: WithInfoValidatorFunction, - *, - field_name: str | None = None, - ref: str | None = None, - json_schema_input_schema: CoreSchema | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> PlainValidatorFunctionSchema: - """ - Returns a schema that uses the provided function for validation, an `info` argument is passed, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: str, info: core_schema.ValidationInfo) -> str: - assert 'hello' in v - return v + 'world' - - schema = core_schema.with_info_plain_validator_function(function=fn) - v = SchemaValidator(schema) - assert v.validate_python('hello ') == 'hello world' - ``` - - Args: - function: The validator function to call - field_name: The name of the field this validator is applied to, if any (deprecated) - ref: optional unique identifier of the schema, used to reference the schema in other places - json_schema_input_schema: The core schema to be used to generate the corresponding JSON Schema input type - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - if field_name is not None: - warnings.warn( - 'The `field_name` argument on `with_info_plain_validator_function` is deprecated, it will be passed to the function through `ValidationState` instead.', - DeprecationWarning, - stacklevel=2, - ) - - return _dict_not_none( - type='function-plain', - function=_dict_not_none(type='with-info', function=function, field_name=field_name), - ref=ref, - json_schema_input_schema=json_schema_input_schema, - metadata=metadata, - serialization=serialization, - ) - - -class WithDefaultSchema(TypedDict, total=False): - type: Required[Literal['default']] - schema: Required[CoreSchema] - default: Any - default_factory: Union[Callable[[], Any], Callable[[dict[str, Any]], Any]] - default_factory_takes_data: bool - on_error: Literal['raise', 'omit', 'default'] # default: 'raise' - validate_default: bool # default: False - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def with_default_schema( - schema: CoreSchema, - *, - default: Any = PydanticUndefined, - default_factory: Union[Callable[[], Any], Callable[[dict[str, Any]], Any], None] = None, - default_factory_takes_data: bool | None = None, - on_error: Literal['raise', 'omit', 'default'] | None = None, - validate_default: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> WithDefaultSchema: - """ - Returns a schema that adds a default value to the given schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.with_default_schema(core_schema.str_schema(), default='hello') - wrapper_schema = core_schema.typed_dict_schema( - {'a': core_schema.typed_dict_field(schema)} - ) - v = SchemaValidator(wrapper_schema) - assert v.validate_python({}) == v.validate_python({'a': 'hello'}) - ``` - - Args: - schema: The schema to add a default value to - default: The default value to use - default_factory: A callable that returns the default value to use - default_factory_takes_data: Whether the default factory takes a validated data argument - on_error: What to do if the schema validation fails. One of 'raise', 'omit', 'default' - validate_default: Whether the default value should be validated - strict: Whether the underlying schema should be validated with strict mode - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - s = _dict_not_none( - type='default', - schema=schema, - default_factory=default_factory, - default_factory_takes_data=default_factory_takes_data, - on_error=on_error, - validate_default=validate_default, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - if default is not PydanticUndefined: - s['default'] = default - return s - - -class NullableSchema(TypedDict, total=False): - type: Required[Literal['nullable']] - schema: Required[CoreSchema] - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def nullable_schema( - schema: CoreSchema, - *, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> NullableSchema: - """ - Returns a schema that matches a nullable value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.nullable_schema(core_schema.str_schema()) - v = SchemaValidator(schema) - assert v.validate_python(None) is None - ``` - - Args: - schema: The schema to wrap - strict: Whether the underlying schema should be validated with strict mode - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='nullable', schema=schema, strict=strict, ref=ref, metadata=metadata, serialization=serialization - ) - - -class UnionSchema(TypedDict, total=False): - type: Required[Literal['union']] - choices: Required[list[Union[CoreSchema, tuple[CoreSchema, str]]]] - # default true, whether to automatically collapse unions with one element to the inner validator - auto_collapse: bool - custom_error_type: str - custom_error_message: str - custom_error_context: dict[str, Union[str, int, float]] - mode: Literal['smart', 'left_to_right'] # default: 'smart' - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def union_schema( - choices: list[CoreSchema | tuple[CoreSchema, str]], - *, - auto_collapse: bool | None = None, - custom_error_type: str | None = None, - custom_error_message: str | None = None, - custom_error_context: dict[str, str | int] | None = None, - mode: Literal['smart', 'left_to_right'] | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> UnionSchema: - """ - Returns a schema that matches a union value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.union_schema([core_schema.str_schema(), core_schema.int_schema()]) - v = SchemaValidator(schema) - assert v.validate_python('hello') == 'hello' - assert v.validate_python(1) == 1 - ``` - - Args: - choices: The schemas to match. If a tuple, the second item is used as the label for the case. - auto_collapse: whether to automatically collapse unions with one element to the inner validator, default true - custom_error_type: The custom error type to use if the validation fails - custom_error_message: The custom error message to use if the validation fails - custom_error_context: The custom error context to use if the validation fails - mode: How to select which choice to return - * `smart` (default) will try to return the choice which is the closest match to the input value - * `left_to_right` will return the first choice in `choices` which succeeds validation - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='union', - choices=choices, - auto_collapse=auto_collapse, - custom_error_type=custom_error_type, - custom_error_message=custom_error_message, - custom_error_context=custom_error_context, - mode=mode, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class TaggedUnionSchema(TypedDict, total=False): - type: Required[Literal['tagged-union']] - choices: Required[dict[Hashable, CoreSchema]] - discriminator: Required[Union[str, list[Union[str, int]], list[list[Union[str, int]]], Callable[[Any], Hashable]]] - custom_error_type: str - custom_error_message: str - custom_error_context: dict[str, Union[str, int, float]] - strict: bool - from_attributes: bool # default: True - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def tagged_union_schema( - choices: dict[Any, CoreSchema], - discriminator: str | list[str | int] | list[list[str | int]] | Callable[[Any], Any], - *, - custom_error_type: str | None = None, - custom_error_message: str | None = None, - custom_error_context: dict[str, int | str | float] | None = None, - strict: bool | None = None, - from_attributes: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> TaggedUnionSchema: - """ - Returns a schema that matches a tagged union value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - apple_schema = core_schema.typed_dict_schema( - { - 'foo': core_schema.typed_dict_field(core_schema.str_schema()), - 'bar': core_schema.typed_dict_field(core_schema.int_schema()), - } - ) - banana_schema = core_schema.typed_dict_schema( - { - 'foo': core_schema.typed_dict_field(core_schema.str_schema()), - 'spam': core_schema.typed_dict_field( - core_schema.list_schema(items_schema=core_schema.int_schema()) - ), - } - ) - schema = core_schema.tagged_union_schema( - choices={ - 'apple': apple_schema, - 'banana': banana_schema, - }, - discriminator='foo', - ) - v = SchemaValidator(schema) - assert v.validate_python({'foo': 'apple', 'bar': '123'}) == {'foo': 'apple', 'bar': 123} - assert v.validate_python({'foo': 'banana', 'spam': [1, 2, 3]}) == { - 'foo': 'banana', - 'spam': [1, 2, 3], - } - ``` - - Args: - choices: The schemas to match - When retrieving a schema from `choices` using the discriminator value, if the value is a str, - it should be fed back into the `choices` map until a schema is obtained - (This approach is to prevent multiple ownership of a single schema in Rust) - discriminator: The discriminator to use to determine the schema to use - * If `discriminator` is a str, it is the name of the attribute to use as the discriminator - * If `discriminator` is a list of int/str, it should be used as a "path" to access the discriminator - * If `discriminator` is a list of lists, each inner list is a path, and the first path that exists is used - * If `discriminator` is a callable, it should return the discriminator when called on the value to validate; - the callable can return `None` to indicate that there is no matching discriminator present on the input - custom_error_type: The custom error type to use if the validation fails - custom_error_message: The custom error message to use if the validation fails - custom_error_context: The custom error context to use if the validation fails - strict: Whether the underlying schemas should be validated with strict mode - from_attributes: Whether to use the attributes of the object to retrieve the discriminator value - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='tagged-union', - choices=choices, - discriminator=discriminator, - custom_error_type=custom_error_type, - custom_error_message=custom_error_message, - custom_error_context=custom_error_context, - strict=strict, - from_attributes=from_attributes, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class ChainSchema(TypedDict, total=False): - type: Required[Literal['chain']] - steps: Required[list[CoreSchema]] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def chain_schema( - steps: list[CoreSchema], - *, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> ChainSchema: - """ - Returns a schema that chains the provided validation schemas, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: str, info: core_schema.ValidationInfo) -> str: - assert 'hello' in v - return v + ' world' - - fn_schema = core_schema.with_info_plain_validator_function(function=fn) - schema = core_schema.chain_schema( - [fn_schema, fn_schema, fn_schema, core_schema.str_schema()] - ) - v = SchemaValidator(schema) - assert v.validate_python('hello') == 'hello world world world' - ``` - - Args: - steps: The schemas to chain - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='chain', steps=steps, ref=ref, metadata=metadata, serialization=serialization) - - -class LaxOrStrictSchema(TypedDict, total=False): - type: Required[Literal['lax-or-strict']] - lax_schema: Required[CoreSchema] - strict_schema: Required[CoreSchema] - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def lax_or_strict_schema( - lax_schema: CoreSchema, - strict_schema: CoreSchema, - *, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> LaxOrStrictSchema: - """ - Returns a schema that uses the lax or strict schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - def fn(v: str, info: core_schema.ValidationInfo) -> str: - assert 'hello' in v - return v + ' world' - - lax_schema = core_schema.int_schema(strict=False) - strict_schema = core_schema.int_schema(strict=True) - - schema = core_schema.lax_or_strict_schema( - lax_schema=lax_schema, strict_schema=strict_schema, strict=True - ) - v = SchemaValidator(schema) - assert v.validate_python(123) == 123 - - schema = core_schema.lax_or_strict_schema( - lax_schema=lax_schema, strict_schema=strict_schema, strict=False - ) - v = SchemaValidator(schema) - assert v.validate_python('123') == 123 - ``` - - Args: - lax_schema: The lax schema to use - strict_schema: The strict schema to use - strict: Whether the strict schema should be used - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='lax-or-strict', - lax_schema=lax_schema, - strict_schema=strict_schema, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class JsonOrPythonSchema(TypedDict, total=False): - type: Required[Literal['json-or-python']] - json_schema: Required[CoreSchema] - python_schema: Required[CoreSchema] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def json_or_python_schema( - json_schema: CoreSchema, - python_schema: CoreSchema, - *, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> JsonOrPythonSchema: - """ - Returns a schema that uses the Json or Python schema depending on the input: - - ```py - from pydantic_core import SchemaValidator, ValidationError, core_schema - - v = SchemaValidator( - core_schema.json_or_python_schema( - json_schema=core_schema.int_schema(), - python_schema=core_schema.int_schema(strict=True), - ) - ) - - assert v.validate_json('"123"') == 123 - - try: - v.validate_python('123') - except ValidationError: - pass - else: - raise AssertionError('Validation should have failed') - ``` - - Args: - json_schema: The schema to use for Json inputs - python_schema: The schema to use for Python inputs - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='json-or-python', - json_schema=json_schema, - python_schema=python_schema, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class TypedDictField(TypedDict, total=False): - type: Required[Literal['typed-dict-field']] - schema: Required[CoreSchema] - required: bool - validation_alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] - serialization_alias: str - serialization_exclude: bool # default: False - metadata: dict[str, Any] - serialization_exclude_if: Callable[[Any], bool] # default None - - -def typed_dict_field( - schema: CoreSchema, - *, - required: bool | None = None, - validation_alias: str | list[str | int] | list[list[str | int]] | None = None, - serialization_alias: str | None = None, - serialization_exclude: bool | None = None, - metadata: dict[str, Any] | None = None, - serialization_exclude_if: Callable[[Any], bool] | None = None, -) -> TypedDictField: - """ - Returns a schema that matches a typed dict field, e.g.: - - ```py - from pydantic_core import core_schema - - field = core_schema.typed_dict_field(schema=core_schema.int_schema(), required=True) - ``` - - Args: - schema: The schema to use for the field - required: Whether the field is required, otherwise uses the value from `total` on the typed dict - validation_alias: The alias(es) to use to find the field in the validation data - serialization_alias: The alias to use as a key when serializing - serialization_exclude: Whether to exclude the field when serializing - serialization_exclude_if: A callable that determines whether to exclude the field when serializing based on its value. - metadata: Any other information you want to include with the schema, not used by pydantic-core - """ - return _dict_not_none( - type='typed-dict-field', - schema=schema, - required=required, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - serialization_exclude=serialization_exclude, - serialization_exclude_if=serialization_exclude_if, - metadata=metadata, - ) - - -class TypedDictSchema(TypedDict, total=False): - type: Required[Literal['typed-dict']] - fields: Required[dict[str, TypedDictField]] - cls: type[Any] - cls_name: str - computed_fields: list[ComputedField] - strict: bool - extras_schema: CoreSchema - # all these values can be set via config, equivalent fields have `typed_dict_` prefix - extra_behavior: ExtraBehavior - total: bool # default: True - ref: str - metadata: dict[str, Any] - serialization: SerSchema - config: CoreConfig - - -def typed_dict_schema( - fields: dict[str, TypedDictField], - *, - cls: type[Any] | None = None, - cls_name: str | None = None, - computed_fields: list[ComputedField] | None = None, - strict: bool | None = None, - extras_schema: CoreSchema | None = None, - extra_behavior: ExtraBehavior | None = None, - total: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, - config: CoreConfig | None = None, -) -> TypedDictSchema: - """ - Returns a schema that matches a typed dict, e.g.: - - ```py - from typing_extensions import TypedDict - - from pydantic_core import SchemaValidator, core_schema - - class MyTypedDict(TypedDict): - a: str - - wrapper_schema = core_schema.typed_dict_schema( - {'a': core_schema.typed_dict_field(core_schema.str_schema())}, cls=MyTypedDict - ) - v = SchemaValidator(wrapper_schema) - assert v.validate_python({'a': 'hello'}) == {'a': 'hello'} - ``` - - Args: - fields: The fields to use for the typed dict - cls: The class to use for the typed dict - cls_name: The name to use in error locations. Falls back to `cls.__name__`, or the validator name if no class - is provided. - computed_fields: Computed fields to use when serializing the model, only applies when directly inside a model - strict: Whether the typed dict is strict - extras_schema: The extra validator to use for the typed dict - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - extra_behavior: The extra behavior to use for the typed dict - total: Whether the typed dict is total, otherwise uses `typed_dict_total` from config - serialization: Custom serialization schema - """ - return _dict_not_none( - type='typed-dict', - fields=fields, - cls=cls, - cls_name=cls_name, - computed_fields=computed_fields, - strict=strict, - extras_schema=extras_schema, - extra_behavior=extra_behavior, - total=total, - ref=ref, - metadata=metadata, - serialization=serialization, - config=config, - ) - - -class ModelField(TypedDict, total=False): - type: Required[Literal['model-field']] - schema: Required[CoreSchema] - validation_alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] - serialization_alias: str - serialization_exclude: bool # default: False - serialization_exclude_if: Callable[[Any], bool] # default: None - frozen: bool - metadata: dict[str, Any] - - -def model_field( - schema: CoreSchema, - *, - validation_alias: str | list[str | int] | list[list[str | int]] | None = None, - serialization_alias: str | None = None, - serialization_exclude: bool | None = None, - serialization_exclude_if: Callable[[Any], bool] | None = None, - frozen: bool | None = None, - metadata: dict[str, Any] | None = None, -) -> ModelField: - """ - Returns a schema for a model field, e.g.: - - ```py - from pydantic_core import core_schema - - field = core_schema.model_field(schema=core_schema.int_schema()) - ``` - - Args: - schema: The schema to use for the field - validation_alias: The alias(es) to use to find the field in the validation data - serialization_alias: The alias to use as a key when serializing - serialization_exclude: Whether to exclude the field when serializing - serialization_exclude_if: A Callable that determines whether to exclude a field during serialization based on its value. - frozen: Whether the field is frozen - metadata: Any other information you want to include with the schema, not used by pydantic-core - """ - return _dict_not_none( - type='model-field', - schema=schema, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - serialization_exclude=serialization_exclude, - serialization_exclude_if=serialization_exclude_if, - frozen=frozen, - metadata=metadata, - ) - - -class ModelFieldsSchema(TypedDict, total=False): - type: Required[Literal['model-fields']] - fields: Required[dict[str, ModelField]] - model_name: str - computed_fields: list[ComputedField] - strict: bool - extras_schema: CoreSchema - extras_keys_schema: CoreSchema - extra_behavior: ExtraBehavior - from_attributes: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def model_fields_schema( - fields: dict[str, ModelField], - *, - model_name: str | None = None, - computed_fields: list[ComputedField] | None = None, - strict: bool | None = None, - extras_schema: CoreSchema | None = None, - extras_keys_schema: CoreSchema | None = None, - extra_behavior: ExtraBehavior | None = None, - from_attributes: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> ModelFieldsSchema: - """ - Returns a schema that matches the fields of a Pydantic model, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - wrapper_schema = core_schema.model_fields_schema( - {'a': core_schema.model_field(core_schema.str_schema())} - ) - v = SchemaValidator(wrapper_schema) - print(v.validate_python({'a': 'hello'})) - #> ({'a': 'hello'}, None, {'a'}) - ``` - - Args: - fields: The fields of the model - model_name: The name of the model, used for error messages, defaults to "Model" - computed_fields: Computed fields to use when serializing the model, only applies when directly inside a model - strict: Whether the model is strict - extras_schema: The schema to use when validating extra input data - extras_keys_schema: The schema to use when validating the keys of extra input data - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - extra_behavior: The extra behavior to use for the model fields - from_attributes: Whether the model fields should be populated from attributes - serialization: Custom serialization schema - """ - return _dict_not_none( - type='model-fields', - fields=fields, - model_name=model_name, - computed_fields=computed_fields, - strict=strict, - extras_schema=extras_schema, - extras_keys_schema=extras_keys_schema, - extra_behavior=extra_behavior, - from_attributes=from_attributes, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class ModelSchema(TypedDict, total=False): - type: Required[Literal['model']] - cls: Required[type[Any]] - generic_origin: type[Any] - schema: Required[CoreSchema] - custom_init: bool - root_model: bool - post_init: str - revalidate_instances: Literal['always', 'never', 'subclass-instances'] # default: 'never' - strict: bool - frozen: bool - extra_behavior: ExtraBehavior - config: CoreConfig - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def model_schema( - cls: type[Any], - schema: CoreSchema, - *, - generic_origin: type[Any] | None = None, - custom_init: bool | None = None, - root_model: bool | None = None, - post_init: str | None = None, - revalidate_instances: Literal['always', 'never', 'subclass-instances'] | None = None, - strict: bool | None = None, - frozen: bool | None = None, - extra_behavior: ExtraBehavior | None = None, - config: CoreConfig | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> ModelSchema: - """ - A model schema generally contains a typed-dict schema. - It will run the typed dict validator, then create a new class - and set the dict and fields set returned from the typed dict validator - to `__dict__` and `__pydantic_fields_set__` respectively. - - Example: - - ```py - from pydantic_core import CoreConfig, SchemaValidator, core_schema - - class MyModel: - __slots__ = ( - '__dict__', - '__pydantic_fields_set__', - '__pydantic_extra__', - '__pydantic_private__', - ) - - schema = core_schema.model_schema( - cls=MyModel, - config=CoreConfig(str_max_length=5), - schema=core_schema.model_fields_schema( - fields={'a': core_schema.model_field(core_schema.str_schema())}, - ), - ) - v = SchemaValidator(schema) - assert v.isinstance_python({'a': 'hello'}) is True - assert v.isinstance_python({'a': 'too long'}) is False - ``` - - Args: - cls: The class to use for the model - schema: The schema to use for the model - generic_origin: The origin type used for this model, if it's a parametrized generic. Ex, - if this model schema represents `SomeModel[int]`, generic_origin is `SomeModel` - custom_init: Whether the model has a custom init method - root_model: Whether the model is a `RootModel` - post_init: The call after init to use for the model - revalidate_instances: whether instances of models and dataclasses (including subclass instances) - should re-validate defaults to config.revalidate_instances, else 'never' - strict: Whether the model is strict - frozen: Whether the model is frozen - extra_behavior: The extra behavior to use for the model, used in serialization - config: The config to use for the model - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='model', - cls=cls, - generic_origin=generic_origin, - schema=schema, - custom_init=custom_init, - root_model=root_model, - post_init=post_init, - revalidate_instances=revalidate_instances, - strict=strict, - frozen=frozen, - extra_behavior=extra_behavior, - config=config, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class DataclassField(TypedDict, total=False): - type: Required[Literal['dataclass-field']] - name: Required[str] - schema: Required[CoreSchema] - kw_only: bool # default: True - init: bool # default: True - init_only: bool # default: False - frozen: bool # default: False - validation_alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] - serialization_alias: str - serialization_exclude: bool # default: False - metadata: dict[str, Any] - serialization_exclude_if: Callable[[Any], bool] # default: None - - -def dataclass_field( - name: str, - schema: CoreSchema, - *, - kw_only: bool | None = None, - init: bool | None = None, - init_only: bool | None = None, - validation_alias: str | list[str | int] | list[list[str | int]] | None = None, - serialization_alias: str | None = None, - serialization_exclude: bool | None = None, - metadata: dict[str, Any] | None = None, - serialization_exclude_if: Callable[[Any], bool] | None = None, - frozen: bool | None = None, -) -> DataclassField: - """ - Returns a schema for a dataclass field, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - field = core_schema.dataclass_field( - name='a', schema=core_schema.str_schema(), kw_only=False - ) - schema = core_schema.dataclass_args_schema('Foobar', [field]) - v = SchemaValidator(schema) - assert v.validate_python({'a': 'hello'}) == ({'a': 'hello'}, None) - ``` - - Args: - name: The name to use for the argument parameter - schema: The schema to use for the argument parameter - kw_only: Whether the field can be set with a positional argument as well as a keyword argument - init: Whether the field should be validated during initialization - init_only: Whether the field should be omitted from `__dict__` and passed to `__post_init__` - validation_alias: The alias(es) to use to find the field in the validation data - serialization_alias: The alias to use as a key when serializing - serialization_exclude: Whether to exclude the field when serializing - serialization_exclude_if: A callable that determines whether to exclude the field when serializing based on its value. - metadata: Any other information you want to include with the schema, not used by pydantic-core - frozen: Whether the field is frozen - """ - return _dict_not_none( - type='dataclass-field', - name=name, - schema=schema, - kw_only=kw_only, - init=init, - init_only=init_only, - validation_alias=validation_alias, - serialization_alias=serialization_alias, - serialization_exclude=serialization_exclude, - serialization_exclude_if=serialization_exclude_if, - metadata=metadata, - frozen=frozen, - ) - - -class DataclassArgsSchema(TypedDict, total=False): - type: Required[Literal['dataclass-args']] - dataclass_name: Required[str] - fields: Required[list[DataclassField]] - computed_fields: list[ComputedField] - collect_init_only: bool # default: False - ref: str - metadata: dict[str, Any] - serialization: SerSchema - extra_behavior: ExtraBehavior - - -def dataclass_args_schema( - dataclass_name: str, - fields: list[DataclassField], - *, - computed_fields: list[ComputedField] | None = None, - collect_init_only: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, - extra_behavior: ExtraBehavior | None = None, -) -> DataclassArgsSchema: - """ - Returns a schema for validating dataclass arguments, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - field_a = core_schema.dataclass_field( - name='a', schema=core_schema.str_schema(), kw_only=False - ) - field_b = core_schema.dataclass_field( - name='b', schema=core_schema.bool_schema(), kw_only=False - ) - schema = core_schema.dataclass_args_schema('Foobar', [field_a, field_b]) - v = SchemaValidator(schema) - assert v.validate_python({'a': 'hello', 'b': True}) == ({'a': 'hello', 'b': True}, None) - ``` - - Args: - dataclass_name: The name of the dataclass being validated - fields: The fields to use for the dataclass - computed_fields: Computed fields to use when serializing the dataclass - collect_init_only: Whether to collect init only fields into a dict to pass to `__post_init__` - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - extra_behavior: How to handle extra fields - """ - return _dict_not_none( - type='dataclass-args', - dataclass_name=dataclass_name, - fields=fields, - computed_fields=computed_fields, - collect_init_only=collect_init_only, - ref=ref, - metadata=metadata, - serialization=serialization, - extra_behavior=extra_behavior, - ) - - -class DataclassSchema(TypedDict, total=False): - type: Required[Literal['dataclass']] - cls: Required[type[Any]] - generic_origin: type[Any] - schema: Required[CoreSchema] - fields: Required[list[str]] - cls_name: str - post_init: bool # default: False - revalidate_instances: Literal['always', 'never', 'subclass-instances'] # default: 'never' - strict: bool # default: False - frozen: bool # default False - ref: str - metadata: dict[str, Any] - serialization: SerSchema - slots: bool - config: CoreConfig - - -def dataclass_schema( - cls: type[Any], - schema: CoreSchema, - fields: list[str], - *, - generic_origin: type[Any] | None = None, - cls_name: str | None = None, - post_init: bool | None = None, - revalidate_instances: Literal['always', 'never', 'subclass-instances'] | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, - frozen: bool | None = None, - slots: bool | None = None, - config: CoreConfig | None = None, -) -> DataclassSchema: - """ - Returns a schema for a dataclass. As with `ModelSchema`, this schema can only be used as a field within - another schema, not as the root type. - - Args: - cls: The dataclass type, used to perform subclass checks - schema: The schema to use for the dataclass fields - fields: Fields of the dataclass, this is used in serialization and in validation during re-validation - and while validating assignment - generic_origin: The origin type used for this dataclass, if it's a parametrized generic. Ex, - if this model schema represents `SomeDataclass[int]`, generic_origin is `SomeDataclass` - cls_name: The name to use in error locs, etc; this is useful for generics (default: `cls.__name__`) - post_init: Whether to call `__post_init__` after validation - revalidate_instances: whether instances of models and dataclasses (including subclass instances) - should re-validate defaults to config.revalidate_instances, else 'never' - strict: Whether to require an exact instance of `cls` - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - frozen: Whether the dataclass is frozen - slots: Whether `slots=True` on the dataclass, means each field is assigned independently, rather than - simply setting `__dict__`, default false - """ - return _dict_not_none( - type='dataclass', - cls=cls, - generic_origin=generic_origin, - fields=fields, - cls_name=cls_name, - schema=schema, - post_init=post_init, - revalidate_instances=revalidate_instances, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - frozen=frozen, - slots=slots, - config=config, - ) - - -class ArgumentsParameter(TypedDict, total=False): - name: Required[str] - schema: Required[CoreSchema] - mode: Literal['positional_only', 'positional_or_keyword', 'keyword_only'] # default positional_or_keyword - alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] - - -def arguments_parameter( - name: str, - schema: CoreSchema, - *, - mode: Literal['positional_only', 'positional_or_keyword', 'keyword_only'] | None = None, - alias: str | list[str | int] | list[list[str | int]] | None = None, -) -> ArgumentsParameter: - """ - Returns a schema that matches an argument parameter, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - param = core_schema.arguments_parameter( - name='a', schema=core_schema.str_schema(), mode='positional_only' - ) - schema = core_schema.arguments_schema([param]) - v = SchemaValidator(schema) - assert v.validate_python(('hello',)) == (('hello',), {}) - ``` - - Args: - name: The name to use for the argument parameter - schema: The schema to use for the argument parameter - mode: The mode to use for the argument parameter - alias: The alias to use for the argument parameter - """ - return _dict_not_none(name=name, schema=schema, mode=mode, alias=alias) - - -VarKwargsMode: TypeAlias = Literal['uniform', 'unpacked-typed-dict'] - - -class ArgumentsSchema(TypedDict, total=False): - type: Required[Literal['arguments']] - arguments_schema: Required[list[ArgumentsParameter]] - validate_by_name: bool - validate_by_alias: bool - var_args_schema: CoreSchema - var_kwargs_mode: VarKwargsMode - var_kwargs_schema: CoreSchema - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def arguments_schema( - arguments: list[ArgumentsParameter], - *, - validate_by_name: bool | None = None, - validate_by_alias: bool | None = None, - var_args_schema: CoreSchema | None = None, - var_kwargs_mode: VarKwargsMode | None = None, - var_kwargs_schema: CoreSchema | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> ArgumentsSchema: - """ - Returns a schema that matches an arguments schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - param_a = core_schema.arguments_parameter( - name='a', schema=core_schema.str_schema(), mode='positional_only' - ) - param_b = core_schema.arguments_parameter( - name='b', schema=core_schema.bool_schema(), mode='positional_only' - ) - schema = core_schema.arguments_schema([param_a, param_b]) - v = SchemaValidator(schema) - assert v.validate_python(('hello', True)) == (('hello', True), {}) - ``` - - Args: - arguments: The arguments to use for the arguments schema - validate_by_name: Whether to populate by the parameter names, defaults to `False`. - validate_by_alias: Whether to populate by the parameter aliases, defaults to `True`. - var_args_schema: The variable args schema to use for the arguments schema - var_kwargs_mode: The validation mode to use for variadic keyword arguments. If `'uniform'`, every value of the - keyword arguments will be validated against the `var_kwargs_schema` schema. If `'unpacked-typed-dict'`, - the `var_kwargs_schema` argument must be a [`typed_dict_schema`][pydantic_core.core_schema.typed_dict_schema] - var_kwargs_schema: The variable kwargs schema to use for the arguments schema - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='arguments', - arguments_schema=arguments, - validate_by_name=validate_by_name, - validate_by_alias=validate_by_alias, - var_args_schema=var_args_schema, - var_kwargs_mode=var_kwargs_mode, - var_kwargs_schema=var_kwargs_schema, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class ArgumentsV3Parameter(TypedDict, total=False): - name: Required[str] - schema: Required[CoreSchema] - mode: Literal[ - 'positional_only', - 'positional_or_keyword', - 'keyword_only', - 'var_args', - 'var_kwargs_uniform', - 'var_kwargs_unpacked_typed_dict', - ] # default positional_or_keyword - alias: Union[str, list[Union[str, int]], list[list[Union[str, int]]]] - - -def arguments_v3_parameter( - name: str, - schema: CoreSchema, - *, - mode: Literal[ - 'positional_only', - 'positional_or_keyword', - 'keyword_only', - 'var_args', - 'var_kwargs_uniform', - 'var_kwargs_unpacked_typed_dict', - ] - | None = None, - alias: str | list[str | int] | list[list[str | int]] | None = None, -) -> ArgumentsV3Parameter: - """ - Returns a schema that matches an argument parameter, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - param = core_schema.arguments_v3_parameter( - name='a', schema=core_schema.str_schema(), mode='positional_only' - ) - schema = core_schema.arguments_v3_schema([param]) - v = SchemaValidator(schema) - assert v.validate_python({'a': 'hello'}) == (('hello',), {}) - ``` - - Args: - name: The name to use for the argument parameter - schema: The schema to use for the argument parameter - mode: The mode to use for the argument parameter - alias: The alias to use for the argument parameter - """ - return _dict_not_none(name=name, schema=schema, mode=mode, alias=alias) - - -class ArgumentsV3Schema(TypedDict, total=False): - type: Required[Literal['arguments-v3']] - arguments_schema: Required[list[ArgumentsV3Parameter]] - validate_by_name: bool - validate_by_alias: bool - extra_behavior: Literal['forbid', 'ignore'] # 'allow' doesn't make sense here. - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def arguments_v3_schema( - arguments: list[ArgumentsV3Parameter], - *, - validate_by_name: bool | None = None, - validate_by_alias: bool | None = None, - extra_behavior: Literal['forbid', 'ignore'] | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> ArgumentsV3Schema: - """ - Returns a schema that matches an arguments schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - param_a = core_schema.arguments_v3_parameter( - name='a', schema=core_schema.str_schema(), mode='positional_only' - ) - param_b = core_schema.arguments_v3_parameter( - name='kwargs', schema=core_schema.bool_schema(), mode='var_kwargs_uniform' - ) - schema = core_schema.arguments_v3_schema([param_a, param_b]) - v = SchemaValidator(schema) - assert v.validate_python({'a': 'hi', 'kwargs': {'b': True}}) == (('hi',), {'b': True}) - ``` - - This schema is currently not used by other Pydantic components. In V3, it will most likely - become the default arguments schema for the `'call'` schema. - - Args: - arguments: The arguments to use for the arguments schema. - validate_by_name: Whether to populate by the parameter names, defaults to `False`. - validate_by_alias: Whether to populate by the parameter aliases, defaults to `True`. - extra_behavior: The extra behavior to use. - ref: optional unique identifier of the schema, used to reference the schema in other places. - metadata: Any other information you want to include with the schema, not used by pydantic-core. - serialization: Custom serialization schema. - """ - return _dict_not_none( - type='arguments-v3', - arguments_schema=arguments, - validate_by_name=validate_by_name, - validate_by_alias=validate_by_alias, - extra_behavior=extra_behavior, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class CallSchema(TypedDict, total=False): - type: Required[Literal['call']] - arguments_schema: Required[CoreSchema] - function: Required[Callable[..., Any]] - function_name: str # default function.__name__ - return_schema: CoreSchema - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def call_schema( - arguments: CoreSchema, - function: Callable[..., Any], - *, - function_name: str | None = None, - return_schema: CoreSchema | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> CallSchema: - """ - Returns a schema that matches an arguments schema, then calls a function, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - param_a = core_schema.arguments_parameter( - name='a', schema=core_schema.str_schema(), mode='positional_only' - ) - param_b = core_schema.arguments_parameter( - name='b', schema=core_schema.bool_schema(), mode='positional_only' - ) - args_schema = core_schema.arguments_schema([param_a, param_b]) - - schema = core_schema.call_schema( - arguments=args_schema, - function=lambda a, b: a + str(not b), - return_schema=core_schema.str_schema(), - ) - v = SchemaValidator(schema) - assert v.validate_python((('hello', True))) == 'helloFalse' - ``` - - Args: - arguments: The arguments to use for the arguments schema - function: The function to use for the call schema - function_name: The function name to use for the call schema, if not provided `function.__name__` is used - return_schema: The return schema to use for the call schema - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='call', - arguments_schema=arguments, - function=function, - function_name=function_name, - return_schema=return_schema, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class CustomErrorSchema(TypedDict, total=False): - type: Required[Literal['custom-error']] - schema: Required[CoreSchema] - custom_error_type: Required[str] - custom_error_message: str - custom_error_context: dict[str, Union[str, int, float]] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def custom_error_schema( - schema: CoreSchema, - custom_error_type: str, - *, - custom_error_message: str | None = None, - custom_error_context: dict[str, Any] | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> CustomErrorSchema: - """ - Returns a schema that matches a custom error value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.custom_error_schema( - schema=core_schema.int_schema(), - custom_error_type='MyError', - custom_error_message='Error msg', - ) - v = SchemaValidator(schema) - v.validate_python(1) - ``` - - Args: - schema: The schema to use for the custom error schema - custom_error_type: The custom error type to use for the custom error schema - custom_error_message: The custom error message to use for the custom error schema - custom_error_context: The custom error context to use for the custom error schema - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='custom-error', - schema=schema, - custom_error_type=custom_error_type, - custom_error_message=custom_error_message, - custom_error_context=custom_error_context, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class JsonSchema(TypedDict, total=False): - type: Required[Literal['json']] - schema: CoreSchema - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def json_schema( - schema: CoreSchema | None = None, - *, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> JsonSchema: - """ - Returns a schema that matches a JSON value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - dict_schema = core_schema.model_fields_schema( - { - 'field_a': core_schema.model_field(core_schema.str_schema()), - 'field_b': core_schema.model_field(core_schema.bool_schema()), - }, - ) - - class MyModel: - __slots__ = ( - '__dict__', - '__pydantic_fields_set__', - '__pydantic_extra__', - '__pydantic_private__', - ) - field_a: str - field_b: bool - - json_schema = core_schema.json_schema(schema=dict_schema) - schema = core_schema.model_schema(cls=MyModel, schema=json_schema) - v = SchemaValidator(schema) - m = v.validate_python('{"field_a": "hello", "field_b": true}') - assert isinstance(m, MyModel) - ``` - - Args: - schema: The schema to use for the JSON schema - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none(type='json', schema=schema, ref=ref, metadata=metadata, serialization=serialization) - - -class UrlSchema(TypedDict, total=False): - type: Required[Literal['url']] - max_length: int - allowed_schemes: list[str] - host_required: bool # default False - default_host: str - default_port: int - default_path: str - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def url_schema( - *, - max_length: int | None = None, - allowed_schemes: list[str] | None = None, - host_required: bool | None = None, - default_host: str | None = None, - default_port: int | None = None, - default_path: str | None = None, - preserve_empty_path: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> UrlSchema: - """ - Returns a schema that matches a URL value, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.url_schema() - v = SchemaValidator(schema) - print(v.validate_python('https://example.com')) - #> https://example.com/ - ``` - - Args: - max_length: The maximum length of the URL - allowed_schemes: The allowed URL schemes - host_required: Whether the URL must have a host - default_host: The default host to use if the URL does not have a host - default_port: The default port to use if the URL does not have a port - default_path: The default path to use if the URL does not have a path - preserve_empty_path: Whether to preserve an empty path or convert it to '/', default False - strict: Whether to use strict URL parsing - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='url', - max_length=max_length, - allowed_schemes=allowed_schemes, - host_required=host_required, - default_host=default_host, - default_port=default_port, - default_path=default_path, - preserve_empty_path=preserve_empty_path, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class MultiHostUrlSchema(TypedDict, total=False): - type: Required[Literal['multi-host-url']] - max_length: int - allowed_schemes: list[str] - host_required: bool # default False - default_host: str - default_port: int - default_path: str - strict: bool - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def multi_host_url_schema( - *, - max_length: int | None = None, - allowed_schemes: list[str] | None = None, - host_required: bool | None = None, - default_host: str | None = None, - default_port: int | None = None, - default_path: str | None = None, - preserve_empty_path: bool | None = None, - strict: bool | None = None, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> MultiHostUrlSchema: - """ - Returns a schema that matches a URL value with possibly multiple hosts, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.multi_host_url_schema() - v = SchemaValidator(schema) - print(v.validate_python('redis://localhost,0.0.0.0,127.0.0.1')) - #> redis://localhost,0.0.0.0,127.0.0.1 - ``` - - Args: - max_length: The maximum length of the URL - allowed_schemes: The allowed URL schemes - host_required: Whether the URL must have a host - default_host: The default host to use if the URL does not have a host - default_port: The default port to use if the URL does not have a port - default_path: The default path to use if the URL does not have a path - preserve_empty_path: Whether to preserve an empty path or convert it to '/', default False - strict: Whether to use strict URL parsing - ref: optional unique identifier of the schema, used to reference the schema in other places - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='multi-host-url', - max_length=max_length, - allowed_schemes=allowed_schemes, - host_required=host_required, - default_host=default_host, - default_port=default_port, - default_path=default_path, - preserve_empty_path=preserve_empty_path, - strict=strict, - ref=ref, - metadata=metadata, - serialization=serialization, - ) - - -class DefinitionsSchema(TypedDict, total=False): - type: Required[Literal['definitions']] - schema: Required[CoreSchema] - definitions: Required[list[CoreSchema]] - metadata: dict[str, Any] - serialization: SerSchema - - -def definitions_schema(schema: CoreSchema, definitions: list[CoreSchema]) -> DefinitionsSchema: - """ - Build a schema that contains both an inner schema and a list of definitions which can be used - within the inner schema. - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema = core_schema.definitions_schema( - core_schema.list_schema(core_schema.definition_reference_schema('foobar')), - [core_schema.int_schema(ref='foobar')], - ) - v = SchemaValidator(schema) - assert v.validate_python([1, 2, '3']) == [1, 2, 3] - ``` - - Args: - schema: The inner schema - definitions: List of definitions which can be referenced within inner schema - """ - return DefinitionsSchema(type='definitions', schema=schema, definitions=definitions) - - -class DefinitionReferenceSchema(TypedDict, total=False): - type: Required[Literal['definition-ref']] - schema_ref: Required[str] - ref: str - metadata: dict[str, Any] - serialization: SerSchema - - -def definition_reference_schema( - schema_ref: str, - ref: str | None = None, - metadata: dict[str, Any] | None = None, - serialization: SerSchema | None = None, -) -> DefinitionReferenceSchema: - """ - Returns a schema that points to a schema stored in "definitions", this is useful for nested recursive - models and also when you want to define validators separately from the main schema, e.g.: - - ```py - from pydantic_core import SchemaValidator, core_schema - - schema_definition = core_schema.definition_reference_schema('list-schema') - schema = core_schema.definitions_schema( - schema=schema_definition, - definitions=[ - core_schema.list_schema(items_schema=schema_definition, ref='list-schema'), - ], - ) - v = SchemaValidator(schema) - assert v.validate_python([()]) == [[]] - ``` - - Args: - schema_ref: The schema ref to use for the definition reference schema - metadata: Any other information you want to include with the schema, not used by pydantic-core - serialization: Custom serialization schema - """ - return _dict_not_none( - type='definition-ref', schema_ref=schema_ref, ref=ref, metadata=metadata, serialization=serialization - ) - - -MYPY = False -# See https://github.com/python/mypy/issues/14034 for details, in summary mypy is extremely slow to process this -# union which kills performance not just for pydantic, but even for code using pydantic -if not MYPY: - CoreSchema = Union[ - InvalidSchema, - AnySchema, - NoneSchema, - BoolSchema, - IntSchema, - FloatSchema, - DecimalSchema, - StringSchema, - BytesSchema, - DateSchema, - TimeSchema, - DatetimeSchema, - TimedeltaSchema, - LiteralSchema, - MissingSentinelSchema, - EnumSchema, - IsInstanceSchema, - IsSubclassSchema, - CallableSchema, - ListSchema, - TupleSchema, - SetSchema, - FrozenSetSchema, - GeneratorSchema, - DictSchema, - AfterValidatorFunctionSchema, - BeforeValidatorFunctionSchema, - WrapValidatorFunctionSchema, - PlainValidatorFunctionSchema, - WithDefaultSchema, - NullableSchema, - UnionSchema, - TaggedUnionSchema, - ChainSchema, - LaxOrStrictSchema, - JsonOrPythonSchema, - TypedDictSchema, - ModelFieldsSchema, - ModelSchema, - DataclassArgsSchema, - DataclassSchema, - ArgumentsSchema, - ArgumentsV3Schema, - CallSchema, - CustomErrorSchema, - JsonSchema, - UrlSchema, - MultiHostUrlSchema, - DefinitionsSchema, - DefinitionReferenceSchema, - UuidSchema, - ComplexSchema, - ] -elif False: - CoreSchema: TypeAlias = Mapping[str, Any] - - -# to update this, call `pytest -k test_core_schema_type_literal` and copy the output -CoreSchemaType = Literal[ - 'invalid', - 'any', - 'none', - 'bool', - 'int', - 'float', - 'decimal', - 'str', - 'bytes', - 'date', - 'time', - 'datetime', - 'timedelta', - 'literal', - 'missing-sentinel', - 'enum', - 'is-instance', - 'is-subclass', - 'callable', - 'list', - 'tuple', - 'set', - 'frozenset', - 'generator', - 'dict', - 'function-after', - 'function-before', - 'function-wrap', - 'function-plain', - 'default', - 'nullable', - 'union', - 'tagged-union', - 'chain', - 'lax-or-strict', - 'json-or-python', - 'typed-dict', - 'model-fields', - 'model', - 'dataclass-args', - 'dataclass', - 'arguments', - 'arguments-v3', - 'call', - 'custom-error', - 'json', - 'url', - 'multi-host-url', - 'definitions', - 'definition-ref', - 'uuid', - 'complex', -] - -CoreSchemaFieldType = Literal['model-field', 'dataclass-field', 'typed-dict-field', 'computed-field'] - - -# used in _pydantic_core.pyi::PydanticKnownError -# to update this, call `pytest -k test_all_errors` and copy the output -ErrorType = Literal[ - 'no_such_attribute', - 'json_invalid', - 'json_type', - 'needs_python_object', - 'recursion_loop', - 'missing', - 'frozen_field', - 'frozen_instance', - 'extra_forbidden', - 'invalid_key', - 'get_attribute_error', - 'model_type', - 'model_attributes_type', - 'dataclass_type', - 'dataclass_exact_type', - 'default_factory_not_called', - 'none_required', - 'greater_than', - 'greater_than_equal', - 'less_than', - 'less_than_equal', - 'multiple_of', - 'finite_number', - 'too_short', - 'too_long', - 'iterable_type', - 'iteration_error', - 'string_type', - 'string_sub_type', - 'string_unicode', - 'string_too_short', - 'string_too_long', - 'string_pattern_mismatch', - 'string_not_ascii', - 'enum', - 'dict_type', - 'mapping_type', - 'list_type', - 'tuple_type', - 'set_type', - 'set_item_not_hashable', - 'bool_type', - 'bool_parsing', - 'int_type', - 'int_parsing', - 'int_parsing_size', - 'int_from_float', - 'float_type', - 'float_parsing', - 'bytes_type', - 'bytes_too_short', - 'bytes_too_long', - 'bytes_invalid_encoding', - 'value_error', - 'assertion_error', - 'literal_error', - 'missing_sentinel_error', - 'date_type', - 'date_parsing', - 'date_from_datetime_parsing', - 'date_from_datetime_inexact', - 'date_past', - 'date_future', - 'time_type', - 'time_parsing', - 'datetime_type', - 'datetime_parsing', - 'datetime_object_invalid', - 'datetime_from_date_parsing', - 'datetime_past', - 'datetime_future', - 'timezone_naive', - 'timezone_aware', - 'timezone_offset', - 'time_delta_type', - 'time_delta_parsing', - 'frozen_set_type', - 'is_instance_of', - 'is_subclass_of', - 'callable_type', - 'union_tag_invalid', - 'union_tag_not_found', - 'arguments_type', - 'missing_argument', - 'unexpected_keyword_argument', - 'missing_keyword_only_argument', - 'unexpected_positional_argument', - 'missing_positional_only_argument', - 'multiple_argument_values', - 'url_type', - 'url_parsing', - 'url_syntax_violation', - 'url_too_long', - 'url_scheme', - 'uuid_type', - 'uuid_parsing', - 'uuid_version', - 'decimal_type', - 'decimal_parsing', - 'decimal_max_digits', - 'decimal_max_places', - 'decimal_whole_digits', - 'complex_type', - 'complex_str_parsing', -] - - -def _dict_not_none(**kwargs: Any) -> Any: - return {k: v for k, v in kwargs.items() if v is not None} - - -def iter_union_choices(union_schema: UnionSchema) -> Generator[CoreSchema]: - """Iterate over the choices of a `'union'` schema.""" - for choice in union_schema['choices']: - if isinstance(choice, tuple): - yield choice[0] - else: - yield choice - - -############################################################################### -# All this stuff is deprecated by #980 and will be removed eventually -# They're kept because some code external code will be using them - - -@deprecated('`field_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.') -def field_before_validator_function(function: WithInfoValidatorFunction, field_name: str, schema: CoreSchema, **kwargs): - warnings.warn( - '`field_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.', - DeprecationWarning, - ) - return with_info_before_validator_function(function, schema, field_name=field_name, **kwargs) - - -@deprecated('`general_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.') -def general_before_validator_function(*args, **kwargs): - warnings.warn( - '`general_before_validator_function` is deprecated, use `with_info_before_validator_function` instead.', - DeprecationWarning, - ) - return with_info_before_validator_function(*args, **kwargs) - - -@deprecated('`field_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.') -def field_after_validator_function(function: WithInfoValidatorFunction, field_name: str, schema: CoreSchema, **kwargs): - warnings.warn( - '`field_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.', - DeprecationWarning, - ) - return with_info_after_validator_function(function, schema, field_name=field_name, **kwargs) - - -@deprecated('`general_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.') -def general_after_validator_function(*args, **kwargs): - warnings.warn( - '`general_after_validator_function` is deprecated, use `with_info_after_validator_function` instead.', - DeprecationWarning, - ) - return with_info_after_validator_function(*args, **kwargs) - - -@deprecated('`field_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.') -def field_wrap_validator_function( - function: WithInfoWrapValidatorFunction, field_name: str, schema: CoreSchema, **kwargs -): - warnings.warn( - '`field_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.', - DeprecationWarning, - ) - return with_info_wrap_validator_function(function, schema, field_name=field_name, **kwargs) - - -@deprecated('`general_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.') -def general_wrap_validator_function(*args, **kwargs): - warnings.warn( - '`general_wrap_validator_function` is deprecated, use `with_info_wrap_validator_function` instead.', - DeprecationWarning, - ) - return with_info_wrap_validator_function(*args, **kwargs) - - -@deprecated('`field_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.') -def field_plain_validator_function(function: WithInfoValidatorFunction, field_name: str, **kwargs): - warnings.warn( - '`field_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.', - DeprecationWarning, - ) - return with_info_plain_validator_function(function, field_name=field_name, **kwargs) - - -@deprecated('`general_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.') -def general_plain_validator_function(*args, **kwargs): - warnings.warn( - '`general_plain_validator_function` is deprecated, use `with_info_plain_validator_function` instead.', - DeprecationWarning, - ) - return with_info_plain_validator_function(*args, **kwargs) - - -_deprecated_import_lookup = { - 'FieldValidationInfo': ValidationInfo, - 'FieldValidatorFunction': WithInfoValidatorFunction, - 'GeneralValidatorFunction': WithInfoValidatorFunction, - 'FieldWrapValidatorFunction': WithInfoWrapValidatorFunction, -} - -if TYPE_CHECKING: - FieldValidationInfo = ValidationInfo - - -def __getattr__(attr_name: str) -> object: - new_attr = _deprecated_import_lookup.get(attr_name) - if new_attr is None: - raise AttributeError(f"module 'pydantic_core' has no attribute '{attr_name}'") - else: - import warnings - - msg = f'`{attr_name}` is deprecated, use `{new_attr.__name__}` instead.' - warnings.warn(msg, DeprecationWarning, stacklevel=1) - return new_attr diff --git a/functions/python/data-processor/pydantic_core/py.typed b/functions/python/data-processor/pydantic_core/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/starlette-1.3.1.dist-info/INSTALLER b/functions/python/data-processor/starlette-1.3.1.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/starlette-1.3.1.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/starlette-1.3.1.dist-info/METADATA b/functions/python/data-processor/starlette-1.3.1.dist-info/METADATA deleted file mode 100644 index 142a852dd..000000000 --- a/functions/python/data-processor/starlette-1.3.1.dist-info/METADATA +++ /dev/null @@ -1,178 +0,0 @@ -Metadata-Version: 2.4 -Name: starlette -Version: 1.3.1 -Summary: The little ASGI library that shines. -Project-URL: Homepage, https://github.com/Kludex/starlette -Project-URL: Documentation, https://starlette.dev/ -Project-URL: Changelog, https://starlette.dev/release-notes/ -Project-URL: Funding, https://github.com/sponsors/Kludex -Project-URL: Source, https://github.com/Kludex/starlette -Author-email: Tom Christie -Maintainer-email: Marcelo Trylesinski -License-Expression: BSD-3-Clause -License-File: LICENSE.md -Classifier: Development Status :: 3 - Alpha -Classifier: Environment :: Web Environment -Classifier: Framework :: AnyIO -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Internet :: WWW/HTTP -Requires-Python: >=3.10 -Requires-Dist: anyio<5,>=3.6.2 -Requires-Dist: typing-extensions>=4.10.0; python_version < '3.13' -Provides-Extra: full -Requires-Dist: httpx2>=2.0.0; extra == 'full' -Requires-Dist: httpx<0.29.0,>=0.27.0; extra == 'full' -Requires-Dist: itsdangerous; extra == 'full' -Requires-Dist: jinja2; extra == 'full' -Requires-Dist: python-multipart>=0.0.18; extra == 'full' -Requires-Dist: pyyaml; extra == 'full' -Description-Content-Type: text/markdown - -

- - - - starlette-logo - -

- -

- ✨ The little ASGI framework that shines. ✨ -

- ---- - -[![Build Status](https://github.com/Kludex/starlette/workflows/Test%20Suite/badge.svg)](https://github.com/Kludex/starlette/actions) -[![Package version](https://badge.fury.io/py/starlette.svg)](https://pypi.python.org/pypi/starlette) -[![Supported Python Version](https://img.shields.io/pypi/pyversions/starlette.svg?color=%2334D058)](https://pypi.org/project/starlette) -[![Discord](https://img.shields.io/discord/1051468649518616576?logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/RxKUF5JuHs) - ---- - -**Documentation**: https://starlette.dev - -**Source Code**: https://github.com/Kludex/starlette - ---- - -# Starlette - -Starlette is a lightweight [ASGI][asgi] framework/toolkit, -which is ideal for building async web services in Python. - -It is production-ready, and gives you the following: - -* A lightweight, low-complexity HTTP web framework. -* WebSocket support. -* In-process background tasks. -* Startup and shutdown events. -* Test client built on `httpx`. -* CORS, GZip, Static Files, Streaming responses. -* Session and Cookie support. -* 100% test coverage. -* 100% type annotated codebase. -* Few hard dependencies. -* Compatible with `asyncio` and `trio` backends. -* Great overall performance [against independent benchmarks][techempower]. - -## Installation - -```shell -$ pip install starlette -``` - -You'll also want to install an ASGI server, such as [uvicorn](https://uvicorn.dev) or any of the [other ASGI server implementations](https://asgi.readthedocs.io/en/latest/implementations.html#servers). - -```shell -$ pip install uvicorn -``` - -## Example - -```python title="main.py" -from starlette.applications import Starlette -from starlette.responses import JSONResponse -from starlette.routing import Route - - -async def homepage(request): - return JSONResponse({'hello': 'world'}) - -routes = [ - Route("/", endpoint=homepage) -] - -app = Starlette(debug=True, routes=routes) -``` - -Then run the application using Uvicorn: - -```shell -$ uvicorn main:app -``` - -## Dependencies - -Starlette only requires `anyio`, and the following are optional: - -* [`httpx2`][httpx2] - Required if you want to use the `TestClient`. -* [`jinja2`][jinja2] - Required if you want to use `Jinja2Templates`. -* [`python-multipart`][python-multipart] - Required if you want to support form parsing, with `request.form()`. -* [`itsdangerous`][itsdangerous] - Required for `SessionMiddleware` support. -* [`pyyaml`][pyyaml] - Required for `SchemaGenerator` support. - -You can install all of these with `pip install starlette[full]`. - -## Framework or Toolkit - -Starlette is designed to be used either as a complete framework, or as -an ASGI toolkit. You can use any of its components independently. - -```python -from starlette.responses import PlainTextResponse - - -async def app(scope, receive, send): - assert scope['type'] == 'http' - response = PlainTextResponse('Hello, world!') - await response(scope, receive, send) -``` - -Run the `app` application in `example.py`: - -```shell -$ uvicorn example:app -INFO: Started server process [11509] -INFO: Uvicorn running on http://127.0.0.1:8000 (Press CTRL+C to quit) -``` - -Run uvicorn with `--reload` to enable auto-reloading on code changes. - -## Modularity - -The modularity that Starlette is designed on promotes building reusable -components that can be shared between any ASGI framework. This should enable -an ecosystem of shared middleware and mountable applications. - -The clean API separation also means it's easier to understand each component -in isolation. - ---- - -

Starlette is BSD licensed code.
Designed & crafted with care.
— ⭐️ —

- -[asgi]: https://asgi.readthedocs.io/en/latest/ -[httpx2]: https://pypi.org/project/httpx2/ -[jinja2]: https://jinja.palletsprojects.com/ -[python-multipart]: https://multipart.fastapiexpert.com/ -[itsdangerous]: https://itsdangerous.palletsprojects.com/ -[sqlalchemy]: https://www.sqlalchemy.org -[pyyaml]: https://pyyaml.org/wiki/PyYAMLDocumentation -[techempower]: https://www.techempower.com/benchmarks/#hw=ph&test=fortune&l=zijzen-sf diff --git a/functions/python/data-processor/starlette-1.3.1.dist-info/RECORD b/functions/python/data-processor/starlette-1.3.1.dist-info/RECORD deleted file mode 100644 index 2a9549362..000000000 --- a/functions/python/data-processor/starlette-1.3.1.dist-info/RECORD +++ /dev/null @@ -1,74 +0,0 @@ -starlette-1.3.1.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -starlette-1.3.1.dist-info/METADATA,sha256=2DzNlmXrBdH5OTx14gnLNTmQcxpiFaYJoPHFwKYobJo,6370 -starlette-1.3.1.dist-info/RECORD,, -starlette-1.3.1.dist-info/WHEEL,sha256=mffPy8wBnZQn2VnJUU5jE99KsxaSfiyMHV9Yt0aLVxs,87 -starlette-1.3.1.dist-info/licenses/LICENSE.md,sha256=3LlWd6AiQCQxh-lk-UGEfRmxeCHPmeWvrmhPqzKMGb8,1518 -starlette/__init__.py,sha256=-ypEJktJToAL9by62JJKWEzDo_KPCQtmE5kwFgX24z4,22 -starlette/__pycache__/__init__.cpython-314.pyc,, -starlette/__pycache__/_exception_handler.cpython-314.pyc,, -starlette/__pycache__/_utils.cpython-314.pyc,, -starlette/__pycache__/applications.cpython-314.pyc,, -starlette/__pycache__/authentication.cpython-314.pyc,, -starlette/__pycache__/background.cpython-314.pyc,, -starlette/__pycache__/concurrency.cpython-314.pyc,, -starlette/__pycache__/config.cpython-314.pyc,, -starlette/__pycache__/convertors.cpython-314.pyc,, -starlette/__pycache__/datastructures.cpython-314.pyc,, -starlette/__pycache__/endpoints.cpython-314.pyc,, -starlette/__pycache__/exceptions.cpython-314.pyc,, -starlette/__pycache__/formparsers.cpython-314.pyc,, -starlette/__pycache__/requests.cpython-314.pyc,, -starlette/__pycache__/responses.cpython-314.pyc,, -starlette/__pycache__/routing.cpython-314.pyc,, -starlette/__pycache__/schemas.cpython-314.pyc,, -starlette/__pycache__/staticfiles.cpython-314.pyc,, -starlette/__pycache__/status.cpython-314.pyc,, -starlette/__pycache__/templating.cpython-314.pyc,, -starlette/__pycache__/testclient.cpython-314.pyc,, -starlette/__pycache__/types.cpython-314.pyc,, -starlette/__pycache__/websockets.cpython-314.pyc,, -starlette/_exception_handler.py,sha256=8Vm915fWYe9xLo_za5iQ5WyrAvFBu2kEEJc7XreTyOk,2257 -starlette/_utils.py,sha256=fLZ_pRlcp-wP4zEiyeXOIl7foyl_zIL5Am6E-cVlg8s,2970 -starlette/applications.py,sha256=H4R7Jvt8tWxIj7QoRRt8_gEVj2xrYnG2AfA6d-2en9I,5291 -starlette/authentication.py,sha256=bydUX4PKnTFJPSUj8B5k8HxYqrJ4eyyOma8AtgqwlD0,5202 -starlette/background.py,sha256=D-XpUpFRgUBnZeA1tHgzMOgube6ICVZFlOA4ML0q1T8,1122 -starlette/concurrency.py,sha256=Pp4kXQYO85cWxY7XVNt5lcYvhYHUquyLguCec7hKKoM,1701 -starlette/config.py,sha256=GVwko0pGSusxuQHO35tL_Nt_ApL0fmVYL3IiHHYpLW4,4426 -starlette/convertors.py,sha256=F1rse3AacN9rsfJnTeuDnjbN51r_ouHc3WLyYkjkX_o,2304 -starlette/datastructures.py,sha256=-eMQA2KZ1qQnRDypa6EY5YSG5ZJ81MO6wP-4FWJuUs0,23213 -starlette/endpoints.py,sha256=s5jPdHUT7VzUUyY-O5qfXDKYim-zbhWUCE-U5TsaFKQ,5271 -starlette/exceptions.py,sha256=45cuOyqdOjJJVZ2fRupfcSYvPDKqhkGZejjXw1cPGXE,1478 -starlette/formparsers.py,sha256=dItaict6sqVLmUs82dltsLUlDzPASBoZPIB8xyIlfsY,12079 -starlette/middleware/__init__.py,sha256=BORnOj0n0mL1OWUUs1PLTu1SASmvT8kEEDA6cbuhuPo,1349 -starlette/middleware/__pycache__/__init__.cpython-314.pyc,, -starlette/middleware/__pycache__/authentication.cpython-314.pyc,, -starlette/middleware/__pycache__/base.cpython-314.pyc,, -starlette/middleware/__pycache__/cors.cpython-314.pyc,, -starlette/middleware/__pycache__/errors.cpython-314.pyc,, -starlette/middleware/__pycache__/exceptions.cpython-314.pyc,, -starlette/middleware/__pycache__/gzip.cpython-314.pyc,, -starlette/middleware/__pycache__/httpsredirect.cpython-314.pyc,, -starlette/middleware/__pycache__/sessions.cpython-314.pyc,, -starlette/middleware/__pycache__/trustedhost.cpython-314.pyc,, -starlette/middleware/__pycache__/wsgi.cpython-314.pyc,, -starlette/middleware/authentication.py,sha256=-RSwwJ0pIkrM0v-2Ip8HXVB24xtSp7ObzR3OfTC8wSA,1800 -starlette/middleware/base.py,sha256=q90T3sHQjhr5kSIJyM4HNXJi9AWstlSNmozqT2_KUc0,10326 -starlette/middleware/cors.py,sha256=iBOpWmK43ENnyR71t4vH2WwfCWzaJNJdE7th3x5Cg80,7509 -starlette/middleware/errors.py,sha256=UZttzAUUejJtNZpPts0PXnYgV9C4h8o3sWDHRkN9ECg,8101 -starlette/middleware/exceptions.py,sha256=7OgSUiBgwHS4VMmpaWlw21uDKNDmOMzLVZrWDLDUqWo,2784 -starlette/middleware/gzip.py,sha256=_thpCRctguw0tMM6J2iDlAj5vZlol9T673IHtfvfxQE,5899 -starlette/middleware/httpsredirect.py,sha256=SNTleaYALGoITV7xwbic4gB6VYdM8Ylea_ykciUz31g,848 -starlette/middleware/sessions.py,sha256=kbSzZhMN8R2ofdrvMQEMz4ex4xEhW-sTSfsTKQ9SSGY,4778 -starlette/middleware/trustedhost.py,sha256=byKCUyPge54Z4MznyunD_2DsMfJc2UsfV4b2Du-WYTc,2219 -starlette/middleware/wsgi.py,sha256=BqXJpFjy3hbMuMa0_kjLAKskNPO0s7k-TPSRP-zcgLU,5501 -starlette/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -starlette/requests.py,sha256=hpGwGuSez6CF_mQZNDZ49zt3yS_GOkERBqd9jpvwrEQ,12708 -starlette/responses.py,sha256=XVKrAI732c5MUU8TuOxi4Voep48poRbw6Mvub07qkRI,21731 -starlette/routing.py,sha256=uV5qR75s8QqJvNGgrOc7qN6MbvW3NA_eSy3iRTsvWEQ,29579 -starlette/schemas.py,sha256=fBDlI1YDKF_rkQIalqBVQ0PQ9-3gNGh7T_39OyE7rV0,5168 -starlette/staticfiles.py,sha256=8GP3DKNN8W3VGZIRPZFPQBIDiqN-YaKcNN2EvaMMQYw,8786 -starlette/status.py,sha256=XYcVn-aU522TYJ8BJxl0WUnf6J_3yBTQz7Tqgye8KO4,6430 -starlette/templating.py,sha256=y-ATlQrNDoLMZs5hrHzQBTapECBDr9wAl-bs8Blk5jw,5447 -starlette/testclient.py,sha256=UqnjIiKkkrNoM5yr8DVxY94u1-gJWdfz6IKZxbjE60w,28356 -starlette/types.py,sha256=87wTuwF1036QxRse3mL5qOuS9i4a9xaIZMqTjY-lvvo,1041 -starlette/websockets.py,sha256=m64W1F4JdLw-7TRGVHqSqgreamlHbVJSOeIQ17EdHyg,8352 diff --git a/functions/python/data-processor/starlette-1.3.1.dist-info/WHEEL b/functions/python/data-processor/starlette-1.3.1.dist-info/WHEEL deleted file mode 100644 index bd8422674..000000000 --- a/functions/python/data-processor/starlette-1.3.1.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.30.1 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/starlette-1.3.1.dist-info/licenses/LICENSE.md b/functions/python/data-processor/starlette-1.3.1.dist-info/licenses/LICENSE.md deleted file mode 100644 index d16a60ec5..000000000 --- a/functions/python/data-processor/starlette-1.3.1.dist-info/licenses/LICENSE.md +++ /dev/null @@ -1,27 +0,0 @@ -Copyright © 2018, [Encode OSS Ltd](https://www.encode.io/). -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/functions/python/data-processor/starlette/__init__.py b/functions/python/data-processor/starlette/__init__.py deleted file mode 100644 index 9c73af26b..000000000 --- a/functions/python/data-processor/starlette/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__version__ = "1.3.1" diff --git a/functions/python/data-processor/starlette/_exception_handler.py b/functions/python/data-processor/starlette/_exception_handler.py deleted file mode 100644 index 9491f8bf0..000000000 --- a/functions/python/data-processor/starlette/_exception_handler.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from starlette._utils import is_async_callable -from starlette.concurrency import run_in_threadpool -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.types import ASGIApp, ExceptionHandler, Message, Receive, Scope, Send -from starlette.websockets import WebSocket - -ExceptionHandlers = dict[Any, ExceptionHandler] -StatusHandlers = dict[int, ExceptionHandler] - - -def _lookup_exception_handler(exc_handlers: ExceptionHandlers, exc: Exception) -> ExceptionHandler | None: - for cls in type(exc).__mro__: - if cls in exc_handlers: - return exc_handlers[cls] - return None - - -def wrap_app_handling_exceptions(app: ASGIApp, conn: Request | WebSocket) -> ASGIApp: - exception_handlers: ExceptionHandlers - status_handlers: StatusHandlers - try: - exception_handlers, status_handlers = conn.scope["starlette.exception_handlers"] - except KeyError: - exception_handlers, status_handlers = {}, {} - - async def wrapped_app(scope: Scope, receive: Receive, send: Send) -> None: - response_started = False - - async def sender(message: Message) -> None: - nonlocal response_started - - if message["type"] == "http.response.start": - response_started = True - await send(message) - - try: - await app(scope, receive, sender) - except Exception as exc: - handler = None - - if isinstance(exc, HTTPException): - handler = status_handlers.get(exc.status_code) - - if handler is None: - handler = _lookup_exception_handler(exception_handlers, exc) - - if handler is None: - raise exc - - if response_started: - raise RuntimeError("Caught handled exception, but response already started.") from exc - - if is_async_callable(handler): - response = await handler(conn, exc) # type: ignore[arg-type] - else: - response = await run_in_threadpool(handler, conn, exc) # type: ignore[arg-type] - if response is not None: - await response(scope, receive, sender) - - return wrapped_app diff --git a/functions/python/data-processor/starlette/_utils.py b/functions/python/data-processor/starlette/_utils.py deleted file mode 100644 index b9b8639b2..000000000 --- a/functions/python/data-processor/starlette/_utils.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -import functools -import sys -from collections.abc import AsyncGenerator, Awaitable, Callable, Generator -from contextlib import AbstractAsyncContextManager, asynccontextmanager -from typing import Any, Generic, Protocol, TypeVar, overload - -import anyio.abc - -from starlette.types import Scope - -if sys.version_info >= (3, 13): # pragma: no cover - from inspect import iscoroutinefunction - from typing import TypeIs -else: # pragma: no cover - from asyncio import iscoroutinefunction - - from typing_extensions import TypeIs - -if sys.version_info < (3, 11): # pragma: no cover - try: - from exceptiongroup import BaseExceptionGroup - except ImportError: - - class BaseExceptionGroup(BaseException): # type: ignore[no-redef] - pass - - -T = TypeVar("T") -AwaitableCallable = Callable[..., Awaitable[T]] - - -@overload -def is_async_callable(obj: AwaitableCallable[T]) -> TypeIs[AwaitableCallable[T]]: ... - - -@overload -def is_async_callable(obj: Any) -> TypeIs[AwaitableCallable[Any]]: ... - - -def is_async_callable(obj: Any) -> Any: - while isinstance(obj, functools.partial): - obj = obj.func - - return iscoroutinefunction(obj) or (callable(obj) and iscoroutinefunction(obj.__call__)) - - -T_co = TypeVar("T_co", covariant=True) - - -class AwaitableOrContextManager( - Awaitable[T_co], AbstractAsyncContextManager[T_co], Protocol[T_co] -): ... # pragma: no branch - - -class SupportsAsyncClose(Protocol): - async def close(self) -> None: ... # pragma: no cover - - -SupportsAsyncCloseType = TypeVar("SupportsAsyncCloseType", bound=SupportsAsyncClose, covariant=False) - - -class AwaitableOrContextManagerWrapper(Generic[SupportsAsyncCloseType]): - __slots__ = ("aw", "entered") - - def __init__(self, aw: Awaitable[SupportsAsyncCloseType]) -> None: - self.aw = aw - - def __await__(self) -> Generator[Any, None, SupportsAsyncCloseType]: - return self.aw.__await__() - - async def __aenter__(self) -> SupportsAsyncCloseType: - self.entered = await self.aw - return self.entered - - async def __aexit__(self, *args: Any) -> None | bool: - await self.entered.close() - return None - - -@asynccontextmanager -async def create_collapsing_task_group() -> AsyncGenerator[anyio.abc.TaskGroup, None]: - try: - async with anyio.create_task_group() as tg: - yield tg - except BaseExceptionGroup as excs: - if len(excs.exceptions) != 1: - raise - - exc = excs.exceptions[0] - context = None if exc.__suppress_context__ else exc.__context__ - raise exc from exc.__cause__ or context - - -def get_route_path(scope: Scope) -> str: - path: str = scope["path"] - root_path = scope.get("root_path", "") - if not root_path: - return path - - if not path.startswith(root_path): - return path - - if path == root_path: - return "" - - if path[len(root_path)] == "/": - return path[len(root_path) :] - - return path diff --git a/functions/python/data-processor/starlette/applications.py b/functions/python/data-processor/starlette/applications.py deleted file mode 100644 index 3e1a7816a..000000000 --- a/functions/python/data-processor/starlette/applications.py +++ /dev/null @@ -1,118 +0,0 @@ -from __future__ import annotations - -from collections.abc import Awaitable, Callable, Mapping, Sequence -from typing import Any, ParamSpec, TypeVar - -from starlette.datastructures import State, URLPath -from starlette.middleware import Middleware, _MiddlewareFactory -from starlette.middleware.errors import ServerErrorMiddleware -from starlette.middleware.exceptions import ExceptionMiddleware -from starlette.requests import Request -from starlette.responses import Response -from starlette.routing import BaseRoute, Router -from starlette.types import ASGIApp, ExceptionHandler, Lifespan, Receive, Scope, Send - -AppType = TypeVar("AppType", bound="Starlette") -P = ParamSpec("P") - - -class Starlette: - """Creates an Starlette application.""" - - def __init__( - self: AppType, - debug: bool = False, - routes: Sequence[BaseRoute] | None = None, - middleware: Sequence[Middleware] | None = None, - exception_handlers: Mapping[Any, ExceptionHandler] | None = None, - lifespan: Lifespan[AppType] | None = None, - ) -> None: - """Initializes the application. - - Parameters: - debug: Boolean indicating if debug tracebacks should be returned on errors. - routes: A list of routes to serve incoming HTTP and WebSocket requests. - middleware: A list of middleware to run for every request. A starlette - application will always automatically include two middleware classes. - `ServerErrorMiddleware` is added as the very outermost middleware, to handle - any uncaught errors occurring anywhere in the entire stack. - `ExceptionMiddleware` is added as the very innermost middleware, to deal - with handled exception cases occurring in the routing or endpoints. - exception_handlers: A mapping of either integer status codes, - or exception class types onto callables which handle the exceptions. - Exception handler callables should be of the form - `handler(request, exc) -> response` and may be either standard functions, or - async functions. - lifespan: A lifespan context function, which can be used to perform - startup and shutdown tasks. This is a newer style that replaces the - `on_startup` and `on_shutdown` handlers. Use one or the other, not both. - """ - self.debug = debug - self.state = State() - self.router = Router(routes, lifespan=lifespan) - self.exception_handlers = {} if exception_handlers is None else dict(exception_handlers) - self.user_middleware = [] if middleware is None else list(middleware) - self.middleware_stack: ASGIApp | None = None - - def build_middleware_stack(self) -> ASGIApp: - debug = self.debug - error_handler = None - exception_handlers: dict[Any, ExceptionHandler] = {} - - for key, value in self.exception_handlers.items(): - if key in (500, Exception): - error_handler = value - else: - exception_handlers[key] = value - - middleware = ( - [Middleware(ServerErrorMiddleware, handler=error_handler, debug=debug)] - + self.user_middleware - + [Middleware(ExceptionMiddleware, handlers=exception_handlers, debug=debug)] - ) - - app = self.router - for cls, args, kwargs in reversed(middleware): - app = cls(app, *args, **kwargs) - return app - - @property - def routes(self) -> list[BaseRoute]: - return self.router.routes - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - return self.router.url_path_for(name, **path_params) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - scope["app"] = self - if self.middleware_stack is None: - self.middleware_stack = self.build_middleware_stack() - await self.middleware_stack(scope, receive, send) - - def mount(self, path: str, app: ASGIApp, name: str | None = None) -> None: - self.router.mount(path, app=app, name=name) # pragma: no cover - - def host(self, host: str, app: ASGIApp, name: str | None = None) -> None: - self.router.host(host, app=app, name=name) # pragma: no cover - - def add_middleware(self, middleware_class: _MiddlewareFactory[P], *args: P.args, **kwargs: P.kwargs) -> None: - if self.middleware_stack is not None: # pragma: no cover - raise RuntimeError("Cannot add middleware after an application has started") - self.user_middleware.insert(0, Middleware(middleware_class, *args, **kwargs)) - - def add_exception_handler( - self, - exc_class_or_status_code: int | type[Exception], - handler: ExceptionHandler, - ) -> None: # pragma: no cover - self.exception_handlers[exc_class_or_status_code] = handler - - def add_route( - self, - path: str, - route: Callable[[Request], Awaitable[Response] | Response], - methods: list[str] | None = None, - name: str | None = None, - include_in_schema: bool = True, - ) -> None: # pragma: no cover - self.router.add_route(path, route, methods=methods, name=name, include_in_schema=include_in_schema) diff --git a/functions/python/data-processor/starlette/authentication.py b/functions/python/data-processor/starlette/authentication.py deleted file mode 100644 index 63c4da004..000000000 --- a/functions/python/data-processor/starlette/authentication.py +++ /dev/null @@ -1,149 +0,0 @@ -from __future__ import annotations - -import functools -import inspect -from collections.abc import Callable, Sequence -from typing import Any, ParamSpec -from urllib.parse import urlencode - -from starlette._utils import is_async_callable -from starlette.exceptions import HTTPException -from starlette.requests import HTTPConnection, Request -from starlette.responses import RedirectResponse -from starlette.websockets import WebSocket - -_P = ParamSpec("_P") - - -def has_required_scope(conn: HTTPConnection, scopes: Sequence[str]) -> bool: - for scope in scopes: - if scope not in conn.auth.scopes: - return False - return True - - -def requires( - scopes: str | Sequence[str], - status_code: int = 403, - redirect: str | None = None, -) -> Callable[[Callable[_P, Any]], Callable[_P, Any]]: - scopes_list = [scopes] if isinstance(scopes, str) else list(scopes) - - def decorator( - func: Callable[_P, Any], - ) -> Callable[_P, Any]: - sig = inspect.signature(func) - for idx, parameter in enumerate(sig.parameters.values()): - if parameter.name == "request" or parameter.name == "websocket": - type_ = parameter.name - break - else: - raise Exception(f'No "request" or "websocket" argument on function "{func}"') - - if type_ == "websocket": - # Handle websocket functions. (Always async) - @functools.wraps(func) - async def websocket_wrapper(*args: _P.args, **kwargs: _P.kwargs) -> None: - websocket = kwargs.get("websocket", args[idx] if idx < len(args) else None) - assert isinstance(websocket, WebSocket), ( - "Parameter with name 'websocket' is required to be of type 'WebSocket'" - f" not '{type(websocket).__name__}'" - ) - - if not has_required_scope(websocket, scopes_list): - await websocket.close() - else: - await func(*args, **kwargs) - - return websocket_wrapper - - elif is_async_callable(func): - # Handle async request/response functions. - @functools.wraps(func) - async def async_wrapper(*args: _P.args, **kwargs: _P.kwargs) -> Any: - request = kwargs.get("request", args[idx] if idx < len(args) else None) - assert isinstance(request, Request), ( - f"Parameter with name 'request' is required to be of type 'Request' not '{type(request).__name__}'" - ) - - if not has_required_scope(request, scopes_list): - if redirect is not None: - orig_request_qparam = urlencode({"next": str(request.url)}) - next_url = f"{request.url_for(redirect)}?{orig_request_qparam}" - return RedirectResponse(url=next_url, status_code=303) - raise HTTPException(status_code=status_code) - return await func(*args, **kwargs) - - return async_wrapper - - else: - # Handle sync request/response functions. - @functools.wraps(func) - def sync_wrapper(*args: _P.args, **kwargs: _P.kwargs) -> Any: - request = kwargs.get("request", args[idx] if idx < len(args) else None) - assert isinstance(request, Request), ( - f"Parameter with name 'request' is required to be of type 'Request' not '{type(request).__name__}'" - ) - - if not has_required_scope(request, scopes_list): - if redirect is not None: - orig_request_qparam = urlencode({"next": str(request.url)}) - next_url = f"{request.url_for(redirect)}?{orig_request_qparam}" - return RedirectResponse(url=next_url, status_code=303) - raise HTTPException(status_code=status_code) - return func(*args, **kwargs) - - return sync_wrapper - - return decorator - - -class AuthenticationError(Exception): - pass - - -class AuthenticationBackend: - async def authenticate(self, conn: HTTPConnection) -> tuple[AuthCredentials, BaseUser] | None: - raise NotImplementedError() # pragma: no cover - - -class AuthCredentials: - def __init__(self, scopes: Sequence[str] | None = None): - self.scopes = [] if scopes is None else list(scopes) - - -class BaseUser: - @property - def is_authenticated(self) -> bool: - raise NotImplementedError() # pragma: no cover - - @property - def display_name(self) -> str: - raise NotImplementedError() # pragma: no cover - - @property - def identity(self) -> str: - raise NotImplementedError() # pragma: no cover - - -class SimpleUser(BaseUser): - def __init__(self, username: str) -> None: - self.username = username - - @property - def is_authenticated(self) -> bool: - return True - - @property - def display_name(self) -> str: - return self.username - - -class UnauthenticatedUser(BaseUser): - @property - def is_authenticated(self) -> bool: - return False - - @property - def display_name(self) -> str: - return "" diff --git a/functions/python/data-processor/starlette/background.py b/functions/python/data-processor/starlette/background.py deleted file mode 100644 index 55135eed3..000000000 --- a/functions/python/data-processor/starlette/background.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Sequence -from typing import Any, ParamSpec - -from starlette._utils import is_async_callable -from starlette.concurrency import run_in_threadpool - -P = ParamSpec("P") - - -class BackgroundTask: - def __init__(self, func: Callable[P, Any], *args: P.args, **kwargs: P.kwargs) -> None: - self.func = func - self.args = args - self.kwargs = kwargs - self.is_async = is_async_callable(func) - - async def __call__(self) -> None: - if self.is_async: - await self.func(*self.args, **self.kwargs) - else: - await run_in_threadpool(self.func, *self.args, **self.kwargs) - - -class BackgroundTasks(BackgroundTask): - def __init__(self, tasks: Sequence[BackgroundTask] | None = None): - self.tasks = list(tasks) if tasks else [] - - def add_task(self, func: Callable[P, Any], *args: P.args, **kwargs: P.kwargs) -> None: - task = BackgroundTask(func, *args, **kwargs) - self.tasks.append(task) - - async def __call__(self) -> None: - for task in self.tasks: - await task() diff --git a/functions/python/data-processor/starlette/concurrency.py b/functions/python/data-processor/starlette/concurrency.py deleted file mode 100644 index a2c55be39..000000000 --- a/functions/python/data-processor/starlette/concurrency.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import annotations - -import functools -import warnings -from collections.abc import AsyncIterator, Callable, Coroutine, Iterable, Iterator -from typing import ParamSpec, TypeVar - -import anyio.to_thread - -from starlette.exceptions import StarletteDeprecationWarning - -P = ParamSpec("P") -T = TypeVar("T") - - -async def run_until_first_complete(*args: tuple[Callable, dict]) -> None: # type: ignore[type-arg] - warnings.warn( - "run_until_first_complete is deprecated and will be removed in a future version.", - StarletteDeprecationWarning, - ) - - async with anyio.create_task_group() as task_group: - - async def run(func: Callable[[], Coroutine]) -> None: # type: ignore[type-arg] - await func() - task_group.cancel_scope.cancel() - - for func, kwargs in args: - task_group.start_soon(run, functools.partial(func, **kwargs)) - - -async def run_in_threadpool(func: Callable[P, T], *args: P.args, **kwargs: P.kwargs) -> T: - func = functools.partial(func, *args, **kwargs) - return await anyio.to_thread.run_sync(func) - - -class _StopIteration(Exception): - pass - - -def _next(iterator: Iterator[T]) -> T: - # We can't raise `StopIteration` from within the threadpool iterator - # and catch it outside that context, so we coerce them into a different - # exception type. - try: - return next(iterator) - except StopIteration: - raise _StopIteration - - -async def iterate_in_threadpool( - iterator: Iterable[T], -) -> AsyncIterator[T]: - as_iterator = iter(iterator) - while True: - try: - yield await anyio.to_thread.run_sync(_next, as_iterator) - except _StopIteration: - break diff --git a/functions/python/data-processor/starlette/config.py b/functions/python/data-processor/starlette/config.py deleted file mode 100644 index 56c25ca3b..000000000 --- a/functions/python/data-processor/starlette/config.py +++ /dev/null @@ -1,140 +0,0 @@ -from __future__ import annotations - -import os -import warnings -from collections.abc import Callable, Iterator, Mapping, MutableMapping -from pathlib import Path -from typing import Any, TypeVar, overload - - -class undefined: - pass - - -class EnvironError(Exception): - pass - - -class Environ(MutableMapping[str, str]): - def __init__(self, environ: MutableMapping[str, str] = os.environ): - self._environ = environ - self._has_been_read: set[str] = set() - - def __getitem__(self, key: str) -> str: - self._has_been_read.add(key) - return self._environ.__getitem__(key) - - def __setitem__(self, key: str, value: str) -> None: - if key in self._has_been_read: - raise EnvironError(f"Attempting to set environ['{key}'], but the value has already been read.") - self._environ.__setitem__(key, value) - - def __delitem__(self, key: str) -> None: - if key in self._has_been_read: - raise EnvironError(f"Attempting to delete environ['{key}'], but the value has already been read.") - self._environ.__delitem__(key) - - def __iter__(self) -> Iterator[str]: - return iter(self._environ) - - def __len__(self) -> int: - return len(self._environ) - - -environ = Environ() - -T = TypeVar("T") - - -class Config: - def __init__( - self, - env_file: str | Path | None = None, - environ: Mapping[str, str] = environ, - env_prefix: str = "", - encoding: str = "utf-8", - ) -> None: - self.environ = environ - self.env_prefix = env_prefix - self.file_values: dict[str, str] = {} - if env_file is not None: - if not os.path.isfile(env_file): - warnings.warn(f"Config file '{env_file}' not found.") - else: - self.file_values = self._read_file(env_file, encoding) - - @overload - def __call__(self, key: str, *, default: None) -> str | None: ... - - @overload - def __call__(self, key: str, cast: type[T], default: T = ...) -> T: ... - - @overload - def __call__(self, key: str, cast: type[str] = ..., default: str = ...) -> str: ... - - @overload - def __call__( - self, - key: str, - cast: Callable[[Any], T] = ..., - default: Any = ..., - ) -> T: ... - - @overload - def __call__(self, key: str, cast: type[str] = ..., default: T = ...) -> T | str: ... - - def __call__( - self, - key: str, - cast: Callable[[Any], Any] | None = None, - default: Any = undefined, - ) -> Any: - return self.get(key, cast, default) - - def get( - self, - key: str, - cast: Callable[[Any], Any] | None = None, - default: Any = undefined, - ) -> Any: - key = self.env_prefix + key - if key in self.environ: - value = self.environ[key] - return self._perform_cast(key, value, cast) - if key in self.file_values: - value = self.file_values[key] - return self._perform_cast(key, value, cast) - if default is not undefined: - return self._perform_cast(key, default, cast) - raise KeyError(f"Config '{key}' is missing, and has no default.") - - def _read_file(self, file_name: str | Path, encoding: str) -> dict[str, str]: - file_values: dict[str, str] = {} - with open(file_name, encoding=encoding) as input_file: - for line in input_file.readlines(): - line = line.strip() - if "=" in line and not line.startswith("#"): - key, value = line.split("=", 1) - key = key.strip() - value = value.strip().strip("\"'") - file_values[key] = value - return file_values - - def _perform_cast( - self, - key: str, - value: Any, - cast: Callable[[Any], Any] | None = None, - ) -> Any: - if cast is None or value is None: - return value - elif cast is bool and isinstance(value, str): - mapping = {"true": True, "1": True, "false": False, "0": False} - value = value.lower() - if value not in mapping: - raise ValueError(f"Config '{key}' has value '{value}'. Not a valid bool.") - return mapping[value] - try: - return cast(value) - except (TypeError, ValueError): - raise ValueError(f"Config '{key}' has value '{value}'. Not a valid {cast.__name__}.") diff --git a/functions/python/data-processor/starlette/convertors.py b/functions/python/data-processor/starlette/convertors.py deleted file mode 100644 index 72b1cf9fd..000000000 --- a/functions/python/data-processor/starlette/convertors.py +++ /dev/null @@ -1,89 +0,0 @@ -from __future__ import annotations - -import math -import uuid -from typing import Any, ClassVar, Generic, TypeVar - -T = TypeVar("T") - - -class Convertor(Generic[T]): - regex: ClassVar[str] = "" - - def convert(self, value: str) -> T: - raise NotImplementedError() # pragma: no cover - - def to_string(self, value: T) -> str: - raise NotImplementedError() # pragma: no cover - - -class StringConvertor(Convertor[str]): - regex = "[^/]+" - - def convert(self, value: str) -> str: - return value - - def to_string(self, value: str) -> str: - value = str(value) - assert "/" not in value, "May not contain path separators" - assert value, "Must not be empty" - return value - - -class PathConvertor(Convertor[str]): - regex = ".*" - - def convert(self, value: str) -> str: - return str(value) - - def to_string(self, value: str) -> str: - return str(value) - - -class IntegerConvertor(Convertor[int]): - regex = "[0-9]+" - - def convert(self, value: str) -> int: - return int(value) - - def to_string(self, value: int) -> str: - value = int(value) - assert value >= 0, "Negative integers are not supported" - return str(value) - - -class FloatConvertor(Convertor[float]): - regex = r"[0-9]+(\.[0-9]+)?" - - def convert(self, value: str) -> float: - return float(value) - - def to_string(self, value: float) -> str: - value = float(value) - assert value >= 0.0, "Negative floats are not supported" - assert not math.isnan(value), "NaN values are not supported" - assert not math.isinf(value), "Infinite values are not supported" - return ("%0.20f" % value).rstrip("0").rstrip(".") - - -class UUIDConvertor(Convertor[uuid.UUID]): - regex = "[0-9a-fA-F]{8}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{4}-?[0-9a-fA-F]{12}" - - def convert(self, value: str) -> uuid.UUID: - return uuid.UUID(value) - - def to_string(self, value: uuid.UUID) -> str: - return str(value) - - -CONVERTOR_TYPES: dict[str, Convertor[Any]] = { - "str": StringConvertor(), - "path": PathConvertor(), - "int": IntegerConvertor(), - "float": FloatConvertor(), - "uuid": UUIDConvertor(), -} - - -def register_url_convertor(key: str, convertor: Convertor[Any]) -> None: - CONVERTOR_TYPES[key] = convertor diff --git a/functions/python/data-processor/starlette/datastructures.py b/functions/python/data-processor/starlette/datastructures.py deleted file mode 100644 index 33ecb1b12..000000000 --- a/functions/python/data-processor/starlette/datastructures.py +++ /dev/null @@ -1,704 +0,0 @@ -from __future__ import annotations - -import re -from collections.abc import ItemsView, Iterable, Iterator, KeysView, Mapping, MutableMapping, Sequence, ValuesView -from shlex import shlex -from typing import Any, BinaryIO, Literal, NamedTuple, TypeVar, cast -from urllib.parse import SplitResult, parse_qsl, urlencode, urlsplit - -from starlette.concurrency import run_in_threadpool -from starlette.types import Scope - - -class Address(NamedTuple): - host: str - port: int - - -_KeyType = TypeVar("_KeyType") -# Mapping keys are invariant but their values are covariant since -# you can only read them -# that is, you can't do `Mapping[str, Animal]()["fido"] = Dog()` -_CovariantValueType = TypeVar("_CovariantValueType", covariant=True) - -# Rejects Host header chars (/, ?, #, @, ...) that would let urlsplit produce a path differing from scope["path"]. -_HOST_RE = re.compile(r"^([a-z0-9.-]+|\[[a-f0-9]*:[a-f0-9.:]+\])(?::[0-9]+)?$", re.IGNORECASE) - - -class URL: - def __init__( - self, - url: str = "", - scope: Scope | None = None, - **components: Any, - ) -> None: - if scope is not None: - assert not url, 'Cannot set both "url" and "scope".' - assert not components, 'Cannot set both "scope" and "**components".' - scheme = scope.get("scheme", "http") - server = scope.get("server", None) - path = scope["path"] - query_string = scope.get("query_string", b"") - - host_header = None - for key, value in scope["headers"]: - if key == b"host": - host_header = value.decode("latin-1") - break - - if host_header is not None and _HOST_RE.fullmatch(host_header): - netloc = host_header - elif server is not None: - host, port = server - default_port = {"http": 80, "https": 443, "ws": 80, "wss": 443}[scheme] - netloc = host if port == default_port else f"{host}:{port}" - else: - netloc = None - - query = query_string.decode() - if netloc is not None: - url = SplitResult(scheme=scheme, netloc=netloc, path=path, query=query, fragment="").geturl() - else: - url = f"{path}?{query}" if query else path - elif components: - assert not url, 'Cannot set both "url" and "**components".' - url = URL("").replace(**components).components.geturl() - - self._url = url - - @property - def components(self) -> SplitResult: - if not hasattr(self, "_components"): - self._components = urlsplit(self._url) - return self._components - - @property - def scheme(self) -> str: - return self.components.scheme - - @property - def netloc(self) -> str: - return self.components.netloc - - @property - def path(self) -> str: - return self.components.path - - @property - def query(self) -> str: - return self.components.query - - @property - def fragment(self) -> str: - return self.components.fragment - - @property - def username(self) -> None | str: - return self.components.username - - @property - def password(self) -> None | str: - return self.components.password - - @property - def hostname(self) -> None | str: - return self.components.hostname - - @property - def port(self) -> int | None: - return self.components.port - - @property - def is_secure(self) -> bool: - return self.scheme in ("https", "wss") - - def replace(self, **kwargs: Any) -> URL: - if "username" in kwargs or "password" in kwargs or "hostname" in kwargs or "port" in kwargs: - hostname = kwargs.pop("hostname", None) - port = kwargs.pop("port", self.port) - username = kwargs.pop("username", self.username) - password = kwargs.pop("password", self.password) - - if hostname is None: - netloc = self.netloc - _, _, hostname = netloc.rpartition("@") - - if hostname and hostname[-1] != "]": - hostname = hostname.rsplit(":", 1)[0] - - netloc = hostname - if port is not None: - netloc += f":{port}" - if username is not None: - userpass = username - if password is not None: - userpass += f":{password}" - netloc = f"{userpass}@{netloc}" - - kwargs["netloc"] = netloc - - components = self.components._replace(**kwargs) - return self.__class__(components.geturl()) - - def include_query_params(self, **kwargs: Any) -> URL: - params = MultiDict(parse_qsl(self.query, keep_blank_values=True)) - params.update({str(key): str(value) for key, value in kwargs.items()}) - query = urlencode(params.multi_items()) - return self.replace(query=query) - - def replace_query_params(self, **kwargs: Any) -> URL: - query = urlencode([(str(key), str(value)) for key, value in kwargs.items()]) - return self.replace(query=query) - - def remove_query_params(self, keys: str | Sequence[str]) -> URL: - if isinstance(keys, str): - keys = [keys] - params = MultiDict(parse_qsl(self.query, keep_blank_values=True)) - for key in keys: - params.pop(key, None) - query = urlencode(params.multi_items()) - return self.replace(query=query) - - def __eq__(self, other: Any) -> bool: - return str(self) == str(other) - - def __str__(self) -> str: - return self._url - - def __repr__(self) -> str: - url = str(self) - if self.password: - url = str(self.replace(password="********")) - return f"{self.__class__.__name__}({repr(url)})" - - -class URLPath(str): - """ - A URL path string that may also hold an associated protocol and/or host. - Used by the routing to return `url_path_for` matches. - """ - - def __new__(cls, path: str, protocol: Literal["http", "websocket", ""] = "", host: str = "") -> URLPath: - assert protocol in ("http", "websocket", "") - return str.__new__(cls, path) - - def __init__(self, path: str, protocol: Literal["http", "websocket", ""] = "", host: str = "") -> None: - self.protocol = protocol - self.host = host - - def make_absolute_url(self, base_url: str | URL) -> URL: - if isinstance(base_url, str): - base_url = URL(base_url) - if self.protocol: - scheme = { - "http": {True: "https", False: "http"}, - "websocket": {True: "wss", False: "ws"}, - }[self.protocol][base_url.is_secure] - else: - scheme = base_url.scheme - - netloc = self.host or base_url.netloc - path = base_url.path.rstrip("/") + str(self) - return URL(scheme=scheme, netloc=netloc, path=path) - - -class Secret: - """ - Holds a string value that should not be revealed in tracebacks etc. - You should cast the value to `str` at the point it is required. - """ - - def __init__(self, value: str): - self._value = value - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - return f"{class_name}('**********')" - - def __str__(self) -> str: - return self._value - - def __bool__(self) -> bool: - return bool(self._value) - - -class CommaSeparatedStrings(Sequence[str]): - def __init__(self, value: str | Sequence[str]): - if isinstance(value, str): - splitter = shlex(value, posix=True) - splitter.whitespace = "," - splitter.whitespace_split = True - self._items = [item.strip() for item in splitter] - else: - self._items = list(value) - - def __len__(self) -> int: - return len(self._items) - - def __getitem__(self, index: int | slice) -> Any: - return self._items[index] - - def __iter__(self) -> Iterator[str]: - return iter(self._items) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - items = [item for item in self] - return f"{class_name}({items!r})" - - def __str__(self) -> str: - return ", ".join(repr(item) for item in self) - - -class ImmutableMultiDict(Mapping[_KeyType, _CovariantValueType]): - _dict: dict[_KeyType, _CovariantValueType] - - def __init__( - self, - *args: ImmutableMultiDict[_KeyType, _CovariantValueType] - | Mapping[_KeyType, _CovariantValueType] - | Iterable[tuple[_KeyType, _CovariantValueType]], - **kwargs: Any, - ) -> None: - assert len(args) < 2, "Too many arguments." - - value: Any = args[0] if args else [] - if kwargs: - value = ImmutableMultiDict(value).multi_items() + ImmutableMultiDict(kwargs).multi_items() - - if not value: - _items: list[tuple[Any, Any]] = [] - elif hasattr(value, "multi_items"): - value = cast(ImmutableMultiDict[_KeyType, _CovariantValueType], value) - _items = list(value.multi_items()) - elif hasattr(value, "items"): - value = cast(Mapping[_KeyType, _CovariantValueType], value) - _items = list(value.items()) - else: - value = cast("list[tuple[Any, Any]]", value) - _items = list(value) - - self._dict = {k: v for k, v in _items} - self._list = _items - - def getlist(self, key: Any) -> list[_CovariantValueType]: - return [item_value for item_key, item_value in self._list if item_key == key] - - def keys(self) -> KeysView[_KeyType]: - return self._dict.keys() - - def values(self) -> ValuesView[_CovariantValueType]: - return self._dict.values() - - def items(self) -> ItemsView[_KeyType, _CovariantValueType]: - return self._dict.items() - - def multi_items(self) -> list[tuple[_KeyType, _CovariantValueType]]: - return list(self._list) - - def __getitem__(self, key: _KeyType) -> _CovariantValueType: - return self._dict[key] - - def __contains__(self, key: Any) -> bool: - return key in self._dict - - def __iter__(self) -> Iterator[_KeyType]: - return iter(self.keys()) - - def __len__(self) -> int: - return len(self._dict) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, self.__class__): - return False - return sorted(self._list) == sorted(other._list) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - items = self.multi_items() - return f"{class_name}({items!r})" - - -class MultiDict(ImmutableMultiDict[Any, Any]): - def __setitem__(self, key: Any, value: Any) -> None: - self.setlist(key, [value]) - - def __delitem__(self, key: Any) -> None: - self._list = [(k, v) for k, v in self._list if k != key] - del self._dict[key] - - def pop(self, key: Any, default: Any = None) -> Any: - self._list = [(k, v) for k, v in self._list if k != key] - return self._dict.pop(key, default) - - def popitem(self) -> tuple[Any, Any]: - key, value = self._dict.popitem() - self._list = [(k, v) for k, v in self._list if k != key] - return key, value - - def poplist(self, key: Any) -> list[Any]: - values = [v for k, v in self._list if k == key] - self.pop(key) - return values - - def clear(self) -> None: - self._dict.clear() - self._list.clear() - - def setdefault(self, key: Any, default: Any = None) -> Any: - if key not in self: - self._dict[key] = default - self._list.append((key, default)) - - return self[key] - - def setlist(self, key: Any, values: list[Any]) -> None: - if not values: - self.pop(key, None) - else: - existing_items = [(k, v) for (k, v) in self._list if k != key] - self._list = existing_items + [(key, value) for value in values] - self._dict[key] = values[-1] - - def append(self, key: Any, value: Any) -> None: - self._list.append((key, value)) - self._dict[key] = value - - def update( - self, - *args: MultiDict | Mapping[Any, Any] | list[tuple[Any, Any]], - **kwargs: Any, - ) -> None: - value = MultiDict(*args, **kwargs) - existing_items = [(k, v) for (k, v) in self._list if k not in value.keys()] - self._list = existing_items + value.multi_items() - self._dict.update(value) - - -class QueryParams(ImmutableMultiDict[str, str]): - """ - An immutable multidict. - """ - - def __init__( - self, - *args: ImmutableMultiDict[Any, Any] | Mapping[Any, Any] | list[tuple[Any, Any]] | str | bytes, - **kwargs: Any, - ) -> None: - assert len(args) < 2, "Too many arguments." - - value = args[0] if args else [] - - if isinstance(value, str): - super().__init__(parse_qsl(value, keep_blank_values=True), **kwargs) - elif isinstance(value, bytes): - super().__init__(parse_qsl(value.decode("latin-1"), keep_blank_values=True), **kwargs) - else: - super().__init__(*args, **kwargs) # type: ignore[arg-type] - self._list = [(str(k), str(v)) for k, v in self._list] - self._dict = {str(k): str(v) for k, v in self._dict.items()} - - def __str__(self) -> str: - return urlencode(self._list) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - query_string = str(self) - return f"{class_name}({query_string!r})" - - -class UploadFile: - """ - An uploaded file included as part of the request data. - """ - - def __init__( - self, - file: BinaryIO, - *, - size: int | None = None, - filename: str | None = None, - headers: Headers | None = None, - ) -> None: - self.filename = filename - self.file = file - self.size = size - self.headers = headers or Headers() - - # Capture max size from SpooledTemporaryFile if one is provided. This slightly speeds up future checks. - # Note 0 means unlimited mirroring SpooledTemporaryFile's __init__ - self._max_mem_size = getattr(self.file, "_max_size", 0) - - @property - def content_type(self) -> str | None: - return self.headers.get("content-type", None) - - @property - def _in_memory(self) -> bool: - # check for SpooledTemporaryFile._rolled - rolled_to_disk = getattr(self.file, "_rolled", True) - return not rolled_to_disk - - def _will_roll(self, size_to_add: int) -> bool: - # If we're not in_memory then we will always roll - if not self._in_memory: - return True - - # Check for SpooledTemporaryFile._max_size - future_size = self.file.tell() + size_to_add - return bool(future_size > self._max_mem_size) if self._max_mem_size else False - - async def write(self, data: bytes) -> None: - new_data_len = len(data) - if self.size is not None: - self.size += new_data_len - - if self._will_roll(new_data_len): - await run_in_threadpool(self.file.write, data) - else: - self.file.write(data) - - async def read(self, size: int = -1) -> bytes: - if self._in_memory: - return self.file.read(size) - return await run_in_threadpool(self.file.read, size) - - async def seek(self, offset: int) -> None: - if self._in_memory: - self.file.seek(offset) - else: - await run_in_threadpool(self.file.seek, offset) - - async def close(self) -> None: - if self._in_memory: - self.file.close() - else: - await run_in_threadpool(self.file.close) - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(filename={self.filename!r}, size={self.size!r}, headers={self.headers!r})" - - -class FormData(ImmutableMultiDict[str, UploadFile | str]): - """ - An immutable multidict, containing both file uploads and text input. - """ - - def __init__( - self, - *args: FormData | Mapping[str, str | UploadFile] | list[tuple[str, str | UploadFile]], - **kwargs: str | UploadFile, - ) -> None: - super().__init__(*args, **kwargs) - - async def close(self) -> None: - for key, value in self.multi_items(): - if isinstance(value, UploadFile): - await value.close() - - -class Headers(Mapping[str, str]): - """ - An immutable, case-insensitive multidict. - """ - - def __init__( - self, - headers: Mapping[str, str] | None = None, - raw: list[tuple[bytes, bytes]] | None = None, - scope: MutableMapping[str, Any] | None = None, - ) -> None: - self._list: list[tuple[bytes, bytes]] = [] - if headers is not None: - assert raw is None, 'Cannot set both "headers" and "raw".' - assert scope is None, 'Cannot set both "headers" and "scope".' - self._list = [(key.lower().encode("latin-1"), value.encode("latin-1")) for key, value in headers.items()] - elif raw is not None: - assert scope is None, 'Cannot set both "raw" and "scope".' - self._list = raw - elif scope is not None: - # scope["headers"] isn't necessarily a list - # it might be a tuple or other iterable - self._list = scope["headers"] = list(scope["headers"]) - - @property - def raw(self) -> list[tuple[bytes, bytes]]: - return list(self._list) - - def keys(self) -> list[str]: # type: ignore[override] - return [key.decode("latin-1") for key, value in self._list] - - def values(self) -> list[str]: # type: ignore[override] - return [value.decode("latin-1") for key, value in self._list] - - def items(self) -> list[tuple[str, str]]: # type: ignore[override] - return [(key.decode("latin-1"), value.decode("latin-1")) for key, value in self._list] - - def getlist(self, key: str) -> list[str]: - get_header_key = key.lower().encode("latin-1") - return [item_value.decode("latin-1") for item_key, item_value in self._list if item_key == get_header_key] - - def mutablecopy(self) -> MutableHeaders: - return MutableHeaders(raw=self._list[:]) - - def __getitem__(self, key: str) -> str: - get_header_key = key.lower().encode("latin-1") - for header_key, header_value in self._list: - if header_key == get_header_key: - return header_value.decode("latin-1") - raise KeyError(key) - - def __contains__(self, key: Any) -> bool: - get_header_key = key.lower().encode("latin-1") - for header_key, header_value in self._list: - if header_key == get_header_key: - return True - return False - - def __iter__(self) -> Iterator[Any]: - return iter(self.keys()) - - def __len__(self) -> int: - return len(self._list) - - def __eq__(self, other: Any) -> bool: - if not isinstance(other, Headers): - return False - return sorted(self._list) == sorted(other._list) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - as_dict = dict(self.items()) - if len(as_dict) == len(self): - return f"{class_name}({as_dict!r})" - return f"{class_name}(raw={self.raw!r})" - - -class MutableHeaders(Headers): - def __setitem__(self, key: str, value: str) -> None: - """ - Set the header `key` to `value`, removing any duplicate entries. - Retains insertion order. - """ - set_key = key.lower().encode("latin-1") - set_value = value.encode("latin-1") - - found_indexes: list[int] = [] - for idx, (item_key, item_value) in enumerate(self._list): - if item_key == set_key: - found_indexes.append(idx) - - for idx in reversed(found_indexes[1:]): - del self._list[idx] - - if found_indexes: - idx = found_indexes[0] - self._list[idx] = (set_key, set_value) - else: - self._list.append((set_key, set_value)) - - def __delitem__(self, key: str) -> None: - """ - Remove the header `key`. - """ - del_key = key.lower().encode("latin-1") - - pop_indexes: list[int] = [] - for idx, (item_key, item_value) in enumerate(self._list): - if item_key == del_key: - pop_indexes.append(idx) - - for idx in reversed(pop_indexes): - del self._list[idx] - - def __ior__(self, other: Mapping[str, str]) -> MutableHeaders: - if not isinstance(other, Mapping): - raise TypeError(f"Expected a mapping but got {other.__class__.__name__}") - self.update(other) - return self - - def __or__(self, other: Mapping[str, str]) -> MutableHeaders: - if not isinstance(other, Mapping): - raise TypeError(f"Expected a mapping but got {other.__class__.__name__}") - new = self.mutablecopy() - new.update(other) - return new - - @property - def raw(self) -> list[tuple[bytes, bytes]]: - return self._list - - def setdefault(self, key: str, value: str) -> str: - """ - If the header `key` does not exist, then set it to `value`. - Returns the header value. - """ - set_key = key.lower().encode("latin-1") - set_value = value.encode("latin-1") - - for idx, (item_key, item_value) in enumerate(self._list): - if item_key == set_key: - return item_value.decode("latin-1") - self._list.append((set_key, set_value)) - return value - - def update(self, other: Mapping[str, str]) -> None: - for key, val in other.items(): - self[key] = val - - def append(self, key: str, value: str) -> None: - """ - Append a header, preserving any duplicate entries. - """ - append_key = key.lower().encode("latin-1") - append_value = value.encode("latin-1") - self._list.append((append_key, append_value)) - - def add_vary_header(self, vary: str) -> None: - existing = self.get("vary") - if existing is not None: - vary = ", ".join([existing, vary]) - self["vary"] = vary - - -class State: - """ - An object that can be used to store arbitrary state. - - Used for `request.state` and `app.state`. - """ - - _state: dict[str, Any] - - def __init__(self, state: dict[str, Any] | None = None): - if state is None: - state = {} - super().__setattr__("_state", state) - - def __setattr__(self, key: Any, value: Any) -> None: - self._state[key] = value - - def __getattr__(self, key: Any) -> Any: - try: - return self._state[key] - except KeyError: - message = "'{}' object has no attribute '{}'" - raise AttributeError(message.format(self.__class__.__name__, key)) - - def __delattr__(self, key: Any) -> None: - del self._state[key] - - def __getitem__(self, key: str) -> Any: - return self._state[key] - - def __setitem__(self, key: str, value: Any) -> None: - self._state[key] = value - - def __delitem__(self, key: str) -> None: - del self._state[key] - - def __iter__(self) -> Iterator[str]: - return iter(self._state) - - def __len__(self) -> int: - return len(self._state) diff --git a/functions/python/data-processor/starlette/endpoints.py b/functions/python/data-processor/starlette/endpoints.py deleted file mode 100644 index bdd51dce9..000000000 --- a/functions/python/data-processor/starlette/endpoints.py +++ /dev/null @@ -1,127 +0,0 @@ -from __future__ import annotations - -import json -from collections.abc import Callable, Generator -from typing import Any, Literal - -from starlette import status -from starlette._utils import is_async_callable -from starlette.concurrency import run_in_threadpool -from starlette.exceptions import HTTPException -from starlette.requests import Request -from starlette.responses import PlainTextResponse, Response -from starlette.types import Message, Receive, Scope, Send -from starlette.websockets import WebSocket - - -class HTTPEndpoint: - def __init__(self, scope: Scope, receive: Receive, send: Send) -> None: - assert scope["type"] == "http" - self.scope = scope - self.receive = receive - self.send = send - self._allowed_methods = [ - method - for method in ("GET", "HEAD", "POST", "PUT", "PATCH", "DELETE", "OPTIONS") - if getattr(self, method.lower(), None) is not None - ] - - def __await__(self) -> Generator[Any, None, None]: - return self.dispatch().__await__() - - async def dispatch(self) -> None: - request = Request(self.scope, receive=self.receive) - handler_name = "get" if request.method == "HEAD" and not hasattr(self, "head") else request.method.lower() - - handler: Callable[[Request], Any] - if request.method in self._allowed_methods or (request.method == "HEAD" and "GET" in self._allowed_methods): - handler = getattr(self, handler_name) - else: - handler = self.method_not_allowed - is_async = is_async_callable(handler) - if is_async: - response = await handler(request) - else: - response = await run_in_threadpool(handler, request) - await response(self.scope, self.receive, self.send) - - async def method_not_allowed(self, request: Request) -> Response: - # If we're running inside a starlette application then raise an - # exception, so that the configurable exception handler can deal with - # returning the response. For plain ASGI apps, just return the response. - headers = {"Allow": ", ".join(self._allowed_methods)} - if "app" in self.scope: - raise HTTPException(status_code=405, headers=headers) - return PlainTextResponse("Method Not Allowed", status_code=405, headers=headers) - - -class WebSocketEndpoint: - encoding: Literal["text", "bytes", "json"] | None = None - - def __init__(self, scope: Scope, receive: Receive, send: Send) -> None: - assert scope["type"] == "websocket" - self.scope = scope - self.receive = receive - self.send = send - - def __await__(self) -> Generator[Any, None, None]: - return self.dispatch().__await__() - - async def dispatch(self) -> None: - websocket = WebSocket(self.scope, receive=self.receive, send=self.send) - await self.on_connect(websocket) - - close_code = status.WS_1000_NORMAL_CLOSURE - - try: - while True: - message = await websocket.receive() - if message["type"] == "websocket.receive": - data = await self.decode(websocket, message) - await self.on_receive(websocket, data) - elif message["type"] == "websocket.disconnect": # pragma: no branch - close_code = int(message.get("code") or status.WS_1000_NORMAL_CLOSURE) - break - except Exception as exc: - close_code = status.WS_1011_INTERNAL_ERROR - raise exc - finally: - await self.on_disconnect(websocket, close_code) - - async def decode(self, websocket: WebSocket, message: Message) -> Any: - if self.encoding == "text": - if "text" not in message: - await websocket.close(code=status.WS_1003_UNSUPPORTED_DATA) - raise RuntimeError("Expected text websocket messages, but got bytes") - return message["text"] - - elif self.encoding == "bytes": - if "bytes" not in message: - await websocket.close(code=status.WS_1003_UNSUPPORTED_DATA) - raise RuntimeError("Expected bytes websocket messages, but got text") - return message["bytes"] - - elif self.encoding == "json": - if message.get("text") is not None: - text = message["text"] - else: - text = message["bytes"].decode("utf-8") - - try: - return json.loads(text) - except json.decoder.JSONDecodeError: - await websocket.close(code=status.WS_1003_UNSUPPORTED_DATA) - raise RuntimeError("Malformed JSON data received.") - - assert self.encoding is None, f"Unsupported 'encoding' attribute {self.encoding}" - return message["text"] if message.get("text") else message["bytes"] - - async def on_connect(self, websocket: WebSocket) -> None: - """Override to handle an incoming websocket connection""" - await websocket.accept() - - async def on_receive(self, websocket: WebSocket, data: Any) -> None: - """Override to handle an incoming websocket message""" - - async def on_disconnect(self, websocket: WebSocket, close_code: int) -> None: - """Override to handle a disconnecting websocket""" diff --git a/functions/python/data-processor/starlette/exceptions.py b/functions/python/data-processor/starlette/exceptions.py deleted file mode 100644 index 3cab4a479..000000000 --- a/functions/python/data-processor/starlette/exceptions.py +++ /dev/null @@ -1,43 +0,0 @@ -from __future__ import annotations - -import http -from collections.abc import Mapping - - -class HTTPException(Exception): - def __init__(self, status_code: int, detail: str | None = None, headers: Mapping[str, str] | None = None) -> None: - if detail is None: - detail = http.HTTPStatus(status_code).phrase - self.status_code = status_code - self.detail = detail - self.headers = headers - - def __str__(self) -> str: - return f"{self.status_code}: {self.detail}" - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - return f"{class_name}(status_code={self.status_code!r}, detail={self.detail!r})" - - -class WebSocketException(Exception): - def __init__(self, code: int, reason: str | None = None) -> None: - self.code = code - self.reason = reason or "" - - def __str__(self) -> str: - return f"{self.code}: {self.reason}" - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - return f"{class_name}(code={self.code!r}, reason={self.reason!r})" - - -class StarletteDeprecationWarning(UserWarning): - """A custom deprecation warning for Starlette. - - Unlike the built-in DeprecationWarning, this inherits from UserWarning to ensure it is visible by default, helping - users discover deprecated features without needing to enable warnings explicitly. - - Reference: https://sethmlarson.dev/deprecations-via-warnings-dont-work-for-python-libraries - """ diff --git a/functions/python/data-processor/starlette/formparsers.py b/functions/python/data-processor/starlette/formparsers.py deleted file mode 100644 index 3f3c598f4..000000000 --- a/functions/python/data-processor/starlette/formparsers.py +++ /dev/null @@ -1,297 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncGenerator -from dataclasses import dataclass, field -from enum import Enum -from tempfile import SpooledTemporaryFile -from typing import TYPE_CHECKING -from urllib.parse import unquote_plus - -from starlette.datastructures import FormData, Headers, UploadFile - -if TYPE_CHECKING: - import python_multipart as multipart - from python_multipart.multipart import MultipartCallbacks, QuerystringCallbacks, parse_options_header -else: - try: - try: - import python_multipart as multipart - from python_multipart.multipart import parse_options_header - except ModuleNotFoundError: # pragma: no cover - import multipart - from multipart.multipart import parse_options_header - except ModuleNotFoundError: # pragma: no cover - multipart = None - parse_options_header = None - - -class FormMessage(Enum): - FIELD_START = 1 - FIELD_NAME = 2 - FIELD_DATA = 3 - FIELD_END = 4 - END = 5 - - -@dataclass -class MultipartPart: - content_disposition: bytes | None = None - field_name: str = "" - data: bytearray = field(default_factory=bytearray) - file: UploadFile | None = None - item_headers: list[tuple[bytes, bytes]] = field(default_factory=list) - - -def _user_safe_decode(src: bytes | bytearray, codec: str) -> str: - try: - return src.decode(codec) - except (UnicodeDecodeError, LookupError): - return src.decode("latin-1") - - -class MultiPartException(Exception): - def __init__(self, message: str) -> None: - self.message = message - - -class FormParser: - def __init__( - self, - headers: Headers, - stream: AsyncGenerator[bytes, None], - *, - max_fields: int | float = 1000, - max_part_size: int = 1024 * 1024, # 1MB - ) -> None: - assert multipart is not None, "The `python-multipart` library must be installed to use form parsing." - self.headers = headers - self.stream = stream - self.max_fields = max_fields - self.max_part_size = max_part_size - self.messages: list[tuple[FormMessage, bytes]] = [] - self._current_field_size = 0 - self._current_fields = 0 - - def on_field_start(self) -> None: - self._current_field_size = 0 - message = (FormMessage.FIELD_START, b"") - self.messages.append(message) - - def on_field_name(self, data: bytes, start: int, end: int) -> None: - self._current_field_size += end - start - if self._current_field_size > self.max_part_size: - raise MultiPartException(f"Field exceeded maximum size of {int(self.max_part_size / 1024)}KB.") - message = (FormMessage.FIELD_NAME, data[start:end]) - self.messages.append(message) - - def on_field_data(self, data: bytes, start: int, end: int) -> None: - self._current_field_size += end - start - if self._current_field_size > self.max_part_size: - raise MultiPartException(f"Field exceeded maximum size of {int(self.max_part_size / 1024)}KB.") - message = (FormMessage.FIELD_DATA, data[start:end]) - self.messages.append(message) - - def on_field_end(self) -> None: - self._current_fields += 1 - if self._current_fields > self.max_fields: - raise MultiPartException(f"Too many fields. Maximum number of fields is {self.max_fields}.") - message = (FormMessage.FIELD_END, b"") - self.messages.append(message) - - def on_end(self) -> None: - message = (FormMessage.END, b"") - self.messages.append(message) - - async def parse(self) -> FormData: - # Callbacks dictionary. - callbacks: QuerystringCallbacks = { - "on_field_start": self.on_field_start, - "on_field_name": self.on_field_name, - "on_field_data": self.on_field_data, - "on_field_end": self.on_field_end, - "on_end": self.on_end, - } - - # Create the parser. - parser = multipart.QuerystringParser(callbacks) - field_name = bytearray() - field_value = bytearray() - - items: list[tuple[str, str | UploadFile]] = [] - - # Feed the parser with data from the request. - async for chunk in self.stream: - if chunk: - parser.write(chunk) - else: - parser.finalize() - messages = list(self.messages) - self.messages.clear() - for message_type, message_bytes in messages: - if message_type == FormMessage.FIELD_START: - field_name = bytearray() - field_value = bytearray() - elif message_type == FormMessage.FIELD_NAME: - field_name.extend(message_bytes) - elif message_type == FormMessage.FIELD_DATA: - field_value.extend(message_bytes) - elif message_type == FormMessage.FIELD_END: - name = unquote_plus(field_name.decode("latin-1")) - value = unquote_plus(field_value.decode("latin-1")) - items.append((name, value)) - - return FormData(items) - - -class MultiPartParser: - spool_max_size = 1024 * 1024 # 1MB - """The maximum size of the spooled temporary file used to store file data.""" - max_part_size = 1024 * 1024 # 1MB - """The maximum size of a part in the multipart request.""" - - def __init__( - self, - headers: Headers, - stream: AsyncGenerator[bytes, None], - *, - max_files: int | float = 1000, - max_fields: int | float = 1000, - max_part_size: int = 1024 * 1024, # 1MB - ) -> None: - assert multipart is not None, "The `python-multipart` library must be installed to use form parsing." - self.headers = headers - self.stream = stream - self.max_files = max_files - self.max_fields = max_fields - self.items: list[tuple[str, str | UploadFile]] = [] - self._current_files = 0 - self._current_fields = 0 - self._current_partial_header_name: bytes = b"" - self._current_partial_header_value: bytes = b"" - self._current_part = MultipartPart() - self._charset = "" - self._file_parts_to_write: list[tuple[MultipartPart, bytes]] = [] - self._file_parts_to_finish: list[MultipartPart] = [] - self._files_to_close_on_error: list[SpooledTemporaryFile[bytes]] = [] - self.max_part_size = max_part_size - - def on_part_begin(self) -> None: - self._current_part = MultipartPart() - - def on_part_data(self, data: bytes, start: int, end: int) -> None: - message_bytes = data[start:end] - if self._current_part.file is None: - if len(self._current_part.data) + len(message_bytes) > self.max_part_size: - raise MultiPartException(f"Part exceeded maximum size of {int(self.max_part_size / 1024)}KB.") - self._current_part.data.extend(message_bytes) - else: - self._file_parts_to_write.append((self._current_part, message_bytes)) - - def on_part_end(self) -> None: - if self._current_part.file is None: - self.items.append( - ( - self._current_part.field_name, - _user_safe_decode(self._current_part.data, self._charset), - ) - ) - else: - self._file_parts_to_finish.append(self._current_part) - # The file can be added to the items right now even though it's not - # finished yet, because it will be finished in the `parse()` method, before - # self.items is used in the return value. - self.items.append((self._current_part.field_name, self._current_part.file)) - - def on_header_field(self, data: bytes, start: int, end: int) -> None: - self._current_partial_header_name += data[start:end] - - def on_header_value(self, data: bytes, start: int, end: int) -> None: - self._current_partial_header_value += data[start:end] - - def on_header_end(self) -> None: - field = self._current_partial_header_name.lower() - if field == b"content-disposition": - self._current_part.content_disposition = self._current_partial_header_value - self._current_part.item_headers.append((field, self._current_partial_header_value)) - self._current_partial_header_name = b"" - self._current_partial_header_value = b"" - - def on_headers_finished(self) -> None: - disposition, options = parse_options_header(self._current_part.content_disposition) - try: - self._current_part.field_name = _user_safe_decode(options[b"name"], self._charset) - except KeyError: - raise MultiPartException('The Content-Disposition header field "name" must be provided.') - if b"filename" in options: - self._current_files += 1 - if self._current_files > self.max_files: - raise MultiPartException(f"Too many files. Maximum number of files is {self.max_files}.") - filename = _user_safe_decode(options[b"filename"], self._charset) - tempfile = SpooledTemporaryFile(max_size=self.spool_max_size) - self._files_to_close_on_error.append(tempfile) - self._current_part.file = UploadFile( - file=tempfile, # type: ignore[arg-type] - size=0, - filename=filename, - headers=Headers(raw=self._current_part.item_headers), - ) - else: - self._current_fields += 1 - if self._current_fields > self.max_fields: - raise MultiPartException(f"Too many fields. Maximum number of fields is {self.max_fields}.") - self._current_part.file = None - - def on_end(self) -> None: - pass - - async def parse(self) -> FormData: - # Parse the Content-Type header to get the multipart boundary. - _, params = parse_options_header(self.headers["Content-Type"]) - charset = params.get(b"charset", "utf-8") - if isinstance(charset, bytes): - charset = charset.decode("latin-1") - self._charset = charset - try: - boundary = params[b"boundary"] - except KeyError: - raise MultiPartException("Missing boundary in multipart.") - - # Callbacks dictionary. - callbacks: MultipartCallbacks = { - "on_part_begin": self.on_part_begin, - "on_part_data": self.on_part_data, - "on_part_end": self.on_part_end, - "on_header_field": self.on_header_field, - "on_header_value": self.on_header_value, - "on_header_end": self.on_header_end, - "on_headers_finished": self.on_headers_finished, - "on_end": self.on_end, - } - - # Create the parser. - parser = multipart.MultipartParser(boundary, callbacks) - try: - # Feed the parser with data from the request. - async for chunk in self.stream: - parser.write(chunk) - # Write file data, it needs to use await with the UploadFile methods - # that call the corresponding file methods *in a threadpool*, - # otherwise, if they were called directly in the callback methods above - # (regular, non-async functions), that would block the event loop in - # the main thread. - for part, data in self._file_parts_to_write: - assert part.file # for type checkers - await part.file.write(data) - for part in self._file_parts_to_finish: - assert part.file # for type checkers - await part.file.seek(0) - self._file_parts_to_write.clear() - self._file_parts_to_finish.clear() - parser.finalize() - except (MultiPartException, OSError) as exc: - # Close all the files if there was an error. - for file in self._files_to_close_on_error: - file.close() - raise exc - - return FormData(self.items) diff --git a/functions/python/data-processor/starlette/middleware/__init__.py b/functions/python/data-processor/starlette/middleware/__init__.py deleted file mode 100644 index f8d47d7a0..000000000 --- a/functions/python/data-processor/starlette/middleware/__init__.py +++ /dev/null @@ -1,37 +0,0 @@ -from __future__ import annotations - -from collections.abc import Awaitable, Callable, Iterator -from typing import Any, ParamSpec, Protocol - -P = ParamSpec("P") - - -_Scope = Any -_Receive = Callable[[], Awaitable[Any]] -_Send = Callable[[Any], Awaitable[None]] -# Since `starlette.types.ASGIApp` type differs from `ASGIApplication` from `asgiref` -# we need to define a more permissive version of ASGIApp that doesn't cause type errors. -_ASGIApp = Callable[[_Scope, _Receive, _Send], Awaitable[None]] - - -class _MiddlewareFactory(Protocol[P]): - def __call__(self, app: _ASGIApp, /, *args: P.args, **kwargs: P.kwargs) -> _ASGIApp: ... # pragma: no cover - - -class Middleware: - def __init__(self, cls: _MiddlewareFactory[P], *args: P.args, **kwargs: P.kwargs) -> None: - self.cls = cls - self.args = args - self.kwargs = kwargs - - def __iter__(self) -> Iterator[Any]: - as_tuple = (self.cls, self.args, self.kwargs) - return iter(as_tuple) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - args_strings = [f"{value!r}" for value in self.args] - option_strings = [f"{key}={value!r}" for key, value in self.kwargs.items()] - name = getattr(self.cls, "__name__", "") - args_repr = ", ".join([name] + args_strings + option_strings) - return f"{class_name}({args_repr})" diff --git a/functions/python/data-processor/starlette/middleware/authentication.py b/functions/python/data-processor/starlette/middleware/authentication.py deleted file mode 100644 index 935385fd3..000000000 --- a/functions/python/data-processor/starlette/middleware/authentication.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable - -from starlette.authentication import ( - AuthCredentials, - AuthenticationBackend, - AuthenticationError, - UnauthenticatedUser, -) -from starlette.requests import HTTPConnection -from starlette.responses import PlainTextResponse, Response -from starlette.types import ASGIApp, Receive, Scope, Send - - -class AuthenticationMiddleware: - def __init__( - self, - app: ASGIApp, - backend: AuthenticationBackend, - on_error: Callable[[HTTPConnection, AuthenticationError], Response] | None = None, - ) -> None: - self.app = app - self.backend = backend - self.on_error: Callable[[HTTPConnection, AuthenticationError], Response] = ( - on_error if on_error is not None else self.default_on_error - ) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] not in ["http", "websocket"]: - await self.app(scope, receive, send) - return - - conn = HTTPConnection(scope) - try: - auth_result = await self.backend.authenticate(conn) - except AuthenticationError as exc: - response = self.on_error(conn, exc) - if scope["type"] == "websocket": - await send({"type": "websocket.close", "code": 1000}) - else: - await response(scope, receive, send) - return - - if auth_result is None: - auth_result = AuthCredentials(), UnauthenticatedUser() - scope["auth"], scope["user"] = auth_result - await self.app(scope, receive, send) - - @staticmethod - def default_on_error(conn: HTTPConnection, exc: Exception) -> Response: - return PlainTextResponse(str(exc), status_code=400) diff --git a/functions/python/data-processor/starlette/middleware/base.py b/functions/python/data-processor/starlette/middleware/base.py deleted file mode 100644 index 2579c031b..000000000 --- a/functions/python/data-processor/starlette/middleware/base.py +++ /dev/null @@ -1,244 +0,0 @@ -from __future__ import annotations - -from collections.abc import AsyncGenerator, AsyncIterable, Awaitable, Callable, Mapping, MutableMapping -from typing import Any, TypeVar - -import anyio - -from starlette._utils import create_collapsing_task_group -from starlette.requests import ClientDisconnect, Request -from starlette.responses import Response -from starlette.types import ASGIApp, Message, Receive, Scope, Send - -RequestResponseEndpoint = Callable[[Request], Awaitable[Response]] -DispatchFunction = Callable[[Request, RequestResponseEndpoint], Awaitable[Response]] -BodyStreamGenerator = AsyncGenerator[bytes | MutableMapping[str, Any], None] -AsyncContentStream = AsyncIterable[str | bytes | memoryview | MutableMapping[str, Any]] -T = TypeVar("T") - - -class _CachedRequest(Request): - """ - If the user calls Request.body() from their dispatch function - we cache the entire request body in memory and pass that to downstream middlewares, - but if they call Request.stream() then all we do is send an - empty body so that downstream things don't hang forever. - """ - - def __init__(self, scope: Scope, receive: Receive): - super().__init__(scope, receive) - self._wrapped_rcv_disconnected = False - self._wrapped_rcv_consumed = False - self._wrapped_rc_stream = self.stream() - - async def wrapped_receive(self) -> Message: - # wrapped_rcv state 1: disconnected - if self._wrapped_rcv_disconnected: - # we've already sent a disconnect to the downstream app - # we don't need to wait to get another one - # (although most ASGI servers will just keep sending it) - return {"type": "http.disconnect"} - # wrapped_rcv state 1: consumed but not yet disconnected - if self._wrapped_rcv_consumed: - # since the downstream app has consumed us all that is left - # is to send it a disconnect - if self._is_disconnected: - # the middleware has already seen the disconnect - # since we know the client is disconnected no need to wait - # for the message - self._wrapped_rcv_disconnected = True - return {"type": "http.disconnect"} - # we don't know yet if the client is disconnected or not - # so we'll wait until we get that message - msg = await self.receive() - if msg["type"] != "http.disconnect": # pragma: no cover - # at this point a disconnect is all that we should be receiving - # if we get something else, things went wrong somewhere - raise RuntimeError(f"Unexpected message received: {msg['type']}") - self._wrapped_rcv_disconnected = True - return msg - - # wrapped_rcv state 3: not yet consumed - if getattr(self, "_body", None) is not None: - # body() was called, we return it even if the client disconnected - self._wrapped_rcv_consumed = True - return { - "type": "http.request", - "body": self._body, - "more_body": False, - } - elif self._stream_consumed: - # stream() was called to completion - # return an empty body so that downstream apps don't hang - # waiting for a disconnect - self._wrapped_rcv_consumed = True - return { - "type": "http.request", - "body": b"", - "more_body": False, - } - else: - # body() was never called and stream() wasn't consumed - try: - stream = self.stream() - chunk = await stream.__anext__() - self._wrapped_rcv_consumed = self._stream_consumed - return { - "type": "http.request", - "body": chunk, - "more_body": not self._stream_consumed, - } - except ClientDisconnect: - self._wrapped_rcv_disconnected = True - return {"type": "http.disconnect"} - - -class BaseHTTPMiddleware: - def __init__(self, app: ASGIApp, dispatch: DispatchFunction | None = None) -> None: - self.app = app - self.dispatch_func = self.dispatch if dispatch is None else dispatch - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - request = _CachedRequest(scope, receive) - wrapped_receive = request.wrapped_receive - response_sent = anyio.Event() - app_exc: Exception | None = None - exception_already_raised = False - - async def call_next(request: Request) -> Response: - async def receive_or_disconnect() -> Message: - if response_sent.is_set(): - return {"type": "http.disconnect"} - - async with anyio.create_task_group() as task_group: - - async def wrap(func: Callable[[], Awaitable[T]]) -> T: - result = await func() - task_group.cancel_scope.cancel() - return result - - task_group.start_soon(wrap, response_sent.wait) - message = await wrap(wrapped_receive) - - if response_sent.is_set(): - return {"type": "http.disconnect"} - - return message - - async def send_no_error(message: Message) -> None: - try: - await send_stream.send(message) - except anyio.BrokenResourceError: - # recv_stream has been closed, i.e. response_sent has been set. - return - - async def coro() -> None: - nonlocal app_exc - - with send_stream: - try: - await self.app(scope, receive_or_disconnect, send_no_error) - except Exception as exc: - app_exc = exc - - task_group.start_soon(coro) - - try: - message = await recv_stream.receive() - info = message.get("info", None) - if message["type"] == "http.response.debug" and info is not None: - message = await recv_stream.receive() - except anyio.EndOfStream: - if app_exc is not None: - nonlocal exception_already_raised - exception_already_raised = True - # Prevent `anyio.EndOfStream` from polluting app exception context. - # If both cause and context are None then the context is suppressed - # and `anyio.EndOfStream` is not present in the exception traceback. - # If exception cause is not None then it is propagated with - # reraising here. - # If exception has no cause but has context set then the context is - # propagated as a cause with the reraise. This is necessary in order - # to prevent `anyio.EndOfStream` from polluting the exception - # context. - raise app_exc from app_exc.__cause__ or app_exc.__context__ - raise RuntimeError("No response returned.") - - assert message["type"] == "http.response.start" - - async def body_stream() -> BodyStreamGenerator: - async for message in recv_stream: - if message["type"] == "http.response.pathsend": - yield message - break - assert message["type"] == "http.response.body", f"Unexpected message: {message}" - body = message.get("body", b"") - if body: - yield body - if not message.get("more_body", False): - break - - response = _StreamingResponse(status_code=message["status"], content=body_stream(), info=info) - response.raw_headers = message["headers"] - return response - - streams: anyio.create_memory_object_stream[Message] = anyio.create_memory_object_stream() - send_stream, recv_stream = streams - with recv_stream, send_stream: - async with create_collapsing_task_group() as task_group: - response = await self.dispatch_func(request, call_next) - await response(scope, wrapped_receive, send) - response_sent.set() - recv_stream.close() - if app_exc is not None and not exception_already_raised: - raise app_exc - - async def dispatch(self, request: Request, call_next: RequestResponseEndpoint) -> Response: - raise NotImplementedError() # pragma: no cover - - -class _StreamingResponse(Response): - def __init__( - self, - content: AsyncContentStream, - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - info: Mapping[str, Any] | None = None, - ) -> None: - self.info = info - self.body_iterator = content - self.status_code = status_code - self.media_type = media_type - self.init_headers(headers) - self.background = None - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if self.info is not None: - await send({"type": "http.response.debug", "info": self.info}) - await send( - { - "type": "http.response.start", - "status": self.status_code, - "headers": self.raw_headers, - } - ) - - should_close_body = True - async for chunk in self.body_iterator: - if isinstance(chunk, dict): - # We got an ASGI message which is not response body (eg: pathsend) - should_close_body = False - await send(chunk) - continue - await send({"type": "http.response.body", "body": chunk, "more_body": True}) - - if should_close_body: - await send({"type": "http.response.body", "body": b"", "more_body": False}) - - if self.background: - await self.background() diff --git a/functions/python/data-processor/starlette/middleware/cors.py b/functions/python/data-processor/starlette/middleware/cors.py deleted file mode 100644 index b65fc15a0..000000000 --- a/functions/python/data-processor/starlette/middleware/cors.py +++ /dev/null @@ -1,179 +0,0 @@ -from __future__ import annotations - -import functools -import re -from collections.abc import Sequence - -from starlette.datastructures import Headers, MutableHeaders -from starlette.responses import PlainTextResponse, Response -from starlette.types import ASGIApp, Message, Receive, Scope, Send - -ALL_METHODS = ("DELETE", "GET", "HEAD", "OPTIONS", "PATCH", "POST", "PUT") -SAFELISTED_HEADERS = {"Accept", "Accept-Language", "Content-Language", "Content-Type"} - - -class CORSMiddleware: - def __init__( - self, - app: ASGIApp, - allow_origins: Sequence[str] = (), - allow_methods: Sequence[str] = ("GET",), - allow_headers: Sequence[str] = (), - allow_credentials: bool = False, - allow_origin_regex: str | None = None, - allow_private_network: bool = False, - expose_headers: Sequence[str] = (), - max_age: int = 600, - ) -> None: - if "*" in allow_methods: - allow_methods = ALL_METHODS - - compiled_allow_origin_regex = None - if allow_origin_regex is not None: - compiled_allow_origin_regex = re.compile(allow_origin_regex) - - allow_all_origins = "*" in allow_origins - allow_all_headers = "*" in allow_headers - preflight_explicit_allow_origin = not allow_all_origins or allow_credentials - - simple_headers: dict[str, str] = {} - if allow_all_origins: - simple_headers["Access-Control-Allow-Origin"] = "*" - if allow_credentials: - simple_headers["Access-Control-Allow-Credentials"] = "true" - if expose_headers: - simple_headers["Access-Control-Expose-Headers"] = ", ".join(expose_headers) - - preflight_headers: dict[str, str] = {} - if preflight_explicit_allow_origin: - # The origin value will be set in preflight_response() if it is allowed. - preflight_headers["Vary"] = "Origin" - else: - preflight_headers["Access-Control-Allow-Origin"] = "*" - preflight_headers.update( - { - "Access-Control-Allow-Methods": ", ".join(allow_methods), - "Access-Control-Max-Age": str(max_age), - } - ) - allow_headers = sorted(SAFELISTED_HEADERS | set(allow_headers)) - if allow_headers and not allow_all_headers: - preflight_headers["Access-Control-Allow-Headers"] = ", ".join(allow_headers) - if allow_credentials: - preflight_headers["Access-Control-Allow-Credentials"] = "true" - - self.app = app - self.allow_origins = allow_origins - self.allow_methods = allow_methods - self.allow_headers = [h.lower() for h in allow_headers] - self.allow_all_origins = allow_all_origins - self.allow_all_headers = allow_all_headers - self.allow_credentials = allow_credentials - self.preflight_explicit_allow_origin = preflight_explicit_allow_origin - self.allow_origin_regex = compiled_allow_origin_regex - self.allow_private_network = allow_private_network - self.simple_headers = simple_headers - self.preflight_headers = preflight_headers - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": # pragma: no cover - await self.app(scope, receive, send) - return - - method = scope["method"] - headers = Headers(scope=scope) - origin = headers.get("origin") - - if origin is None: - await self.app(scope, receive, send) - return - - if method == "OPTIONS" and "access-control-request-method" in headers: - response = self.preflight_response(request_headers=headers) - await response(scope, receive, send) - return - - await self.simple_response(scope, receive, send, request_headers=headers) - - def is_allowed_origin(self, origin: str) -> bool: - if self.allow_all_origins: - return True - - if self.allow_origin_regex is not None and self.allow_origin_regex.fullmatch(origin): - return True - - return origin in self.allow_origins - - def preflight_response(self, request_headers: Headers) -> Response: - requested_origin = request_headers["origin"] - requested_method = request_headers["access-control-request-method"] - requested_headers = request_headers.get("access-control-request-headers") - requested_private_network = request_headers.get("access-control-request-private-network") - - headers = dict(self.preflight_headers) - failures: list[str] = [] - - if self.is_allowed_origin(origin=requested_origin): - if self.preflight_explicit_allow_origin: - # The "else" case is already accounted for in self.preflight_headers - # and the value would be "*". - headers["Access-Control-Allow-Origin"] = requested_origin - else: - failures.append("origin") - - if requested_method not in self.allow_methods: - failures.append("method") - - # If we allow all headers, then we have to mirror back any requested - # headers in the response. - if self.allow_all_headers and requested_headers is not None: - headers["Access-Control-Allow-Headers"] = requested_headers - elif requested_headers is not None: - for header in [h.lower() for h in requested_headers.split(",")]: - if header.strip() not in self.allow_headers: - failures.append("headers") - break - - if requested_private_network is not None: - if self.allow_private_network: - headers["Access-Control-Allow-Private-Network"] = "true" - else: - failures.append("private-network") - - # We don't strictly need to use 400 responses here, since its up to - # the browser to enforce the CORS policy, but its more informative - # if we do. - if failures: - failure_text = "Disallowed CORS " + ", ".join(failures) - return PlainTextResponse(failure_text, status_code=400, headers=headers) - - return PlainTextResponse("OK", status_code=200, headers=headers) - - async def simple_response(self, scope: Scope, receive: Receive, send: Send, request_headers: Headers) -> None: - send = functools.partial(self.send, send=send, request_headers=request_headers) - await self.app(scope, receive, send) - - async def send(self, message: Message, send: Send, request_headers: Headers) -> None: - if message["type"] != "http.response.start": - await send(message) - return - - message.setdefault("headers", []) - headers = MutableHeaders(scope=message) - headers.update(self.simple_headers) - origin = request_headers["Origin"] - - # If credentials are allowed, then we must respond with the specific origin instead of '*'. - if self.allow_all_origins and self.allow_credentials: - self.allow_explicit_origin(headers, origin) - - # If we only allow specific origins, then we have to mirror back the Origin header in the response. - elif not self.allow_all_origins and self.is_allowed_origin(origin=origin): - self.allow_explicit_origin(headers, origin) - - await send(message) - - @staticmethod - def allow_explicit_origin(headers: MutableHeaders, origin: str) -> None: - headers["Access-Control-Allow-Origin"] = origin - headers.add_vary_header("Origin") diff --git a/functions/python/data-processor/starlette/middleware/errors.py b/functions/python/data-processor/starlette/middleware/errors.py deleted file mode 100644 index 5a302a836..000000000 --- a/functions/python/data-processor/starlette/middleware/errors.py +++ /dev/null @@ -1,259 +0,0 @@ -from __future__ import annotations - -import html -import inspect -import sys -import traceback - -from starlette._utils import is_async_callable -from starlette.concurrency import run_in_threadpool -from starlette.requests import Request -from starlette.responses import HTMLResponse, PlainTextResponse, Response -from starlette.types import ASGIApp, ExceptionHandler, Message, Receive, Scope, Send - -STYLES = """ -p { - color: #211c1c; -} -.traceback-container { - border: 1px solid #038BB8; -} -.traceback-title { - background-color: #038BB8; - color: lemonchiffon; - padding: 12px; - font-size: 20px; - margin-top: 0px; -} -.frame-line { - padding-left: 10px; - font-family: monospace; -} -.frame-filename { - font-family: monospace; -} -.center-line { - background-color: #038BB8; - color: #f9f6e1; - padding: 5px 0px 5px 5px; -} -.lineno { - margin-right: 5px; -} -.frame-title { - font-weight: unset; - padding: 10px 10px 10px 10px; - background-color: #E4F4FD; - margin-right: 10px; - color: #191f21; - font-size: 17px; - border: 1px solid #c7dce8; -} -.collapse-btn { - float: right; - padding: 0px 5px 1px 5px; - border: solid 1px #96aebb; - cursor: pointer; -} -.collapsed { - display: none; -} -.source-code { - font-family: courier; - font-size: small; - padding-bottom: 10px; -} -""" - -JS = """ - -""" - -TEMPLATE = """ - - - - Starlette Debugger - - -

500 Server Error

-

{error}

-
-

Traceback

-
{exc_html}
-
- {js} - - -""" - -FRAME_TEMPLATE = """ -
-

File {frame_filename}, - line {frame_lineno}, - in {frame_name} - {collapse_button} -

-
{code_context}
-
-""" # noqa: E501 - -LINE = """ -

-{lineno}. {line}

-""" - -CENTER_LINE = """ -

-{lineno}. {line}

-""" - - -class ServerErrorMiddleware: - """ - Handles returning 500 responses when a server error occurs. - - If 'debug' is set, then traceback responses will be returned, - otherwise the designated 'handler' will be called. - - This middleware class should generally be used to wrap *everything* - else up, so that unhandled exceptions anywhere in the stack - always result in an appropriate 500 response. - """ - - def __init__( - self, - app: ASGIApp, - handler: ExceptionHandler | None = None, - debug: bool = False, - ) -> None: - self.app = app - self.handler = handler - self.debug = debug - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": - await self.app(scope, receive, send) - return - - response_started = False - - async def _send(message: Message) -> None: - nonlocal response_started, send - - if message["type"] == "http.response.start": - response_started = True - await send(message) - - try: - await self.app(scope, receive, _send) - except Exception as exc: - request = Request(scope) - if self.debug: - # In debug mode, return traceback responses. - response = self.debug_response(request, exc) - elif self.handler is None: - # Use our default 500 error handler. - response = self.error_response(request, exc) - else: - # Use an installed 500 error handler. - if is_async_callable(self.handler): - response = await self.handler(request, exc) # type: ignore[assignment, arg-type] - else: - response = await run_in_threadpool(self.handler, request, exc) # type: ignore[arg-type] - - if not response_started: - await response(scope, receive, send) - - # We always continue to raise the exception. - # This allows servers to log the error, or allows test clients - # to optionally raise the error within the test case. - raise exc - - def format_line(self, index: int, line: str, frame_lineno: int, frame_index: int) -> str: - values = { - # HTML escape - line could contain < or > - "line": html.escape(line).replace(" ", " "), - "lineno": (frame_lineno - frame_index) + index, - } - - if index != frame_index: - return LINE.format(**values) - return CENTER_LINE.format(**values) - - def generate_frame_html(self, frame: inspect.FrameInfo, is_collapsed: bool) -> str: - code_context = "".join( - self.format_line( - index, - line, - frame.lineno, - frame.index, # type: ignore[arg-type] - ) - for index, line in enumerate(frame.code_context or []) - ) - - values = { - # HTML escape - filename could contain < or >, especially if it's a virtual - # file e.g. in the REPL - "frame_filename": html.escape(frame.filename), - "frame_lineno": frame.lineno, - # HTML escape - if you try very hard it's possible to name a function with < - # or > - "frame_name": html.escape(frame.function), - "code_context": code_context, - "collapsed": "collapsed" if is_collapsed else "", - "collapse_button": "+" if is_collapsed else "‒", - } - return FRAME_TEMPLATE.format(**values) - - def generate_html(self, exc: Exception, limit: int = 7) -> str: - traceback_obj = traceback.TracebackException.from_exception(exc, capture_locals=True) - - exc_html = "" - is_collapsed = False - exc_traceback = exc.__traceback__ - if exc_traceback is not None: - frames = inspect.getinnerframes(exc_traceback, limit) - for frame in reversed(frames): - exc_html += self.generate_frame_html(frame, is_collapsed) - is_collapsed = True - - if sys.version_info >= (3, 13): # pragma: no cover - exc_type_str = traceback_obj.exc_type_str - else: # pragma: no cover - exc_type_str = traceback_obj.exc_type.__name__ - - # escape error class and text - error = f"{html.escape(exc_type_str)}: {html.escape(str(traceback_obj))}" - - return TEMPLATE.format(styles=STYLES, js=JS, error=error, exc_html=exc_html) - - def generate_plain_text(self, exc: Exception) -> str: - return "".join(traceback.format_exception(type(exc), exc, exc.__traceback__)) - - def debug_response(self, request: Request, exc: Exception) -> Response: - accept = request.headers.get("accept", "") - - if "text/html" in accept: - content = self.generate_html(exc) - return HTMLResponse(content, status_code=500) - content = self.generate_plain_text(exc) - return PlainTextResponse(content, status_code=500) - - def error_response(self, request: Request, exc: Exception) -> Response: - return PlainTextResponse("Internal Server Error", status_code=500) diff --git a/functions/python/data-processor/starlette/middleware/exceptions.py b/functions/python/data-processor/starlette/middleware/exceptions.py deleted file mode 100644 index 6c98558f4..000000000 --- a/functions/python/data-processor/starlette/middleware/exceptions.py +++ /dev/null @@ -1,73 +0,0 @@ -from __future__ import annotations - -from collections.abc import Mapping -from typing import Any - -from starlette._exception_handler import ( - ExceptionHandlers, - StatusHandlers, - wrap_app_handling_exceptions, -) -from starlette.exceptions import HTTPException, WebSocketException -from starlette.requests import Request -from starlette.responses import PlainTextResponse, Response -from starlette.types import ASGIApp, ExceptionHandler, Receive, Scope, Send -from starlette.websockets import WebSocket - - -class ExceptionMiddleware: - def __init__( - self, - app: ASGIApp, - handlers: Mapping[Any, ExceptionHandler] | None = None, - debug: bool = False, - ) -> None: - self.app = app - self.debug = debug # TODO: We ought to handle 404 cases if debug is set. - self._status_handlers: StatusHandlers = {} - self._exception_handlers: ExceptionHandlers = { - HTTPException: self.http_exception, - WebSocketException: self.websocket_exception, - } - if handlers is not None: # pragma: no branch - for key, value in handlers.items(): - self.add_exception_handler(key, value) - - def add_exception_handler( - self, - exc_class_or_status_code: int | type[Exception], - handler: ExceptionHandler, - ) -> None: - if isinstance(exc_class_or_status_code, int): - self._status_handlers[exc_class_or_status_code] = handler - else: - assert issubclass(exc_class_or_status_code, Exception) - self._exception_handlers[exc_class_or_status_code] = handler - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] not in ("http", "websocket"): - await self.app(scope, receive, send) - return - - scope["starlette.exception_handlers"] = ( - self._exception_handlers, - self._status_handlers, - ) - - conn: Request | WebSocket - if scope["type"] == "http": - conn = Request(scope, receive, send) - else: - conn = WebSocket(scope, receive, send) - - await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send) - - async def http_exception(self, request: Request, exc: Exception) -> Response: - assert isinstance(exc, HTTPException) - if exc.status_code in {204, 304}: - return Response(status_code=exc.status_code, headers=exc.headers) - return PlainTextResponse(exc.detail, status_code=exc.status_code, headers=exc.headers) - - async def websocket_exception(self, websocket: WebSocket, exc: Exception) -> None: - assert isinstance(exc, WebSocketException) - await websocket.close(code=exc.code, reason=exc.reason) # pragma: no cover diff --git a/functions/python/data-processor/starlette/middleware/gzip.py b/functions/python/data-processor/starlette/middleware/gzip.py deleted file mode 100644 index abd898b2d..000000000 --- a/functions/python/data-processor/starlette/middleware/gzip.py +++ /dev/null @@ -1,145 +0,0 @@ -import gzip -import io -from typing import NoReturn - -from starlette.datastructures import Headers, MutableHeaders -from starlette.types import ASGIApp, Message, Receive, Scope, Send - -DEFAULT_EXCLUDED_CONTENT_TYPES = ("text/event-stream",) - - -class GZipMiddleware: - def __init__(self, app: ASGIApp, minimum_size: int = 500, compresslevel: int = 9) -> None: - self.app = app - self.minimum_size = minimum_size - self.compresslevel = compresslevel - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] != "http": # pragma: no cover - await self.app(scope, receive, send) - return - - headers = Headers(scope=scope) - responder: ASGIApp - if "gzip" in headers.get("Accept-Encoding", ""): - responder = GZipResponder(self.app, self.minimum_size, compresslevel=self.compresslevel) - else: - responder = IdentityResponder(self.app, self.minimum_size) - - await responder(scope, receive, send) - - -class IdentityResponder: - content_encoding: str - - def __init__(self, app: ASGIApp, minimum_size: int) -> None: - self.app = app - self.minimum_size = minimum_size - self.send: Send = unattached_send - self.initial_message: Message = {} - self.started = False - self.content_encoding_set = False - self.content_type_is_excluded = False - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - self.send = send - await self.app(scope, receive, self.send_with_compression) - - async def send_with_compression(self, message: Message) -> None: - message_type = message["type"] - if message_type == "http.response.start": - # Don't send the initial message until we've determined how to - # modify the outgoing headers correctly. - self.initial_message = message - headers = Headers(raw=self.initial_message["headers"]) - self.content_encoding_set = "content-encoding" in headers - self.content_type_is_excluded = headers.get("content-type", "").startswith(DEFAULT_EXCLUDED_CONTENT_TYPES) - elif message_type == "http.response.body" and (self.content_encoding_set or self.content_type_is_excluded): - if not self.started: - self.started = True - await self.send(self.initial_message) - await self.send(message) - elif message_type == "http.response.body" and not self.started: - self.started = True - body = message.get("body", b"") - more_body = message.get("more_body", False) - if len(body) < self.minimum_size and not more_body: - # Don't apply compression to small outgoing responses. - await self.send(self.initial_message) - await self.send(message) - elif not more_body: - # Standard response. - body = self.apply_compression(body, more_body=False) - - headers = MutableHeaders(raw=self.initial_message["headers"]) - headers.add_vary_header("Accept-Encoding") - if body != message["body"]: - headers["Content-Encoding"] = self.content_encoding - headers["Content-Length"] = str(len(body)) - message["body"] = body - - await self.send(self.initial_message) - await self.send(message) - else: - # Initial body in streaming response. - body = self.apply_compression(body, more_body=True) - - headers = MutableHeaders(raw=self.initial_message["headers"]) - headers.add_vary_header("Accept-Encoding") - if body != message["body"]: - headers["Content-Encoding"] = self.content_encoding - del headers["Content-Length"] - message["body"] = body - - await self.send(self.initial_message) - await self.send(message) - elif message_type == "http.response.body": - # Remaining body in streaming response. - body = message.get("body", b"") - more_body = message.get("more_body", False) - - message["body"] = self.apply_compression(body, more_body=more_body) - - await self.send(message) - elif message_type == "http.response.pathsend": # pragma: no branch - # Don't apply GZip to pathsend responses - await self.send(self.initial_message) - await self.send(message) - - def apply_compression(self, body: bytes, *, more_body: bool) -> bytes: - """Apply compression on the response body. - - If more_body is False, any compression file should be closed. If it - isn't, it won't be closed automatically until all background tasks - complete. - """ - return body - - -class GZipResponder(IdentityResponder): - content_encoding = "gzip" - - def __init__(self, app: ASGIApp, minimum_size: int, compresslevel: int = 9) -> None: - super().__init__(app, minimum_size) - - self.gzip_buffer = io.BytesIO() - self.gzip_file = gzip.GzipFile(mode="wb", fileobj=self.gzip_buffer, compresslevel=compresslevel) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - with self.gzip_buffer, self.gzip_file: - await super().__call__(scope, receive, send) - - def apply_compression(self, body: bytes, *, more_body: bool) -> bytes: - self.gzip_file.write(body) - if not more_body: - self.gzip_file.close() - - body = self.gzip_buffer.getvalue() - self.gzip_buffer.seek(0) - self.gzip_buffer.truncate() - - return body - - -async def unattached_send(message: Message) -> NoReturn: - raise RuntimeError("send awaitable not set") # pragma: no cover diff --git a/functions/python/data-processor/starlette/middleware/httpsredirect.py b/functions/python/data-processor/starlette/middleware/httpsredirect.py deleted file mode 100644 index a8359067f..000000000 --- a/functions/python/data-processor/starlette/middleware/httpsredirect.py +++ /dev/null @@ -1,19 +0,0 @@ -from starlette.datastructures import URL -from starlette.responses import RedirectResponse -from starlette.types import ASGIApp, Receive, Scope, Send - - -class HTTPSRedirectMiddleware: - def __init__(self, app: ASGIApp) -> None: - self.app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] in ("http", "websocket") and scope["scheme"] in ("http", "ws"): - url = URL(scope=scope) - redirect_scheme = {"http": "https", "ws": "wss"}[url.scheme] - netloc = url.hostname if url.port in (80, 443) else url.netloc - url = url.replace(scheme=redirect_scheme, netloc=netloc) - response = RedirectResponse(url, status_code=307) - await response(scope, receive, send) - else: - await self.app(scope, receive, send) diff --git a/functions/python/data-processor/starlette/middleware/sessions.py b/functions/python/data-processor/starlette/middleware/sessions.py deleted file mode 100644 index aa0383f98..000000000 --- a/functions/python/data-processor/starlette/middleware/sessions.py +++ /dev/null @@ -1,125 +0,0 @@ -from __future__ import annotations - -import json -import typing -from base64 import b64decode, b64encode -from typing import Literal - -import itsdangerous -from itsdangerous.exc import BadSignature - -from starlette.datastructures import MutableHeaders, Secret -from starlette.requests import HTTPConnection -from starlette.types import ASGIApp, Message, Receive, Scope, Send - - -class SessionMiddleware: - def __init__( - self, - app: ASGIApp, - secret_key: str | Secret, - session_cookie: str = "session", - max_age: int | None = 14 * 24 * 60 * 60, # 14 days, in seconds - path: str = "/", - same_site: Literal["lax", "strict", "none"] = "lax", - https_only: bool = False, - domain: str | None = None, - ) -> None: - self.app = app - self.signer = itsdangerous.TimestampSigner(str(secret_key)) - self.session_cookie = session_cookie - self.max_age = max_age - self.path = path - self.security_flags = "httponly; samesite=" + same_site - if https_only: # Secure flag can be used with HTTPS only - self.security_flags += "; secure" - if domain is not None: - self.security_flags += f"; domain={domain}" - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] not in ("http", "websocket"): # pragma: no cover - await self.app(scope, receive, send) - return - - connection = HTTPConnection(scope) - initial_session_was_empty = True - - if self.session_cookie in connection.cookies: - data = connection.cookies[self.session_cookie].encode("utf-8") - try: - data = self.signer.unsign(data, max_age=self.max_age) - scope["session"] = Session(json.loads(b64decode(data))) - initial_session_was_empty = False - except BadSignature: - scope["session"] = Session() - else: - scope["session"] = Session() - - async def send_wrapper(message: Message) -> None: - if message["type"] == "http.response.start": - session: Session = scope["session"] - headers = MutableHeaders(scope=message) - if session.accessed: - headers.add_vary_header("Cookie") - if session.modified and session: - # We have session data to persist. - data = b64encode(json.dumps(session).encode("utf-8")) - data = self.signer.sign(data) - header_value = "{session_cookie}={data}; path={path}; {max_age}{security_flags}".format( - session_cookie=self.session_cookie, - data=data.decode("utf-8"), - path=self.path, - max_age=f"Max-Age={self.max_age}; " if self.max_age else "", - security_flags=self.security_flags, - ) - headers.append("Set-Cookie", header_value) - elif session.modified and not initial_session_was_empty: - # The session has been cleared. - header_value = "{session_cookie}={data}; path={path}; {expires}{security_flags}".format( - session_cookie=self.session_cookie, - data="null", - path=self.path, - expires="expires=Thu, 01 Jan 1970 00:00:00 GMT; ", - security_flags=self.security_flags, - ) - headers.append("Set-Cookie", header_value) - await send(message) - - await self.app(scope, receive, send_wrapper) - - -class Session(dict[str, typing.Any]): - accessed: bool = False - modified: bool = False - - def mark_accessed(self) -> None: - self.accessed = True - - def mark_modified(self) -> None: - self.accessed = True - self.modified = True - - def __setitem__(self, key: str, value: typing.Any) -> None: - self.mark_modified() - super().__setitem__(key, value) - - def __delitem__(self, key: str) -> None: - self.mark_modified() - super().__delitem__(key) - - def clear(self) -> None: - self.mark_modified() - super().clear() - - def pop(self, key: str, *args: typing.Any) -> typing.Any: - self.modified = self.modified or key in self - return super().pop(key, *args) - - def setdefault(self, key: str, default: typing.Any = None) -> typing.Any: - if key not in self: - self.mark_modified() - return super().setdefault(key, default) - - def update(self, *args: typing.Any, **kwargs: typing.Any) -> None: - self.mark_modified() - super().update(*args, **kwargs) diff --git a/functions/python/data-processor/starlette/middleware/trustedhost.py b/functions/python/data-processor/starlette/middleware/trustedhost.py deleted file mode 100644 index 98451e29f..000000000 --- a/functions/python/data-processor/starlette/middleware/trustedhost.py +++ /dev/null @@ -1,60 +0,0 @@ -from __future__ import annotations - -from collections.abc import Sequence - -from starlette.datastructures import URL, Headers -from starlette.responses import PlainTextResponse, RedirectResponse, Response -from starlette.types import ASGIApp, Receive, Scope, Send - -ENFORCE_DOMAIN_WILDCARD = "Domain wildcard patterns must be like '*.example.com'." - - -class TrustedHostMiddleware: - def __init__( - self, - app: ASGIApp, - allowed_hosts: Sequence[str] | None = None, - www_redirect: bool = True, - ) -> None: - if allowed_hosts is None: - allowed_hosts = ["*"] - - for pattern in allowed_hosts: - assert "*" not in pattern[1:], ENFORCE_DOMAIN_WILDCARD - if pattern.startswith("*") and pattern != "*": - assert pattern.startswith("*."), ENFORCE_DOMAIN_WILDCARD - self.app = app - self.allowed_hosts = list(allowed_hosts) - self.allow_any = "*" in allowed_hosts - self.www_redirect = www_redirect - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if self.allow_any or scope["type"] not in ( - "http", - "websocket", - ): # pragma: no cover - await self.app(scope, receive, send) - return - - headers = Headers(scope=scope) - host = headers.get("host", "").split(":")[0] - is_valid_host = False - found_www_redirect = False - for pattern in self.allowed_hosts: - if host == pattern or (pattern.startswith("*") and host.endswith(pattern[1:])): - is_valid_host = True - break - elif "www." + host == pattern: - found_www_redirect = True - - if is_valid_host: - await self.app(scope, receive, send) - else: - response: Response - if found_www_redirect and self.www_redirect: - url = URL(scope=scope) - redirect_url = url.replace(netloc="www." + url.netloc) - response = RedirectResponse(url=str(redirect_url)) - else: - response = PlainTextResponse("Invalid host header", status_code=400) - await response(scope, receive, send) diff --git a/functions/python/data-processor/starlette/middleware/wsgi.py b/functions/python/data-processor/starlette/middleware/wsgi.py deleted file mode 100644 index f456fc413..000000000 --- a/functions/python/data-processor/starlette/middleware/wsgi.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -import io -import math -import sys -import warnings -from collections.abc import Callable, MutableMapping -from typing import Any - -import anyio -from anyio.abc import ObjectReceiveStream, ObjectSendStream - -from starlette._utils import create_collapsing_task_group -from starlette.exceptions import StarletteDeprecationWarning -from starlette.types import Receive, Scope, Send - -warnings.warn( - "starlette.middleware.wsgi is deprecated and will be removed in a future release. " - "Please refer to https://github.com/abersheeran/a2wsgi as a replacement.", - StarletteDeprecationWarning, - stacklevel=2, -) - - -def build_environ(scope: Scope, body: bytes) -> dict[str, Any]: - """ - Builds a scope and request body into a WSGI environ object. - """ - - script_name = scope.get("root_path", "").encode("utf8").decode("latin1") - path_info = scope["path"].encode("utf8").decode("latin1") - if path_info.startswith(script_name): - path_info = path_info[len(script_name) :] - - environ = { - "REQUEST_METHOD": scope["method"], - "SCRIPT_NAME": script_name, - "PATH_INFO": path_info, - "QUERY_STRING": scope["query_string"].decode("ascii"), - "SERVER_PROTOCOL": f"HTTP/{scope['http_version']}", - "wsgi.version": (1, 0), - "wsgi.url_scheme": scope.get("scheme", "http"), - "wsgi.input": io.BytesIO(body), - "wsgi.errors": sys.stdout, - "wsgi.multithread": True, - "wsgi.multiprocess": True, - "wsgi.run_once": False, - } - - # Get server name and port - required in WSGI, not in ASGI - server = scope.get("server") or ("localhost", 80) - environ["SERVER_NAME"] = server[0] - environ["SERVER_PORT"] = server[1] - - # Get client IP address - if scope.get("client"): - environ["REMOTE_ADDR"] = scope["client"][0] - - # Go through headers and make them into environ entries - for name, value in scope.get("headers", []): - name = name.decode("latin1") - if name == "content-length": - corrected_name = "CONTENT_LENGTH" - elif name == "content-type": - corrected_name = "CONTENT_TYPE" - else: - corrected_name = f"HTTP_{name}".upper().replace("-", "_") - # HTTPbis say only ASCII chars are allowed in headers, but we latin1 just in - # case - value = value.decode("latin1") - if corrected_name in environ: - value = environ[corrected_name] + "," + value - environ[corrected_name] = value - return environ - - -class WSGIMiddleware: - def __init__(self, app: Callable[..., Any]) -> None: - self.app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - assert scope["type"] == "http" - responder = WSGIResponder(self.app, scope) - await responder(receive, send) - - -class WSGIResponder: - stream_send: ObjectSendStream[MutableMapping[str, Any]] - stream_receive: ObjectReceiveStream[MutableMapping[str, Any]] - - def __init__(self, app: Callable[..., Any], scope: Scope) -> None: - self.app = app - self.scope = scope - self.status = None - self.response_headers = None - self.stream_send, self.stream_receive = anyio.create_memory_object_stream(math.inf) - self.response_started = False - self.exc_info: Any = None - - async def __call__(self, receive: Receive, send: Send) -> None: - body = b"" - more_body = True - while more_body: - message = await receive() - body += message.get("body", b"") - more_body = message.get("more_body", False) - environ = build_environ(self.scope, body) - - async with create_collapsing_task_group() as task_group: - task_group.start_soon(self.sender, send) - async with self.stream_send: - await anyio.to_thread.run_sync(self.wsgi, environ, self.start_response) - if self.exc_info is not None: - raise self.exc_info[0].with_traceback(self.exc_info[1], self.exc_info[2]) - - async def sender(self, send: Send) -> None: - async with self.stream_receive: - async for message in self.stream_receive: - await send(message) - - def start_response( - self, - status: str, - response_headers: list[tuple[str, str]], - exc_info: Any = None, - ) -> None: - self.exc_info = exc_info - if not self.response_started: # pragma: no branch - self.response_started = True - status_code_string, _ = status.split(" ", 1) - status_code = int(status_code_string) - headers = [ - (name.strip().encode("ascii").lower(), value.strip().encode("ascii")) - for name, value in response_headers - ] - anyio.from_thread.run( - self.stream_send.send, - { - "type": "http.response.start", - "status": status_code, - "headers": headers, - }, - ) - - def wsgi( - self, - environ: dict[str, Any], - start_response: Callable[..., Any], - ) -> None: - for chunk in self.app(environ, start_response): - anyio.from_thread.run( - self.stream_send.send, - {"type": "http.response.body", "body": chunk, "more_body": True}, - ) - - anyio.from_thread.run(self.stream_send.send, {"type": "http.response.body", "body": b""}) diff --git a/functions/python/data-processor/starlette/py.typed b/functions/python/data-processor/starlette/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/starlette/requests.py b/functions/python/data-processor/starlette/requests.py deleted file mode 100644 index d4f3f4be6..000000000 --- a/functions/python/data-processor/starlette/requests.py +++ /dev/null @@ -1,348 +0,0 @@ -from __future__ import annotations - -import json -import sys -from collections.abc import AsyncGenerator, Iterator, Mapping -from http import cookies as http_cookies -from typing import TYPE_CHECKING, Any, Generic, NoReturn, cast - -import anyio - -from starlette._utils import AwaitableOrContextManager, AwaitableOrContextManagerWrapper -from starlette.datastructures import URL, Address, FormData, Headers, QueryParams, State -from starlette.exceptions import HTTPException -from starlette.formparsers import FormParser, MultiPartException, MultiPartParser -from starlette.types import Message, Receive, Scope, Send - -if TYPE_CHECKING: - from python_multipart.multipart import parse_options_header - - from starlette.applications import Starlette - from starlette.middleware.sessions import Session - from starlette.routing import Router -else: - try: - try: - from python_multipart.multipart import parse_options_header - except ModuleNotFoundError: # pragma: no cover - from multipart.multipart import parse_options_header - except ModuleNotFoundError: # pragma: no cover - parse_options_header = None - -if sys.version_info >= (3, 13): # pragma: no cover - from typing import TypeVar -else: # pragma: no cover - from typing_extensions import TypeVar - -SERVER_PUSH_HEADERS_TO_COPY = { - "accept", - "accept-encoding", - "accept-language", - "cache-control", - "user-agent", -} - - -def cookie_parser(cookie_string: str) -> dict[str, str]: - """ - This function parses a ``Cookie`` HTTP header into a dict of key/value pairs. - - It attempts to mimic browser cookie parsing behavior: browsers and web servers - frequently disregard the spec (RFC 6265) when setting and reading cookies, - so we attempt to suit the common scenarios here. - - This function has been adapted from Django 3.1.0. - Note: we are explicitly _NOT_ using `SimpleCookie.load` because it is based - on an outdated spec and will fail on lots of input we want to support - """ - cookie_dict: dict[str, str] = {} - for chunk in cookie_string.split(";"): - if "=" in chunk: - key, val = chunk.split("=", 1) - else: - # Assume an empty name per - # https://bugzilla.mozilla.org/show_bug.cgi?id=169091 - key, val = "", chunk - key, val = key.strip(), val.strip() - if key or val: - # unquote using Python's algorithm. - cookie_dict[key] = http_cookies._unquote(val) - return cookie_dict - - -class ClientDisconnect(Exception): - pass - - -StateT = TypeVar("StateT", bound=Mapping[str, Any] | State, default=State) - - -class HTTPConnection(Mapping[str, Any], Generic[StateT]): - """ - A base class for incoming HTTP connections, that is used to provide - any functionality that is common to both `Request` and `WebSocket`. - """ - - def __init__(self, scope: Scope, receive: Receive | None = None) -> None: - assert scope["type"] in ("http", "websocket") - self.scope = scope - - def __getitem__(self, key: str) -> Any: - return self.scope[key] - - def __iter__(self) -> Iterator[str]: - return iter(self.scope) - - def __len__(self) -> int: - return len(self.scope) - - # Don't use the `abc.Mapping.__eq__` implementation. - # Connection instances should never be considered equal - # unless `self is other`. - __eq__ = object.__eq__ - __hash__ = object.__hash__ - - @property - def app(self) -> Any: - return self.scope["app"] - - @property - def url(self) -> URL: - if not hasattr(self, "_url"): # pragma: no branch - self._url = URL(scope=self.scope) - return self._url - - @property - def base_url(self) -> URL: - if not hasattr(self, "_base_url"): - base_url_scope = dict(self.scope) - # This is used by request.url_for, it might be used inside a Mount which - # would have its own child scope with its own root_path, but the base URL - # for url_for should still be the top level app root path. - app_root_path = base_url_scope.get("app_root_path", base_url_scope.get("root_path", "")) - path = app_root_path - if not path.endswith("/"): - path += "/" - base_url_scope["path"] = path - base_url_scope["query_string"] = b"" - base_url_scope["root_path"] = app_root_path - self._base_url = URL(scope=base_url_scope) - return self._base_url - - @property - def headers(self) -> Headers: - if not hasattr(self, "_headers"): - self._headers = Headers(scope=self.scope) - return self._headers - - @property - def query_params(self) -> QueryParams: - if not hasattr(self, "_query_params"): # pragma: no branch - self._query_params = QueryParams(self.scope["query_string"]) - return self._query_params - - @property - def path_params(self) -> dict[str, Any]: - path_params: dict[str, Any] = self.scope.get("path_params", {}) - return path_params - - @property - def cookies(self) -> dict[str, str]: - if not hasattr(self, "_cookies"): - cookies: dict[str, str] = {} - cookie_headers = self.headers.getlist("cookie") - - for header in cookie_headers: - cookies.update(cookie_parser(header)) - - self._cookies = cookies - return self._cookies - - @property - def client(self) -> Address | None: - # client is a 2 item tuple of (host, port), None if missing - host_port = self.scope.get("client") - if host_port is not None: - return Address(*host_port) - return None - - @property - def session(self) -> dict[str, Any]: - assert "session" in self.scope, "SessionMiddleware must be installed to access request.session" - session: Session = self.scope["session"] - # We keep the hasattr in case people actually use their own `SessionMiddleware` implementation. - if hasattr(session, "mark_accessed"): # pragma: no branch - session.mark_accessed() - return session - - @property - def auth(self) -> Any: - assert "auth" in self.scope, "AuthenticationMiddleware must be installed to access request.auth" - return self.scope["auth"] - - @property - def user(self) -> Any: - assert "user" in self.scope, "AuthenticationMiddleware must be installed to access request.user" - return self.scope["user"] - - @property - def state(self) -> StateT: - if not hasattr(self, "_state"): - # Ensure 'state' has an empty dict if it's not already populated. - self.scope.setdefault("state", {}) - # Create a state instance with a reference to the dict in which it should - # store info - self._state = State(self.scope["state"]) - return cast(StateT, self._state) - - def url_for(self, name: str, /, **path_params: Any) -> URL: - url_path_provider: Router | Starlette | None = self.scope.get("router") or self.scope.get("app") - if url_path_provider is None: - raise RuntimeError("The `url_for` method can only be used inside a Starlette application or with a router.") - url_path = url_path_provider.url_path_for(name, **path_params) - return url_path.make_absolute_url(base_url=self.base_url) - - -async def empty_receive() -> NoReturn: - raise RuntimeError("Receive channel has not been made available") - - -async def empty_send(message: Message) -> NoReturn: - raise RuntimeError("Send channel has not been made available") - - -class Request(HTTPConnection[StateT]): - _form: FormData | None - - def __init__(self, scope: Scope, receive: Receive = empty_receive, send: Send = empty_send): - super().__init__(scope) - assert scope["type"] == "http" - self._receive = receive - self._send = send - self._stream_consumed = False - self._is_disconnected = False - self._form = None - - @property - def method(self) -> str: - return cast(str, self.scope["method"]) - - @property - def receive(self) -> Receive: - return self._receive - - async def stream(self) -> AsyncGenerator[bytes, None]: - if hasattr(self, "_body"): - yield self._body - yield b"" - return - if self._stream_consumed: - raise RuntimeError("Stream consumed") - while not self._stream_consumed: - message = await self._receive() - if message["type"] == "http.request": - body = message.get("body", b"") - if not message.get("more_body", False): - self._stream_consumed = True - if body: - yield body - elif message["type"] == "http.disconnect": # pragma: no branch - self._is_disconnected = True - raise ClientDisconnect() - yield b"" - - async def body(self) -> bytes: - if not hasattr(self, "_body"): - chunks: list[bytes] = [] - async for chunk in self.stream(): - chunks.append(chunk) - self._body = b"".join(chunks) - return self._body - - async def json(self) -> Any: - if not hasattr(self, "_json"): # pragma: no branch - body = await self.body() - self._json = json.loads(body) - return self._json - - async def _get_form( - self, - *, - max_files: int | float = 1000, - max_fields: int | float = 1000, - max_part_size: int = 1024 * 1024, - ) -> FormData: - if self._form is None: # pragma: no branch - assert parse_options_header is not None, ( - "The `python-multipart` library must be installed to use form parsing." - ) - content_type_header = self.headers.get("Content-Type") - content_type: bytes - content_type, _ = parse_options_header(content_type_header) - if content_type == b"multipart/form-data": - try: - multipart_parser = MultiPartParser( - self.headers, - self.stream(), - max_files=max_files, - max_fields=max_fields, - max_part_size=max_part_size, - ) - self._form = await multipart_parser.parse() - except MultiPartException as exc: - if "app" in self.scope: - raise HTTPException(status_code=400, detail=exc.message) - raise exc - elif content_type == b"application/x-www-form-urlencoded": - try: - form_parser = FormParser( - self.headers, - self.stream(), - max_fields=max_fields, - max_part_size=max_part_size, - ) - self._form = await form_parser.parse() - except MultiPartException as exc: - if "app" in self.scope: - raise HTTPException(status_code=400, detail=exc.message) - raise exc - else: - self._form = FormData() - return self._form - - def form( - self, - *, - max_files: int | float = 1000, - max_fields: int | float = 1000, - max_part_size: int = 1024 * 1024, - ) -> AwaitableOrContextManager[FormData]: - return AwaitableOrContextManagerWrapper( - self._get_form(max_files=max_files, max_fields=max_fields, max_part_size=max_part_size) - ) - - async def close(self) -> None: - if self._form is not None: # pragma: no branch - await self._form.close() - - async def is_disconnected(self) -> bool: - if not self._is_disconnected: - message: Message = {} - - # If message isn't immediately available, move on - with anyio.CancelScope() as cs: - cs.cancel() - message = await self._receive() - - if message.get("type") == "http.disconnect": - self._is_disconnected = True - - return self._is_disconnected - - async def send_push_promise(self, path: str) -> None: - if "http.response.push" in self.scope.get("extensions", {}): - raw_headers: list[tuple[bytes, bytes]] = [] - for name in SERVER_PUSH_HEADERS_TO_COPY: - for value in self.headers.getlist(name): - raw_headers.append((name.encode("latin-1"), value.encode("latin-1"))) - await self._send({"type": "http.response.push", "path": path, "headers": raw_headers}) diff --git a/functions/python/data-processor/starlette/responses.py b/functions/python/data-processor/starlette/responses.py deleted file mode 100644 index 63b49ecf4..000000000 --- a/functions/python/data-processor/starlette/responses.py +++ /dev/null @@ -1,567 +0,0 @@ -from __future__ import annotations - -import hashlib -import http.cookies -import json -import os -import stat -import sys -from collections.abc import AsyncIterable, Awaitable, Callable, Iterable, Mapping, Sequence -from datetime import datetime -from email.utils import format_datetime, formatdate -from functools import partial -from mimetypes import guess_type -from secrets import token_hex -from typing import Any, Literal -from urllib.parse import quote - -import anyio -import anyio.to_thread - -from starlette._utils import create_collapsing_task_group -from starlette.background import BackgroundTask -from starlette.concurrency import iterate_in_threadpool -from starlette.datastructures import URL, Headers, MutableHeaders -from starlette.requests import ClientDisconnect -from starlette.types import Message, Receive, Scope, Send - - -class Response: - media_type = None - charset = "utf-8" - - def __init__( - self, - content: Any = None, - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - ) -> None: - self.status_code = status_code - if media_type is not None: - self.media_type = media_type - self.background = background - self.body = self.render(content) - self.init_headers(headers) - - def render(self, content: Any) -> bytes | memoryview: - if content is None: - return b"" - if isinstance(content, bytes | memoryview): - return content - return content.encode(self.charset) # type: ignore - - def init_headers(self, headers: Mapping[str, str] | None = None) -> None: - if headers is None: - raw_headers: list[tuple[bytes, bytes]] = [] - populate_content_length = True - populate_content_type = True - else: - raw_headers = [(k.lower().encode("latin-1"), v.encode("latin-1")) for k, v in headers.items()] - keys = [h[0] for h in raw_headers] - populate_content_length = b"content-length" not in keys - populate_content_type = b"content-type" not in keys - - body = getattr(self, "body", None) - if ( - body is not None - and populate_content_length - and not (self.status_code < 200 or self.status_code in (204, 304)) - ): - content_length = str(len(body)) - raw_headers.append((b"content-length", content_length.encode("latin-1"))) - - content_type = self.media_type - if content_type is not None and populate_content_type: - if content_type.startswith("text/") and "charset=" not in content_type.lower(): - content_type += "; charset=" + self.charset - raw_headers.append((b"content-type", content_type.encode("latin-1"))) - - self.raw_headers = raw_headers - - @property - def headers(self) -> MutableHeaders: - if not hasattr(self, "_headers"): - self._headers = MutableHeaders(raw=self.raw_headers) - return self._headers - - def set_cookie( - self, - key: str, - value: str = "", - max_age: int | None = None, - expires: datetime | str | int | None = None, - path: str | None = "/", - domain: str | None = None, - secure: bool = False, - httponly: bool = False, - samesite: Literal["lax", "strict", "none"] | None = "lax", - partitioned: bool = False, - ) -> None: - cookie: http.cookies.BaseCookie[str] = http.cookies.SimpleCookie() - cookie[key] = value - if max_age is not None: - cookie[key]["max-age"] = max_age - if expires is not None: - if isinstance(expires, datetime): - cookie[key]["expires"] = format_datetime(expires, usegmt=True) - else: - cookie[key]["expires"] = expires - if path is not None: - cookie[key]["path"] = path - if domain is not None: - cookie[key]["domain"] = domain - if secure: - cookie[key]["secure"] = True - if httponly: - cookie[key]["httponly"] = True - if samesite is not None: - assert samesite.lower() in [ - "strict", - "lax", - "none", - ], "samesite must be either 'strict', 'lax' or 'none'" - cookie[key]["samesite"] = samesite - if partitioned: - if sys.version_info < (3, 14): - raise ValueError("Partitioned cookies are only supported in Python 3.14 and above.") # pragma: no cover - cookie[key]["partitioned"] = True # pragma: no cover - - cookie_val = cookie.output(header="").strip() - self.raw_headers.append((b"set-cookie", cookie_val.encode("latin-1"))) - - def delete_cookie( - self, - key: str, - path: str = "/", - domain: str | None = None, - secure: bool = False, - httponly: bool = False, - samesite: Literal["lax", "strict", "none"] | None = "lax", - ) -> None: - self.set_cookie( - key, - max_age=0, - expires=0, - path=path, - domain=domain, - secure=secure, - httponly=httponly, - samesite=samesite, - ) - - def _wrap_websocket_denial_send(self, send: Send) -> Send: - async def wrapped(message: Message) -> None: - message_type = message["type"] - if message_type in {"http.response.start", "http.response.body"}: # pragma: no branch - message = {**message, "type": "websocket." + message_type} - await send(message) - - return wrapped - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] == "websocket": - send = self._wrap_websocket_denial_send(send) - await send({"type": "http.response.start", "status": self.status_code, "headers": self.raw_headers}) - await send({"type": "http.response.body", "body": self.body}) - - if self.background is not None: - await self.background() - - -class HTMLResponse(Response): - media_type = "text/html" - - -class PlainTextResponse(Response): - media_type = "text/plain" - - -class JSONResponse(Response): - media_type = "application/json" - - def __init__( - self, - content: Any, - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - ) -> None: - super().__init__(content, status_code, headers, media_type, background) - - def render(self, content: Any) -> bytes: - return json.dumps( - content, - ensure_ascii=False, - allow_nan=False, - indent=None, - separators=(",", ":"), - ).encode("utf-8") - - -class RedirectResponse(Response): - def __init__( - self, - url: str | URL, - status_code: int = 307, - headers: Mapping[str, str] | None = None, - background: BackgroundTask | None = None, - ) -> None: - super().__init__(content=b"", status_code=status_code, headers=headers, background=background) - self.headers["location"] = quote(str(url), safe=":/%#?=@[]!$&'()*+,;") - - -Content = str | bytes | memoryview -SyncContentStream = Iterable[Content] -AsyncContentStream = AsyncIterable[Content] -ContentStream = AsyncContentStream | SyncContentStream - - -class StreamingResponse(Response): - body_iterator: AsyncContentStream - - def __init__( - self, - content: ContentStream, - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - ) -> None: - if isinstance(content, AsyncIterable): - self.body_iterator = content - else: - self.body_iterator = iterate_in_threadpool(content) - self.status_code = status_code - self.media_type = self.media_type if media_type is None else media_type - self.background = background - self.init_headers(headers) - - async def listen_for_disconnect(self, receive: Receive) -> None: - while True: - message = await receive() - if message["type"] == "http.disconnect": - break - - async def stream_response(self, send: Send) -> None: - await send({"type": "http.response.start", "status": self.status_code, "headers": self.raw_headers}) - async for chunk in self.body_iterator: - if not isinstance(chunk, bytes | memoryview): - chunk = chunk.encode(self.charset) - await send({"type": "http.response.body", "body": chunk, "more_body": True}) - - await send({"type": "http.response.body", "body": b"", "more_body": False}) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] == "websocket": - send = self._wrap_websocket_denial_send(send) - await self.stream_response(send) - if self.background is not None: - await self.background() - return - - spec_version = tuple(map(int, scope.get("asgi", {}).get("spec_version", "2.0").split("."))) - - if spec_version >= (2, 4): - try: - await self.stream_response(send) - except OSError: - raise ClientDisconnect() - else: - async with create_collapsing_task_group() as task_group: - - async def wrap(func: Callable[[], Awaitable[None]]) -> None: - await func() - task_group.cancel_scope.cancel() - - task_group.start_soon(wrap, partial(self.stream_response, send)) - await wrap(partial(self.listen_for_disconnect, receive)) - - if self.background is not None: - await self.background() - - -class MalformedRangeHeader(Exception): - def __init__(self, content: str = "Malformed range header.") -> None: - self.content = content - - -class RangeNotSatisfiable(Exception): - def __init__(self, max_size: int) -> None: - self.max_size = max_size - - -class FileResponse(Response): - chunk_size = 64 * 1024 - - def __init__( - self, - path: str | os.PathLike[str], - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - filename: str | None = None, - stat_result: os.stat_result | None = None, - content_disposition_type: str = "attachment", - ) -> None: - self.path = path - self.status_code = status_code - self.filename = filename - if media_type is None: - media_type = guess_type(filename or path)[0] or "application/octet-stream" - self.media_type = media_type - self.background = background - self.init_headers(headers) - self.headers.setdefault("accept-ranges", "bytes") - if self.filename is not None: - content_disposition_filename = quote(self.filename) - if content_disposition_filename != self.filename: - content_disposition = f"{content_disposition_type}; filename*=utf-8''{content_disposition_filename}" - else: - content_disposition = f'{content_disposition_type}; filename="{self.filename}"' - self.headers.setdefault("content-disposition", content_disposition) - self.stat_result = stat_result - if stat_result is not None: - self.set_stat_headers(stat_result) - - def set_stat_headers(self, stat_result: os.stat_result) -> None: - content_length = str(stat_result.st_size) - last_modified = formatdate(stat_result.st_mtime, usegmt=True) - etag_base = str(stat_result.st_mtime) + "-" + str(stat_result.st_size) - etag = f'"{hashlib.md5(etag_base.encode(), usedforsecurity=False).hexdigest()}"' - - self.headers.setdefault("content-length", content_length) - self.headers.setdefault("last-modified", last_modified) - self.headers.setdefault("etag", etag) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - scope_type = scope["type"] - send_header_only = scope_type == "http" and scope["method"].upper() == "HEAD" - send_pathsend = scope_type == "http" and "http.response.pathsend" in scope.get("extensions", {}) - if scope_type == "websocket": - send = self._wrap_websocket_denial_send(send) - - if self.stat_result is None: - try: - stat_result = await anyio.to_thread.run_sync(os.stat, self.path) - self.set_stat_headers(stat_result) - except FileNotFoundError: - raise RuntimeError(f"File at path {self.path} does not exist.") - else: - mode = stat_result.st_mode - if not stat.S_ISREG(mode): - raise RuntimeError(f"File at path {self.path} is not a file.") - else: - stat_result = self.stat_result - - headers = Headers(scope=scope) - http_range = headers.get("range") - http_if_range = headers.get("if-range") - - if http_range is None or (http_if_range is not None and not self._should_use_range(http_if_range)): - await self._handle_simple(send, send_header_only, send_pathsend) - else: - try: - ranges = self._parse_range_header(http_range, stat_result.st_size) - except MalformedRangeHeader as exc: - return await PlainTextResponse(exc.content, status_code=400)(scope, receive, send) - except RangeNotSatisfiable as exc: - response = PlainTextResponse(status_code=416, headers={"Content-Range": f"bytes */{exc.max_size}"}) - return await response(scope, receive, send) - - if len(ranges) == 1: - start, end = ranges[0] - await self._handle_single_range(send, start, end, stat_result.st_size, send_header_only) - else: - await self._handle_multiple_ranges(send, ranges, stat_result.st_size, send_header_only) - - if self.background is not None: - await self.background() - - async def _handle_simple(self, send: Send, send_header_only: bool, send_pathsend: bool) -> None: - await send({"type": "http.response.start", "status": self.status_code, "headers": self.raw_headers}) - if send_header_only: - await send({"type": "http.response.body", "body": b"", "more_body": False}) - elif send_pathsend: - await send({"type": "http.response.pathsend", "path": str(self.path)}) - else: - async with await anyio.open_file(self.path, mode="rb") as file: - more_body = True - while more_body: - chunk = await file.read(self.chunk_size) - more_body = len(chunk) == self.chunk_size - await send({"type": "http.response.body", "body": chunk, "more_body": more_body}) - - async def _handle_single_range( - self, send: Send, start: int, end: int, file_size: int, send_header_only: bool - ) -> None: - headers = MutableHeaders(raw=list(self.raw_headers)) - headers["content-range"] = f"bytes {start}-{end - 1}/{file_size}" - headers["content-length"] = str(end - start) - await send({"type": "http.response.start", "status": 206, "headers": headers.raw}) - if send_header_only: - await send({"type": "http.response.body", "body": b"", "more_body": False}) - else: - async with await anyio.open_file(self.path, mode="rb") as file: - await file.seek(start) - more_body = True - while more_body: - chunk = await file.read(min(self.chunk_size, end - start)) - start += len(chunk) - more_body = len(chunk) == self.chunk_size and start < end - await send({"type": "http.response.body", "body": chunk, "more_body": more_body}) - - async def _handle_multiple_ranges( - self, - send: Send, - ranges: list[tuple[int, int]], - file_size: int, - send_header_only: bool, - ) -> None: - # In firefox and chrome, they use boundary with 95-96 bits entropy (that's roughly 13 bytes). - boundary = token_hex(13) - content_length, header_generator = self.generate_multipart( - ranges, boundary, file_size, self.headers["content-type"] - ) - headers = MutableHeaders(raw=list(self.raw_headers)) - headers["content-type"] = f"multipart/byteranges; boundary={boundary}" - headers["content-length"] = str(content_length) - await send({"type": "http.response.start", "status": 206, "headers": headers.raw}) - if send_header_only: - await send({"type": "http.response.body", "body": b"", "more_body": False}) - else: - async with await anyio.open_file(self.path, mode="rb") as file: - for start, end in ranges: - await send({"type": "http.response.body", "body": header_generator(start, end), "more_body": True}) - await file.seek(start) - while start < end: - chunk = await file.read(min(self.chunk_size, end - start)) - start += len(chunk) - await send({"type": "http.response.body", "body": chunk, "more_body": True}) - await send({"type": "http.response.body", "body": b"\r\n", "more_body": True}) - await send( - { - "type": "http.response.body", - "body": f"--{boundary}--".encode("latin-1"), - "more_body": False, - } - ) - - def _should_use_range(self, http_if_range: str) -> bool: - return http_if_range == self.headers["last-modified"] or http_if_range == self.headers["etag"] - - @classmethod - def _parse_range_header(cls, http_range: str, file_size: int) -> list[tuple[int, int]]: - ranges: list[tuple[int, int]] = [] - try: - units, range_ = http_range.split("=", 1) - except ValueError: - raise MalformedRangeHeader() - - units = units.strip().lower() - - if units != "bytes": - raise MalformedRangeHeader("Only support bytes range") - - ranges = cls._parse_ranges(range_, file_size) - - if len(ranges) == 0: - raise MalformedRangeHeader("Range header: range must be requested") - - if any(not (0 <= start < file_size) for start, _ in ranges): - raise RangeNotSatisfiable(file_size) - - if any(start > end for start, end in ranges): - raise MalformedRangeHeader("Range header: start must be less than end") - - if len(ranges) == 1: - return ranges - - # Merge overlapping ranges - ranges.sort() - result: list[tuple[int, int]] = [ranges[0]] - for start, end in ranges[1:]: - last_start, last_end = result[-1] - if start <= last_end: - result[-1] = (last_start, max(last_end, end)) - else: - result.append((start, end)) - - return result - - @classmethod - def _parse_ranges(cls, range_: str, file_size: int) -> list[tuple[int, int]]: - ranges: list[tuple[int, int]] = [] - - for part in range_.split(","): - part = part.strip() - - # If the range is empty or a single dash, we ignore it. - if not part or part == "-": - continue - - # If the range is not in the format "start-end", we ignore it. - if "-" not in part: - continue - - start_str, end_str = part.split("-", 1) - start_str = start_str.strip() - end_str = end_str.strip() - - try: - start = int(start_str) if start_str else max(file_size - int(end_str), 0) - end = int(end_str) + 1 if start_str and end_str and int(end_str) < file_size else file_size - ranges.append((start, end)) - except ValueError: - # If the range is not numeric, we ignore it. - continue - - return ranges - - def generate_multipart( - self, - ranges: Sequence[tuple[int, int]], - boundary: str, - max_size: int, - content_type: str, - ) -> tuple[int, Callable[[int, int], bytes]]: - r""" - Multipart response headers generator. - - ``` - --{boundary}\r\n - Content-Type: {content_type}\r\n - Content-Range: bytes {start}-{end-1}/{max_size}\r\n - \r\n - ..........content...........\r\n - --{boundary}\r\n - Content-Type: {content_type}\r\n - Content-Range: bytes {start}-{end-1}/{max_size}\r\n - \r\n - ..........content...........\r\n - --{boundary}-- - ``` - """ - boundary_len = len(boundary) - static_header_part_len = 49 + boundary_len + len(content_type) + len(str(max_size)) - content_length = sum( - (len(str(start)) + len(str(end - 1)) + static_header_part_len) # Headers - + (end - start) # Content - for start, end in ranges - ) + ( - 4 + boundary_len # --boundary-- - ) - return ( - content_length, - lambda start, end: ( - f"""\ ---{boundary}\r -Content-Type: {content_type}\r -Content-Range: bytes {start}-{end - 1}/{max_size}\r -\r -""" - ).encode("latin-1"), - ) diff --git a/functions/python/data-processor/starlette/routing.py b/functions/python/data-processor/starlette/routing.py deleted file mode 100644 index a035d18cd..000000000 --- a/functions/python/data-processor/starlette/routing.py +++ /dev/null @@ -1,747 +0,0 @@ -from __future__ import annotations - -import contextlib -import functools -import inspect -import re -import traceback -import types -import warnings -from collections.abc import Awaitable, Callable, Collection, Generator, Sequence -from contextlib import AbstractAsyncContextManager, AbstractContextManager, asynccontextmanager -from enum import Enum -from re import Pattern -from typing import Any, TypeVar - -from starlette._exception_handler import wrap_app_handling_exceptions -from starlette._utils import get_route_path, is_async_callable -from starlette.concurrency import run_in_threadpool -from starlette.convertors import CONVERTOR_TYPES, Convertor -from starlette.datastructures import URL, Headers, URLPath -from starlette.exceptions import HTTPException, StarletteDeprecationWarning -from starlette.middleware import Middleware -from starlette.requests import Request -from starlette.responses import PlainTextResponse, RedirectResponse, Response -from starlette.types import ASGIApp, Lifespan, Receive, Scope, Send -from starlette.websockets import WebSocket, WebSocketClose - - -class NoMatchFound(Exception): - """ - Raised by `.url_for(name, **path_params)` and `.url_path_for(name, **path_params)` - if no matching route exists. - """ - - def __init__(self, name: str, path_params: dict[str, Any]) -> None: - params = ", ".join(list(path_params.keys())) - super().__init__(f'No route exists for name "{name}" and params "{params}".') - - -class Match(Enum): - NONE = 0 - PARTIAL = 1 - FULL = 2 - - -def request_response( - func: Callable[[Request], Awaitable[Response] | Response], -) -> ASGIApp: - """ - Takes a function or coroutine `func(request) -> response`, - and returns an ASGI application. - """ - f: Callable[[Request], Awaitable[Response]] = ( - func if is_async_callable(func) else functools.partial(run_in_threadpool, func) # type: ignore[assignment, call-arg] - ) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - request = Request(scope, receive, send) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - response = await f(request) - await response(scope, receive, send) - - await wrap_app_handling_exceptions(app, request)(scope, receive, send) - - return app - - -def websocket_session( - func: Callable[[WebSocket], Awaitable[None]], -) -> ASGIApp: - """ - Takes a coroutine `func(session)`, and returns an ASGI application. - """ - # assert asyncio.iscoroutinefunction(func), "WebSocket endpoints must be async" - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - session = WebSocket(scope, receive=receive, send=send) - - async def app(scope: Scope, receive: Receive, send: Send) -> None: - await func(session) - - await wrap_app_handling_exceptions(app, session)(scope, receive, send) - - return app - - -def get_name(endpoint: Callable[..., Any]) -> str: - return getattr(endpoint, "__name__", endpoint.__class__.__name__) - - -def replace_params( - path: str, - param_convertors: dict[str, Convertor[Any]], - path_params: dict[str, str], -) -> tuple[str, dict[str, str]]: - for key, value in list(path_params.items()): - if "{" + key + "}" in path: - convertor = param_convertors[key] - value = convertor.to_string(value) - path = path.replace("{" + key + "}", value) - path_params.pop(key) - return path, path_params - - -# Match parameters in URL paths, eg. '{param}', and '{param:int}' -PARAM_REGEX = re.compile("{([a-zA-Z_][a-zA-Z0-9_]*)(:[a-zA-Z_][a-zA-Z0-9_]*)?}") - - -def compile_path( - path: str, -) -> tuple[Pattern[str], str, dict[str, Convertor[Any]]]: - """ - Given a path string, like: "/{username:str}", - or a host string, like: "{subdomain}.mydomain.org", return a three-tuple - of (regex, format, {param_name:convertor}). - - regex: "/(?P[^/]+)" - format: "/{username}" - convertors: {"username": StringConvertor()} - """ - is_host = not path.startswith("/") - - path_regex = "^" - path_format = "" - duplicated_params: set[str] = set() - - idx = 0 - param_convertors = {} - for match in PARAM_REGEX.finditer(path): - param_name, convertor_type = match.groups("str") - convertor_type = convertor_type.lstrip(":") - assert convertor_type in CONVERTOR_TYPES, f"Unknown path convertor '{convertor_type}'" - convertor = CONVERTOR_TYPES[convertor_type] - - path_regex += re.escape(path[idx : match.start()]) - path_regex += f"(?P<{param_name}>{convertor.regex})" - - path_format += path[idx : match.start()] - path_format += "{%s}" % param_name - - if param_name in param_convertors: - duplicated_params.add(param_name) - - param_convertors[param_name] = convertor - - idx = match.end() - - if duplicated_params: - names = ", ".join(sorted(duplicated_params)) - ending = "s" if len(duplicated_params) > 1 else "" - raise ValueError(f"Duplicated param name{ending} {names} at path {path}") - - if is_host: - # Align with `Host.matches()` behavior, which ignores port. - hostname = path[idx:].split(":")[0] - path_regex += re.escape(hostname) + "$" - else: - path_regex += re.escape(path[idx:]) + "$" - - path_format += path[idx:] - - return re.compile(path_regex), path_format, param_convertors - - -class BaseRoute: - def matches(self, scope: Scope) -> tuple[Match, Scope]: - raise NotImplementedError() # pragma: no cover - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - raise NotImplementedError() # pragma: no cover - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - raise NotImplementedError() # pragma: no cover - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - """ - A route may be used in isolation as a stand-alone ASGI app. - This is a somewhat contrived case, as they'll almost always be used - within a Router, but could be useful for some tooling and minimal apps. - """ - match, child_scope = self.matches(scope) - if match == Match.NONE: - if scope["type"] == "http": - response = PlainTextResponse("Not Found", status_code=404) - await response(scope, receive, send) - elif scope["type"] == "websocket": # pragma: no branch - websocket_close = WebSocketClose() - await websocket_close(scope, receive, send) - return - - scope.update(child_scope) - await self.handle(scope, receive, send) - - -class Route(BaseRoute): - def __init__( - self, - path: str, - endpoint: Callable[..., Any], - *, - methods: Collection[str] | None = None, - name: str | None = None, - include_in_schema: bool = True, - middleware: Sequence[Middleware] | None = None, - ) -> None: - assert path.startswith("/"), "Routed paths must start with '/'" - self.path = path - self.endpoint = endpoint - self.name = get_name(endpoint) if name is None else name - self.include_in_schema = include_in_schema - - endpoint_handler = endpoint - while isinstance(endpoint_handler, functools.partial): - endpoint_handler = endpoint_handler.func - if inspect.isfunction(endpoint_handler) or inspect.ismethod(endpoint_handler): - # Endpoint is function or method. Treat it as `func(request) -> response`. - self.app = request_response(endpoint) - if methods is None: - methods = ["GET"] - else: - # Endpoint is a class. Treat it as ASGI. - self.app = endpoint - - if middleware is not None: - for cls, args, kwargs in reversed(middleware): - self.app = cls(self.app, *args, **kwargs) - - if methods is None: - self.methods = None - else: - self.methods = {method.upper() for method in methods} - if "GET" in self.methods: - self.methods.add("HEAD") - - self.path_regex, self.path_format, self.param_convertors = compile_path(path) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - path_params: dict[str, Any] - if scope["type"] == "http": - route_path = get_route_path(scope) - match = self.path_regex.match(route_path) - if match: - matched_params = match.groupdict() - for key, value in matched_params.items(): - matched_params[key] = self.param_convertors[key].convert(value) - path_params = dict(scope.get("path_params", {})) - path_params.update(matched_params) - child_scope = {"endpoint": self.endpoint, "path_params": path_params} - if self.methods and scope["method"] not in self.methods: - return Match.PARTIAL, child_scope - else: - return Match.FULL, child_scope - return Match.NONE, {} - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - seen_params = set(path_params.keys()) - expected_params = set(self.param_convertors.keys()) - - if name != self.name or seen_params != expected_params: - raise NoMatchFound(name, path_params) - - path, remaining_params = replace_params(self.path_format, self.param_convertors, path_params) - assert not remaining_params - return URLPath(path=path, protocol="http") - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - if self.methods and scope["method"] not in self.methods: - headers = {"Allow": ", ".join(self.methods)} - if "app" in scope: - raise HTTPException(status_code=405, headers=headers) - else: - response = PlainTextResponse("Method Not Allowed", status_code=405, headers=headers) - await response(scope, receive, send) - else: - await self.app(scope, receive, send) - - def __eq__(self, other: Any) -> bool: - return ( - isinstance(other, Route) - and self.path == other.path - and self.endpoint == other.endpoint - and self.methods == other.methods - ) - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - methods = sorted(self.methods or []) - path, name = self.path, self.name - return f"{class_name}(path={path!r}, name={name!r}, methods={methods!r})" - - -class WebSocketRoute(BaseRoute): - def __init__( - self, - path: str, - endpoint: Callable[..., Any], - *, - name: str | None = None, - middleware: Sequence[Middleware] | None = None, - ) -> None: - assert path.startswith("/"), "Routed paths must start with '/'" - self.path = path - self.endpoint = endpoint - self.name = get_name(endpoint) if name is None else name - - endpoint_handler = endpoint - while isinstance(endpoint_handler, functools.partial): - endpoint_handler = endpoint_handler.func - if inspect.isfunction(endpoint_handler) or inspect.ismethod(endpoint_handler): - # Endpoint is function or method. Treat it as `func(websocket)`. - self.app = websocket_session(endpoint) - else: - # Endpoint is a class. Treat it as ASGI. - self.app = endpoint - - if middleware is not None: - for cls, args, kwargs in reversed(middleware): - self.app = cls(self.app, *args, **kwargs) - - self.path_regex, self.path_format, self.param_convertors = compile_path(path) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - path_params: dict[str, Any] - if scope["type"] == "websocket": - route_path = get_route_path(scope) - match = self.path_regex.match(route_path) - if match: - matched_params = match.groupdict() - for key, value in matched_params.items(): - matched_params[key] = self.param_convertors[key].convert(value) - path_params = dict(scope.get("path_params", {})) - path_params.update(matched_params) - child_scope = {"endpoint": self.endpoint, "path_params": path_params} - return Match.FULL, child_scope - return Match.NONE, {} - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - seen_params = set(path_params.keys()) - expected_params = set(self.param_convertors.keys()) - - if name != self.name or seen_params != expected_params: - raise NoMatchFound(name, path_params) - - path, remaining_params = replace_params(self.path_format, self.param_convertors, path_params) - assert not remaining_params - return URLPath(path=path, protocol="websocket") - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - await self.app(scope, receive, send) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, WebSocketRoute) and self.path == other.path and self.endpoint == other.endpoint - - def __repr__(self) -> str: - return f"{self.__class__.__name__}(path={self.path!r}, name={self.name!r})" - - -class Mount(BaseRoute): - def __init__( - self, - path: str, - app: ASGIApp | None = None, - routes: Sequence[BaseRoute] | None = None, - name: str | None = None, - *, - middleware: Sequence[Middleware] | None = None, - ) -> None: - assert path == "" or path.startswith("/"), "Routed paths must start with '/'" - assert app is not None or routes is not None, "Either 'app=...', or 'routes=' must be specified" - self.path = path.rstrip("/") - if app is not None: - self._base_app: ASGIApp = app - else: - self._base_app = Router(routes=routes) - self.app = self._base_app - if middleware is not None: - for cls, args, kwargs in reversed(middleware): - self.app = cls(self.app, *args, **kwargs) - self.name = name - self.path_regex, self.path_format, self.param_convertors = compile_path(self.path + "/{path:path}") - - @property - def routes(self) -> list[BaseRoute]: - return getattr(self._base_app, "routes", []) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - path_params: dict[str, Any] - if scope["type"] in ("http", "websocket"): # pragma: no branch - root_path = scope.get("root_path", "") - route_path = get_route_path(scope) - match = self.path_regex.match(route_path) - if match: - matched_params = match.groupdict() - for key, value in matched_params.items(): - matched_params[key] = self.param_convertors[key].convert(value) - remaining_path = "/" + matched_params.pop("path") - matched_path = route_path[: -len(remaining_path)] - path_params = dict(scope.get("path_params", {})) - path_params.update(matched_params) - child_scope = { - "path_params": path_params, - # app_root_path will only be set at the top level scope, - # initialized with the (optional) value of a root_path - # set above/before Starlette. And even though any - # mount will have its own child scope with its own respective - # root_path, the app_root_path will always be available in all - # the child scopes with the same top level value because it's - # set only once here with a default, any other child scope will - # just inherit that app_root_path default value stored in the - # scope. All this is needed to support Request.url_for(), as it - # uses the app_root_path to build the URL path. - "app_root_path": scope.get("app_root_path", root_path), - "root_path": root_path + matched_path, - "endpoint": self.app, - } - return Match.FULL, child_scope - return Match.NONE, {} - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - if self.name is not None and name == self.name and "path" in path_params: - # 'name' matches "". - path_params["path"] = path_params["path"].lstrip("/") - path, remaining_params = replace_params(self.path_format, self.param_convertors, path_params) - if not remaining_params: - return URLPath(path=path) - elif self.name is None or name.startswith(self.name + ":"): - if self.name is None: - # No mount name. - remaining_name = name - else: - # 'name' matches ":". - remaining_name = name[len(self.name) + 1 :] - path_kwarg = path_params.get("path") - path_params["path"] = "" - path_prefix, remaining_params = replace_params(self.path_format, self.param_convertors, path_params) - if path_kwarg is not None: - remaining_params["path"] = path_kwarg - for route in self.routes or []: - try: - url = route.url_path_for(remaining_name, **remaining_params) - return URLPath(path=path_prefix.rstrip("/") + str(url), protocol=url.protocol) - except NoMatchFound: - pass - raise NoMatchFound(name, path_params) - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - await self.app(scope, receive, send) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Mount) and self.path == other.path and self.app == other.app - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - name = self.name or "" - return f"{class_name}(path={self.path!r}, name={name!r}, app={self.app!r})" - - -class Host(BaseRoute): - def __init__(self, host: str, app: ASGIApp, name: str | None = None) -> None: - assert not host.startswith("/"), "Host must not start with '/'" - self.host = host - self.app = app - self.name = name - self.host_regex, self.host_format, self.param_convertors = compile_path(host) - - @property - def routes(self) -> list[BaseRoute]: - return getattr(self.app, "routes", []) - - def matches(self, scope: Scope) -> tuple[Match, Scope]: - if scope["type"] in ("http", "websocket"): # pragma:no branch - headers = Headers(scope=scope) - host = headers.get("host", "").split(":")[0] - match = self.host_regex.match(host) - if match: - matched_params = match.groupdict() - for key, value in matched_params.items(): - matched_params[key] = self.param_convertors[key].convert(value) - path_params = dict(scope.get("path_params", {})) - path_params.update(matched_params) - child_scope = {"path_params": path_params, "endpoint": self.app} - return Match.FULL, child_scope - return Match.NONE, {} - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - if self.name is not None and name == self.name and "path" in path_params: - # 'name' matches "". - path = path_params.pop("path") - host, remaining_params = replace_params(self.host_format, self.param_convertors, path_params) - if not remaining_params: - return URLPath(path=path, host=host) - elif self.name is None or name.startswith(self.name + ":"): - if self.name is None: - # No mount name. - remaining_name = name - else: - # 'name' matches ":". - remaining_name = name[len(self.name) + 1 :] - host, remaining_params = replace_params(self.host_format, self.param_convertors, path_params) - for route in self.routes or []: - try: - url = route.url_path_for(remaining_name, **remaining_params) - return URLPath(path=str(url), protocol=url.protocol, host=host) - except NoMatchFound: - pass - raise NoMatchFound(name, path_params) - - async def handle(self, scope: Scope, receive: Receive, send: Send) -> None: - await self.app(scope, receive, send) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Host) and self.host == other.host and self.app == other.app - - def __repr__(self) -> str: - class_name = self.__class__.__name__ - name = self.name or "" - return f"{class_name}(host={self.host!r}, name={name!r}, app={self.app!r})" - - -_T = TypeVar("_T") - - -class _AsyncLiftContextManager(AbstractAsyncContextManager[_T]): - def __init__(self, cm: AbstractContextManager[_T]): - self._cm = cm - - async def __aenter__(self) -> _T: - return self._cm.__enter__() - - async def __aexit__( - self, - exc_type: type[BaseException] | None, - exc_value: BaseException | None, - traceback: types.TracebackType | None, - ) -> bool | None: - return self._cm.__exit__(exc_type, exc_value, traceback) - - -def _wrap_gen_lifespan_context( - lifespan_context: Callable[[Any], Generator[Any, Any, Any]], -) -> Callable[[Any], AbstractAsyncContextManager[Any]]: - cmgr = contextlib.contextmanager(lifespan_context) - - @functools.wraps(cmgr) - def wrapper(app: Any) -> _AsyncLiftContextManager[Any]: - return _AsyncLiftContextManager(cmgr(app)) - - return wrapper - - -class _DefaultLifespan: - def __init__(self, router: Router): - self._router = router - - async def __aenter__(self) -> None: - pass - - async def __aexit__(self, *exc_info: object) -> None: - pass - - def __call__(self: _T, app: object) -> _T: - return self - - -class Router: - def __init__( - self, - routes: Sequence[BaseRoute] | None = None, - redirect_slashes: bool = True, - default: ASGIApp | None = None, - # the generic to Lifespan[AppType] is the type of the top level application - # which the router cannot know statically, so we use Any - lifespan: Lifespan[Any] | None = None, - *, - middleware: Sequence[Middleware] | None = None, - ) -> None: - self.routes = [] if routes is None else list(routes) - self.redirect_slashes = redirect_slashes - self.default = self.not_found if default is None else default - - if lifespan is None: - self.lifespan_context: Lifespan[Any] = _DefaultLifespan(self) - - elif inspect.isasyncgenfunction(lifespan): - warnings.warn( - "async generator function lifespans are deprecated, " - "use an @contextlib.asynccontextmanager function instead", - StarletteDeprecationWarning, - ) - self.lifespan_context = asynccontextmanager(lifespan) - elif inspect.isgeneratorfunction(lifespan): - warnings.warn( - "generator function lifespans are deprecated, use an @contextlib.asynccontextmanager function instead", - StarletteDeprecationWarning, - ) - self.lifespan_context = _wrap_gen_lifespan_context(lifespan) - else: - self.lifespan_context = lifespan - - self.middleware_stack = self.app - if middleware: - for cls, args, kwargs in reversed(middleware): - self.middleware_stack = cls(self.middleware_stack, *args, **kwargs) - - async def not_found(self, scope: Scope, receive: Receive, send: Send) -> None: - if scope["type"] == "websocket": - websocket_close = WebSocketClose() - await websocket_close(scope, receive, send) - return - - # If we're running inside a starlette application then raise an - # exception, so that the configurable exception handler can deal with - # returning the response. For plain ASGI apps, just return the response. - if "app" in scope: - raise HTTPException(status_code=404) - else: - response = PlainTextResponse("Not Found", status_code=404) - await response(scope, receive, send) - - def url_path_for(self, name: str, /, **path_params: Any) -> URLPath: - for route in self.routes: - try: - return route.url_path_for(name, **path_params) - except NoMatchFound: - pass - raise NoMatchFound(name, path_params) - - async def lifespan(self, scope: Scope, receive: Receive, send: Send) -> None: - """ - Handle ASGI lifespan messages, which allows us to manage application - startup and shutdown events. - """ - started = False - app: Any = scope.get("app") - await receive() - try: - async with self.lifespan_context(app) as maybe_state: - if maybe_state is not None: - if "state" not in scope: - raise RuntimeError('The server does not support "state" in the lifespan scope.') - scope["state"].update(maybe_state) - await send({"type": "lifespan.startup.complete"}) - started = True - await receive() - except BaseException: - exc_text = traceback.format_exc() - if started: - await send({"type": "lifespan.shutdown.failed", "message": exc_text}) - else: - await send({"type": "lifespan.startup.failed", "message": exc_text}) - raise - else: - await send({"type": "lifespan.shutdown.complete"}) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - """ - The main entry point to the Router class. - """ - await self.middleware_stack(scope, receive, send) - - async def app(self, scope: Scope, receive: Receive, send: Send) -> None: - assert scope["type"] in ("http", "websocket", "lifespan") - - if "router" not in scope: - scope["router"] = self - - if scope["type"] == "lifespan": - await self.lifespan(scope, receive, send) - return - - partial = None - - for route in self.routes: - # Determine if any route matches the incoming scope, - # and hand over to the matching route if found. - match, child_scope = route.matches(scope) - if match == Match.FULL: - scope.update(child_scope) - await route.handle(scope, receive, send) - return - elif match == Match.PARTIAL and partial is None: - partial = route - partial_scope = child_scope - - if partial is not None: - #  Handle partial matches. These are cases where an endpoint is - # able to handle the request, but is not a preferred option. - # We use this in particular to deal with "405 Method Not Allowed". - scope.update(partial_scope) - await partial.handle(scope, receive, send) - return - - route_path = get_route_path(scope) - if scope["type"] == "http" and self.redirect_slashes and route_path != "/": - redirect_scope = dict(scope) - if route_path.endswith("/"): - redirect_scope["path"] = redirect_scope["path"].rstrip("/") - else: - redirect_scope["path"] = redirect_scope["path"] + "/" - - for route in self.routes: - match, child_scope = route.matches(redirect_scope) - if match != Match.NONE: - redirect_url = URL(scope=redirect_scope) - response = RedirectResponse(url=str(redirect_url)) - await response(scope, receive, send) - return - - await self.default(scope, receive, send) - - def __eq__(self, other: Any) -> bool: - return isinstance(other, Router) and self.routes == other.routes - - def mount(self, path: str, app: ASGIApp, name: str | None = None) -> None: # pragma: no cover - route = Mount(path, app=app, name=name) - self.routes.append(route) - - def host(self, host: str, app: ASGIApp, name: str | None = None) -> None: # pragma: no cover - route = Host(host, app=app, name=name) - self.routes.append(route) - - def add_route( - self, - path: str, - endpoint: Callable[[Request], Awaitable[Response] | Response], - methods: Collection[str] | None = None, - name: str | None = None, - include_in_schema: bool = True, - ) -> None: # pragma: no cover - route = Route( - path, - endpoint=endpoint, - methods=methods, - name=name, - include_in_schema=include_in_schema, - ) - self.routes.append(route) - - def add_websocket_route( - self, - path: str, - endpoint: Callable[[WebSocket], Awaitable[None]], - name: str | None = None, - ) -> None: # pragma: no cover - route = WebSocketRoute(path, endpoint=endpoint, name=name) - self.routes.append(route) diff --git a/functions/python/data-processor/starlette/schemas.py b/functions/python/data-processor/starlette/schemas.py deleted file mode 100644 index cf7d439e3..000000000 --- a/functions/python/data-processor/starlette/schemas.py +++ /dev/null @@ -1,148 +0,0 @@ -from __future__ import annotations - -import inspect -import re -from collections.abc import Callable -from typing import Any, NamedTuple - -from starlette.requests import Request -from starlette.responses import Response -from starlette.routing import BaseRoute, Host, Mount, Route - -try: - import yaml -except ModuleNotFoundError: # pragma: no cover - yaml = None # type: ignore[assignment] - - -class OpenAPIResponse(Response): - media_type = "application/vnd.oai.openapi" - - def render(self, content: Any) -> bytes: - assert yaml is not None, "`pyyaml` must be installed to use OpenAPIResponse." - assert isinstance(content, dict), "The schema passed to OpenAPIResponse should be a dictionary." - return yaml.dump(content, default_flow_style=False).encode("utf-8") - - -class EndpointInfo(NamedTuple): - path: str - http_method: str - func: Callable[..., Any] - - -_remove_converter_pattern = re.compile(r":\w+}") - - -class BaseSchemaGenerator: - def get_schema(self, routes: list[BaseRoute]) -> dict[str, Any]: - raise NotImplementedError() # pragma: no cover - - def get_endpoints(self, routes: list[BaseRoute]) -> list[EndpointInfo]: - """ - Given the routes, yields the following information: - - - path - eg: /users/ - - http_method - one of 'get', 'post', 'put', 'patch', 'delete', 'options' - - func - method ready to extract the docstring - """ - endpoints_info: list[EndpointInfo] = [] - - for route in routes: - if isinstance(route, Mount | Host): - routes = route.routes or [] - if isinstance(route, Mount): - path = self._remove_converter(route.path) - else: - path = "" - sub_endpoints = [ - EndpointInfo( - path="".join((path, sub_endpoint.path)), - http_method=sub_endpoint.http_method, - func=sub_endpoint.func, - ) - for sub_endpoint in self.get_endpoints(routes) - ] - endpoints_info.extend(sub_endpoints) - - elif not isinstance(route, Route) or not route.include_in_schema: - continue - - elif inspect.isfunction(route.endpoint) or inspect.ismethod(route.endpoint): - path = self._remove_converter(route.path) - for method in route.methods or ["GET"]: - if method == "HEAD": - continue - endpoints_info.append(EndpointInfo(path, method.lower(), route.endpoint)) - else: - path = self._remove_converter(route.path) - for method in ["get", "post", "put", "patch", "delete", "options"]: - if not hasattr(route.endpoint, method): - continue - func = getattr(route.endpoint, method) - endpoints_info.append(EndpointInfo(path, method.lower(), func)) - - return endpoints_info - - def _remove_converter(self, path: str) -> str: - """ - Remove the converter from the path. - For example, a route like this: - Route("/users/{id:int}", endpoint=get_user, methods=["GET"]) - Should be represented as `/users/{id}` in the OpenAPI schema. - """ - return _remove_converter_pattern.sub("}", path) - - def parse_docstring(self, func_or_method: Callable[..., Any]) -> dict[str, Any]: - """ - Given a function, parse the docstring as YAML and return a dictionary of info. - """ - docstring = func_or_method.__doc__ - if not docstring: - return {} - - assert yaml is not None, "`pyyaml` must be installed to use parse_docstring." - - # We support having regular docstrings before the schema - # definition. Here we return just the schema part from - # the docstring. - docstring = docstring.split("---")[-1] - - parsed = yaml.safe_load(docstring) - - if not isinstance(parsed, dict): - # A regular docstring (not yaml formatted) can return - # a simple string here, which wouldn't follow the schema. - return {} - - return parsed - - def OpenAPIResponse(self, request: Request) -> Response: - routes = request.app.routes - schema = self.get_schema(routes=routes) - return OpenAPIResponse(schema) - - -class SchemaGenerator(BaseSchemaGenerator): - def __init__(self, base_schema: dict[str, Any]) -> None: - self.base_schema = base_schema - - def get_schema(self, routes: list[BaseRoute]) -> dict[str, Any]: - schema = dict(self.base_schema) - schema.setdefault("paths", {}) - endpoints_info = self.get_endpoints(routes) - - for endpoint in endpoints_info: - parsed = self.parse_docstring(endpoint.func) - - if not parsed: - continue - - if endpoint.path not in schema["paths"]: - schema["paths"][endpoint.path] = {} - - schema["paths"][endpoint.path][endpoint.http_method] = parsed - - return schema diff --git a/functions/python/data-processor/starlette/staticfiles.py b/functions/python/data-processor/starlette/staticfiles.py deleted file mode 100644 index ac39acc43..000000000 --- a/functions/python/data-processor/starlette/staticfiles.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import errno -import importlib.util -import os -import stat -from email.utils import parsedate -from typing import Union - -import anyio -import anyio.to_thread - -from starlette._utils import get_route_path -from starlette.datastructures import URL, Headers -from starlette.exceptions import HTTPException -from starlette.responses import FileResponse, RedirectResponse, Response -from starlette.types import Receive, Scope, Send - -PathLike = Union[str, "os.PathLike[str]"] - - -class NotModifiedResponse(Response): - NOT_MODIFIED_HEADERS = ( - "cache-control", - "content-location", - "date", - "etag", - "expires", - "vary", - ) - - def __init__(self, headers: Headers): - super().__init__( - status_code=304, - headers={name: value for name, value in headers.items() if name in self.NOT_MODIFIED_HEADERS}, - ) - - -class StaticFiles: - def __init__( - self, - *, - directory: PathLike | None = None, - packages: list[str | tuple[str, str]] | None = None, - html: bool = False, - check_dir: bool = True, - follow_symlink: bool = False, - ) -> None: - self.directory = directory - self.packages = packages - self.all_directories = self.get_directories(directory, packages) - self.html = html - self.config_checked = False - self.follow_symlink = follow_symlink - if check_dir and directory is not None and not os.path.isdir(directory): - raise RuntimeError(f"Directory '{directory}' does not exist") - - def get_directories( - self, - directory: PathLike | None = None, - packages: list[str | tuple[str, str]] | None = None, - ) -> list[PathLike]: - """ - Given `directory` and `packages` arguments, return a list of all the - directories that should be used for serving static files from. - """ - directories = [] - if directory is not None: - directories.append(directory) - - for package in packages or []: - if isinstance(package, tuple): - package, statics_dir = package - else: - statics_dir = "statics" - spec = importlib.util.find_spec(package) - assert spec is not None, f"Package {package!r} could not be found." - assert spec.origin is not None, f"Package {package!r} could not be found." - package_directory = os.path.normpath(os.path.join(spec.origin, "..", statics_dir)) - assert os.path.isdir(package_directory), ( - f"Directory '{statics_dir!r}' in package {package!r} could not be found." - ) - directories.append(package_directory) - - return directories - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - """ - The ASGI entry point. - """ - assert scope["type"] == "http" - - if not self.config_checked: - await self.check_config() - self.config_checked = True - - path = self.get_path(scope) - response = await self.get_response(path, scope) - await response(scope, receive, send) - - def get_path(self, scope: Scope) -> str: - """ - Given the ASGI scope, return the `path` string to serve up, - with OS specific path separators, and any '..', '.' components removed. - """ - route_path = get_route_path(scope) - return os.path.normpath(os.path.join(*route_path.split("/"))) - - async def get_response(self, path: str, scope: Scope) -> Response: - """ - Returns an HTTP response, given the incoming path, method and request headers. - """ - if scope["method"] not in ("GET", "HEAD"): - raise HTTPException(status_code=405) - - try: - full_path, stat_result = await anyio.to_thread.run_sync(self.lookup_path, path) - except PermissionError: - raise HTTPException(status_code=401) - except OSError as exc: - # Filename is too long, so it can't be a valid static file. - if exc.errno == errno.ENAMETOOLONG: - raise HTTPException(status_code=404) - - raise exc - except ValueError: - # Null bytes or other invalid characters in the path. - raise HTTPException(status_code=404) - - if stat_result and stat.S_ISREG(stat_result.st_mode): - # We have a static file to serve. - return self.file_response(full_path, stat_result, scope) - - elif stat_result and stat.S_ISDIR(stat_result.st_mode) and self.html: - # We're in HTML mode, and have got a directory URL. - # Check if we have 'index.html' file to serve. - index_path = os.path.join(path, "index.html") - full_path, stat_result = await anyio.to_thread.run_sync(self.lookup_path, index_path) - if stat_result is not None and stat.S_ISREG(stat_result.st_mode): - if not scope["path"].endswith("/"): - # Directory URLs should redirect to always end in "/". - url = URL(scope=scope) - url = url.replace(path=url.path + "/") - return RedirectResponse(url=url) - return self.file_response(full_path, stat_result, scope) - - if self.html: - # Check for '404.html' if we're in HTML mode. - full_path, stat_result = await anyio.to_thread.run_sync(self.lookup_path, "404.html") - if stat_result and stat.S_ISREG(stat_result.st_mode): - return FileResponse(full_path, stat_result=stat_result, status_code=404) - raise HTTPException(status_code=404) - - def lookup_path(self, path: str) -> tuple[str, os.stat_result | None]: - # Reject absolute paths so they cannot escape the served directory. - if path.startswith(("/", "\\")): - return "", None - for directory in self.all_directories: - joined_path = os.path.join(directory, path) - if self.follow_symlink: - full_path = os.path.abspath(joined_path) - directory = os.path.abspath(directory) - else: - full_path = os.path.realpath(joined_path) - directory = os.path.realpath(directory) - if os.path.commonpath([full_path, directory]) != str(directory): - # Don't allow misbehaving clients to break out of the static files directory. - continue - try: - return full_path, os.stat(full_path) - except (FileNotFoundError, NotADirectoryError): - continue - return "", None - - def file_response( - self, - full_path: PathLike, - stat_result: os.stat_result, - scope: Scope, - status_code: int = 200, - ) -> Response: - request_headers = Headers(scope=scope) - - response = FileResponse(full_path, status_code=status_code, stat_result=stat_result) - if self.is_not_modified(response.headers, request_headers): - return NotModifiedResponse(response.headers) - return response - - async def check_config(self) -> None: - """ - Perform a one-off configuration check that StaticFiles is actually - pointed at a directory, so that we can raise loud errors rather than - just returning 404 responses. - """ - if self.directory is None: - return - - try: - stat_result = await anyio.to_thread.run_sync(os.stat, self.directory) - except FileNotFoundError: - raise RuntimeError(f"StaticFiles directory '{self.directory}' does not exist.") - if not (stat.S_ISDIR(stat_result.st_mode) or stat.S_ISLNK(stat_result.st_mode)): - raise RuntimeError(f"StaticFiles path '{self.directory}' is not a directory.") - - def is_not_modified(self, response_headers: Headers, request_headers: Headers) -> bool: - """ - Given the request and response headers, return `True` if an HTTP - "Not Modified" response could be returned instead. - """ - if if_none_match := request_headers.get("if-none-match"): - # The "etag" header is added by FileResponse, so it's always present. - etag = response_headers["etag"] - return etag in [tag.strip().removeprefix("W/") for tag in if_none_match.split(",")] - - try: - if_modified_since = parsedate(request_headers["if-modified-since"]) - last_modified = parsedate(response_headers["last-modified"]) - if if_modified_since is not None and last_modified is not None and if_modified_since >= last_modified: - return True - except KeyError: - pass - - return False diff --git a/functions/python/data-processor/starlette/status.py b/functions/python/data-processor/starlette/status.py deleted file mode 100644 index 202b6b52c..000000000 --- a/functions/python/data-processor/starlette/status.py +++ /dev/null @@ -1,211 +0,0 @@ -""" -HTTP codes -See HTTP Status Code Registry: -https://www.iana.org/assignments/http-status-codes/http-status-codes.xhtml - -And RFC 9110 - https://www.rfc-editor.org/rfc/rfc9110 -""" - -from __future__ import annotations - -import warnings - -from starlette.exceptions import StarletteDeprecationWarning - -__all__ = [ - "HTTP_100_CONTINUE", - "HTTP_101_SWITCHING_PROTOCOLS", - "HTTP_102_PROCESSING", - "HTTP_103_EARLY_HINTS", - "HTTP_200_OK", - "HTTP_201_CREATED", - "HTTP_202_ACCEPTED", - "HTTP_203_NON_AUTHORITATIVE_INFORMATION", - "HTTP_204_NO_CONTENT", - "HTTP_205_RESET_CONTENT", - "HTTP_206_PARTIAL_CONTENT", - "HTTP_207_MULTI_STATUS", - "HTTP_208_ALREADY_REPORTED", - "HTTP_226_IM_USED", - "HTTP_300_MULTIPLE_CHOICES", - "HTTP_301_MOVED_PERMANENTLY", - "HTTP_302_FOUND", - "HTTP_303_SEE_OTHER", - "HTTP_304_NOT_MODIFIED", - "HTTP_305_USE_PROXY", - "HTTP_306_RESERVED", - "HTTP_307_TEMPORARY_REDIRECT", - "HTTP_308_PERMANENT_REDIRECT", - "HTTP_400_BAD_REQUEST", - "HTTP_401_UNAUTHORIZED", - "HTTP_402_PAYMENT_REQUIRED", - "HTTP_403_FORBIDDEN", - "HTTP_404_NOT_FOUND", - "HTTP_405_METHOD_NOT_ALLOWED", - "HTTP_406_NOT_ACCEPTABLE", - "HTTP_407_PROXY_AUTHENTICATION_REQUIRED", - "HTTP_408_REQUEST_TIMEOUT", - "HTTP_409_CONFLICT", - "HTTP_410_GONE", - "HTTP_411_LENGTH_REQUIRED", - "HTTP_412_PRECONDITION_FAILED", - "HTTP_413_CONTENT_TOO_LARGE", - "HTTP_414_URI_TOO_LONG", - "HTTP_415_UNSUPPORTED_MEDIA_TYPE", - "HTTP_416_RANGE_NOT_SATISFIABLE", - "HTTP_417_EXPECTATION_FAILED", - "HTTP_418_IM_A_TEAPOT", - "HTTP_421_MISDIRECTED_REQUEST", - "HTTP_422_UNPROCESSABLE_CONTENT", - "HTTP_423_LOCKED", - "HTTP_424_FAILED_DEPENDENCY", - "HTTP_425_TOO_EARLY", - "HTTP_426_UPGRADE_REQUIRED", - "HTTP_428_PRECONDITION_REQUIRED", - "HTTP_429_TOO_MANY_REQUESTS", - "HTTP_431_REQUEST_HEADER_FIELDS_TOO_LARGE", - "HTTP_451_UNAVAILABLE_FOR_LEGAL_REASONS", - "HTTP_500_INTERNAL_SERVER_ERROR", - "HTTP_501_NOT_IMPLEMENTED", - "HTTP_502_BAD_GATEWAY", - "HTTP_503_SERVICE_UNAVAILABLE", - "HTTP_504_GATEWAY_TIMEOUT", - "HTTP_505_HTTP_VERSION_NOT_SUPPORTED", - "HTTP_506_VARIANT_ALSO_NEGOTIATES", - "HTTP_507_INSUFFICIENT_STORAGE", - "HTTP_508_LOOP_DETECTED", - "HTTP_510_NOT_EXTENDED", - "HTTP_511_NETWORK_AUTHENTICATION_REQUIRED", - "WS_1000_NORMAL_CLOSURE", - "WS_1001_GOING_AWAY", - "WS_1002_PROTOCOL_ERROR", - "WS_1003_UNSUPPORTED_DATA", - "WS_1005_NO_STATUS_RCVD", - "WS_1006_ABNORMAL_CLOSURE", - "WS_1007_INVALID_FRAME_PAYLOAD_DATA", - "WS_1008_POLICY_VIOLATION", - "WS_1009_MESSAGE_TOO_BIG", - "WS_1010_MANDATORY_EXT", - "WS_1011_INTERNAL_ERROR", - "WS_1012_SERVICE_RESTART", - "WS_1013_TRY_AGAIN_LATER", - "WS_1014_BAD_GATEWAY", - "WS_1015_TLS_HANDSHAKE", -] - -HTTP_100_CONTINUE = 100 -HTTP_101_SWITCHING_PROTOCOLS = 101 -HTTP_102_PROCESSING = 102 -HTTP_103_EARLY_HINTS = 103 -HTTP_200_OK = 200 -HTTP_201_CREATED = 201 -HTTP_202_ACCEPTED = 202 -HTTP_203_NON_AUTHORITATIVE_INFORMATION = 203 -HTTP_204_NO_CONTENT = 204 -HTTP_205_RESET_CONTENT = 205 -HTTP_206_PARTIAL_CONTENT = 206 -HTTP_207_MULTI_STATUS = 207 -HTTP_208_ALREADY_REPORTED = 208 -HTTP_226_IM_USED = 226 -HTTP_300_MULTIPLE_CHOICES = 300 -HTTP_301_MOVED_PERMANENTLY = 301 -HTTP_302_FOUND = 302 -HTTP_303_SEE_OTHER = 303 -HTTP_304_NOT_MODIFIED = 304 -HTTP_305_USE_PROXY = 305 -HTTP_306_RESERVED = 306 -HTTP_307_TEMPORARY_REDIRECT = 307 -HTTP_308_PERMANENT_REDIRECT = 308 -HTTP_400_BAD_REQUEST = 400 -HTTP_401_UNAUTHORIZED = 401 -HTTP_402_PAYMENT_REQUIRED = 402 -HTTP_403_FORBIDDEN = 403 -HTTP_404_NOT_FOUND = 404 -HTTP_405_METHOD_NOT_ALLOWED = 405 -HTTP_406_NOT_ACCEPTABLE = 406 -HTTP_407_PROXY_AUTHENTICATION_REQUIRED = 407 -HTTP_408_REQUEST_TIMEOUT = 408 -HTTP_409_CONFLICT = 409 -HTTP_410_GONE = 410 -HTTP_411_LENGTH_REQUIRED = 411 -HTTP_412_PRECONDITION_FAILED = 412 -HTTP_413_CONTENT_TOO_LARGE = 413 -HTTP_414_URI_TOO_LONG = 414 -HTTP_415_UNSUPPORTED_MEDIA_TYPE = 415 -HTTP_416_RANGE_NOT_SATISFIABLE = 416 -HTTP_417_EXPECTATION_FAILED = 417 -HTTP_418_IM_A_TEAPOT = 418 -HTTP_421_MISDIRECTED_REQUEST = 421 -HTTP_422_UNPROCESSABLE_CONTENT = 422 -HTTP_423_LOCKED = 423 -HTTP_424_FAILED_DEPENDENCY = 424 -HTTP_425_TOO_EARLY = 425 -HTTP_426_UPGRADE_REQUIRED = 426 -HTTP_428_PRECONDITION_REQUIRED = 428 -HTTP_429_TOO_MANY_REQUESTS = 429 -HTTP_431_REQUEST_HEADER_FIELDS_TOO_LARGE = 431 -HTTP_451_UNAVAILABLE_FOR_LEGAL_REASONS = 451 -HTTP_500_INTERNAL_SERVER_ERROR = 500 -HTTP_501_NOT_IMPLEMENTED = 501 -HTTP_502_BAD_GATEWAY = 502 -HTTP_503_SERVICE_UNAVAILABLE = 503 -HTTP_504_GATEWAY_TIMEOUT = 504 -HTTP_505_HTTP_VERSION_NOT_SUPPORTED = 505 -HTTP_506_VARIANT_ALSO_NEGOTIATES = 506 -HTTP_507_INSUFFICIENT_STORAGE = 507 -HTTP_508_LOOP_DETECTED = 508 -HTTP_510_NOT_EXTENDED = 510 -HTTP_511_NETWORK_AUTHENTICATION_REQUIRED = 511 - - -""" -WebSocket codes -https://www.iana.org/assignments/websocket/websocket.xml#close-code-number -https://developer.mozilla.org/en-US/docs/Web/API/CloseEvent -""" -WS_1000_NORMAL_CLOSURE = 1000 -WS_1001_GOING_AWAY = 1001 -WS_1002_PROTOCOL_ERROR = 1002 -WS_1003_UNSUPPORTED_DATA = 1003 -WS_1005_NO_STATUS_RCVD = 1005 -WS_1006_ABNORMAL_CLOSURE = 1006 -WS_1007_INVALID_FRAME_PAYLOAD_DATA = 1007 -WS_1008_POLICY_VIOLATION = 1008 -WS_1009_MESSAGE_TOO_BIG = 1009 -WS_1010_MANDATORY_EXT = 1010 -WS_1011_INTERNAL_ERROR = 1011 -WS_1012_SERVICE_RESTART = 1012 -WS_1013_TRY_AGAIN_LATER = 1013 -WS_1014_BAD_GATEWAY = 1014 -WS_1015_TLS_HANDSHAKE = 1015 - -__deprecated__ = { - "HTTP_413_REQUEST_ENTITY_TOO_LARGE": 413, - "HTTP_414_REQUEST_URI_TOO_LONG": 414, - "HTTP_416_REQUESTED_RANGE_NOT_SATISFIABLE": 416, - "HTTP_422_UNPROCESSABLE_ENTITY": 422, -} - - -def __getattr__(name: str) -> int: - deprecation_changes = { - "HTTP_413_REQUEST_ENTITY_TOO_LARGE": "HTTP_413_CONTENT_TOO_LARGE", - "HTTP_414_REQUEST_URI_TOO_LONG": "HTTP_414_URI_TOO_LONG", - "HTTP_416_REQUESTED_RANGE_NOT_SATISFIABLE": "HTTP_416_RANGE_NOT_SATISFIABLE", - "HTTP_422_UNPROCESSABLE_ENTITY": "HTTP_422_UNPROCESSABLE_CONTENT", - } - - deprecated = __deprecated__.get(name) - if deprecated: - warnings.warn( - f"'{name}' is deprecated. Use '{deprecation_changes[name]}' instead.", - category=StarletteDeprecationWarning, - stacklevel=3, - ) - return deprecated - - raise AttributeError(f"module 'starlette.status' has no attribute '{name}'") - - -def __dir__() -> list[str]: - return sorted(list(__all__) + list(__deprecated__.keys())) # pragma: no cover diff --git a/functions/python/data-processor/starlette/templating.py b/functions/python/data-processor/starlette/templating.py deleted file mode 100644 index 67872b377..000000000 --- a/functions/python/data-processor/starlette/templating.py +++ /dev/null @@ -1,156 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable, Mapping, Sequence -from os import PathLike -from typing import TYPE_CHECKING, Any, overload - -from starlette.background import BackgroundTask -from starlette.datastructures import URL -from starlette.requests import Request -from starlette.responses import HTMLResponse -from starlette.types import Receive, Scope, Send - -try: - import jinja2 - - # @contextfunction was renamed to @pass_context in Jinja 3.0, and was removed in 3.1 - # hence we try to get pass_context (most installs will be >=3.1) - # and fall back to contextfunction, - # adding a type ignore for mypy to let us access an attribute that may not exist - if TYPE_CHECKING: - pass_context = jinja2.pass_context - else: - if hasattr(jinja2, "pass_context"): - pass_context = jinja2.pass_context - else: # pragma: no cover - pass_context = jinja2.contextfunction # type: ignore[attr-defined] -except ImportError as _import_error: # pragma: no cover - raise ImportError("jinja2 must be installed to use Jinja2Templates") from _import_error - - -class _TemplateResponse(HTMLResponse): - def __init__( - self, - template: Any, - context: dict[str, Any], - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - ): - self.template = template - self.context = context - content = template.render(context) - super().__init__(content, status_code, headers, media_type, background) - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - request = self.context.get("request", {}) - extensions = request.get("extensions", {}) - if "http.response.debug" in extensions: # pragma: no branch - await send({"type": "http.response.debug", "info": {"template": self.template, "context": self.context}}) - await super().__call__(scope, receive, send) - - -class Jinja2Templates: - """Jinja2 template renderer. - - Example: - ```python - from starlette.templating import Jinja2Templates - - templates = Jinja2Templates(directory="templates") - - async def homepage(request: Request) -> Response: - return templates.TemplateResponse(request, "index.html") - ``` - """ - - @overload - def __init__( - self, - directory: str | PathLike[str] | Sequence[str | PathLike[str]], - *, - context_processors: list[Callable[[Request], dict[str, Any]]] | None = None, - ) -> None: ... - - @overload - def __init__( - self, - *, - env: jinja2.Environment, - context_processors: list[Callable[[Request], dict[str, Any]]] | None = None, - ) -> None: ... - - def __init__( - self, - directory: str | PathLike[str] | Sequence[str | PathLike[str]] | None = None, - *, - context_processors: list[Callable[[Request], dict[str, Any]]] | None = None, - env: jinja2.Environment | None = None, - ) -> None: - assert bool(directory) ^ bool(env), "either 'directory' or 'env' arguments must be passed" - self.context_processors = context_processors or [] - if directory is not None: - loader = jinja2.FileSystemLoader(directory) - self.env = jinja2.Environment(loader=loader, autoescape=jinja2.select_autoescape()) - elif env is not None: # pragma: no branch - self.env = env - - self._setup_env_defaults(self.env) - - def _setup_env_defaults(self, env: jinja2.Environment) -> None: - @pass_context - def url_for( - context: dict[str, Any], - name: str, - /, - **path_params: Any, - ) -> URL: - request: Request = context["request"] - return request.url_for(name, **path_params) - - env.globals.setdefault("url_for", url_for) - - def get_template(self, name: str) -> jinja2.Template: - return self.env.get_template(name) - - def TemplateResponse( - self, - request: Request, - name: str, - context: dict[str, Any] | None = None, - status_code: int = 200, - headers: Mapping[str, str] | None = None, - media_type: str | None = None, - background: BackgroundTask | None = None, - ) -> _TemplateResponse: - """ - Render a template and return an HTML response. - - Args: - request: The incoming request instance. - name: The template file name to render. - context: Variables to pass to the template. - status_code: HTTP status code for the response. - headers: Additional headers to include in the response. - media_type: Media type for the response. - background: Background task to run after response is sent. - - Returns: - An HTML response with the rendered template content. - """ - context = context or {} - - context.setdefault("request", request) - for context_processor in self.context_processors: - context.update(context_processor(request)) - - template = self.get_template(name) - return _TemplateResponse( - template, - context, - status_code=status_code, - headers=headers, - media_type=media_type, - background=background, - ) diff --git a/functions/python/data-processor/starlette/testclient.py b/functions/python/data-processor/starlette/testclient.py deleted file mode 100644 index 63d879098..000000000 --- a/functions/python/data-processor/starlette/testclient.py +++ /dev/null @@ -1,748 +0,0 @@ -from __future__ import annotations - -import contextlib -import inspect -import io -import json -import math -import sys -import warnings -from collections.abc import Awaitable, Callable, Generator, Iterable, Mapping, MutableMapping, Sequence -from concurrent.futures import Future -from contextlib import AbstractContextManager -from types import GeneratorType -from typing import TYPE_CHECKING, Any, Literal, TypedDict, TypeGuard, cast -from urllib.parse import unquote, urljoin - -import anyio -import anyio.abc -import anyio.from_thread -from anyio.streams.stapled import StapledObjectStream - -from starlette._utils import is_async_callable -from starlette.exceptions import StarletteDeprecationWarning -from starlette.types import ASGIApp, Message, Receive, Scope, Send -from starlette.websockets import WebSocketDisconnect - -if sys.version_info >= (3, 11): # pragma: no cover - from typing import Self -else: # pragma: no cover - from typing_extensions import Self - -if TYPE_CHECKING: - import httpx2 as httpx -else: - try: - import httpx2 as httpx - except ModuleNotFoundError: # pragma: no cover - try: - import httpx - except ModuleNotFoundError: - raise RuntimeError( - "The starlette.testclient module requires the httpx2 package to be installed.\n" - "You can install this with:\n" - " $ pip install httpx2\n" - ) from None - else: - warnings.warn( - "Using `httpx` with `starlette.testclient` is deprecated; install `httpx2` instead.", - StarletteDeprecationWarning, - stacklevel=2, - ) - -_PortalFactoryType = Callable[[], AbstractContextManager[anyio.abc.BlockingPortal]] - -ASGIInstance = Callable[[Receive, Send], Awaitable[None]] -ASGI2App = Callable[[Scope], ASGIInstance] -ASGI3App = Callable[[Scope, Receive, Send], Awaitable[None]] - - -_RequestData = Mapping[str, str | Iterable[str] | bytes] - - -def _is_asgi3(app: ASGI2App | ASGI3App) -> TypeGuard[ASGI3App]: - if inspect.isclass(app): - return hasattr(app, "__await__") - return is_async_callable(app) - - -class _WrapASGI2: - """ - Provide an ASGI3 interface onto an ASGI2 app. - """ - - def __init__(self, app: ASGI2App) -> None: - self.app = app - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - instance = self.app(scope) - await instance(receive, send) - - -class _AsyncBackend(TypedDict): - backend: str - backend_options: dict[str, Any] - - -class _Upgrade(Exception): - def __init__(self, session: WebSocketTestSession) -> None: - self.session = session - - -class WebSocketDenialResponse( # type: ignore[misc] - httpx.Response, - WebSocketDisconnect, -): - """ - A special case of `WebSocketDisconnect`, raised in the `TestClient` if the - `WebSocket` is closed before being accepted with a `send_denial_response()`. - """ - - -class WebSocketTestSession: - def __init__( - self, - app: ASGI3App, - scope: Scope, - portal_factory: _PortalFactoryType, - ) -> None: - self.app = app - self.scope = scope - self.accepted_subprotocol = None - self.portal_factory = portal_factory - self.extra_headers = None - - def __enter__(self) -> Self: - with contextlib.ExitStack() as stack: - self.portal = portal = stack.enter_context(self.portal_factory()) - fut, cs = portal.start_task(self._run) - stack.callback(fut.result) - stack.callback(portal.call, cs.cancel) - self.send({"type": "websocket.connect"}) - message = self.receive() - self._raise_on_close(message) - self.accepted_subprotocol = message.get("subprotocol", None) - self.extra_headers = message.get("headers", None) - stack.callback(self.close, 1000) - self.exit_stack = stack.pop_all() - return self - - def __exit__(self, *args: Any) -> bool | None: - return self.exit_stack.__exit__(*args) - - async def _run(self, *, task_status: anyio.abc.TaskStatus[anyio.CancelScope]) -> None: - """ - The sub-thread in which the websocket session runs. - """ - send: anyio.create_memory_object_stream[Message] = anyio.create_memory_object_stream(math.inf) - send_tx, send_rx = send - receive: anyio.create_memory_object_stream[Message] = anyio.create_memory_object_stream(math.inf) - receive_tx, receive_rx = receive - with send_tx, send_rx, receive_tx, receive_rx, anyio.CancelScope() as cs: - self._receive_tx = receive_tx - self._send_rx = send_rx - task_status.started(cs) - await self.app(self.scope, receive_rx.receive, send_tx.send) - - # wait for cs.cancel to be called before closing streams - await anyio.sleep_forever() - - def _raise_on_close(self, message: Message) -> None: - if message["type"] == "websocket.close": - raise WebSocketDisconnect(code=message.get("code", 1000), reason=message.get("reason", "")) - elif message["type"] == "websocket.http.response.start": - status_code: int = message["status"] - headers: list[tuple[bytes, bytes]] = message["headers"] - body: list[bytes] = [] - while True: - message = self.receive() - assert message["type"] == "websocket.http.response.body" - body.append(message["body"]) - if not message.get("more_body", False): - break - raise WebSocketDenialResponse(status_code=status_code, headers=headers, content=b"".join(body)) - - def send(self, message: Message) -> None: - self.portal.call(self._receive_tx.send, message) - - def send_text(self, data: str) -> None: - self.send({"type": "websocket.receive", "text": data}) - - def send_bytes(self, data: bytes) -> None: - self.send({"type": "websocket.receive", "bytes": data}) - - def send_json(self, data: Any, mode: Literal["text", "binary"] = "text") -> None: - text = json.dumps(data, separators=(",", ":"), ensure_ascii=False) - if mode == "text": - self.send({"type": "websocket.receive", "text": text}) - else: - self.send({"type": "websocket.receive", "bytes": text.encode("utf-8")}) - - def close(self, code: int = 1000, reason: str | None = None) -> None: - self.send({"type": "websocket.disconnect", "code": code, "reason": reason}) - - def receive(self) -> Message: - return self.portal.call(self._send_rx.receive) - - def receive_text(self) -> str: - message = self.receive() - self._raise_on_close(message) - return cast(str, message["text"]) - - def receive_bytes(self) -> bytes: - message = self.receive() - self._raise_on_close(message) - return cast(bytes, message["bytes"]) - - def receive_json(self, mode: Literal["text", "binary"] = "text") -> Any: - message = self.receive() - self._raise_on_close(message) - if mode == "text": - text = message["text"] - else: - text = message["bytes"].decode("utf-8") - return json.loads(text) - - -class _TestClientTransport(httpx.BaseTransport): - def __init__( - self, - app: ASGI3App, - portal_factory: _PortalFactoryType, - raise_server_exceptions: bool = True, - root_path: str = "", - *, - client: tuple[str, int], - app_state: dict[str, Any], - ) -> None: - self.app = app - self.raise_server_exceptions = raise_server_exceptions - self.root_path = root_path - self.portal_factory = portal_factory - self.app_state = app_state - self.client = client - - def handle_request(self, request: httpx.Request) -> httpx.Response: - scheme = request.url.scheme - netloc = request.url.netloc.decode(encoding="ascii") - path = request.url.path - raw_path = request.url.raw_path - query = request.url.query.decode(encoding="ascii") - - default_port = {"http": 80, "ws": 80, "https": 443, "wss": 443}[scheme] - - if ":" in netloc: - host, port_string = netloc.split(":", 1) - port = int(port_string) - else: - host = netloc - port = default_port - - # Include the 'host' header. - if "host" in request.headers: - headers: list[tuple[bytes, bytes]] = [] - elif port == default_port: # pragma: no cover - headers = [(b"host", host.encode())] - else: # pragma: no cover - headers = [(b"host", (f"{host}:{port}").encode())] - - # Include other request headers. - headers += [(key.lower().encode(), value.encode()) for key, value in request.headers.multi_items()] - - scope: dict[str, Any] - - if scheme in {"ws", "wss"}: - subprotocol = request.headers.get("sec-websocket-protocol", None) - if subprotocol is None: - subprotocols: Sequence[str] = [] - else: - subprotocols = [value.strip() for value in subprotocol.split(",")] - scope = { - "type": "websocket", - "path": unquote(path), - "raw_path": raw_path.split(b"?", 1)[0], - "root_path": self.root_path, - "scheme": scheme, - "query_string": query.encode(), - "headers": headers, - "client": self.client, - "server": [host, port], - "subprotocols": subprotocols, - "state": self.app_state.copy(), - "extensions": {"websocket.http.response": {}}, - } - session = WebSocketTestSession(self.app, scope, self.portal_factory) - raise _Upgrade(session) - - scope = { - "type": "http", - "http_version": "1.1", - "method": request.method, - "path": unquote(path), - "raw_path": raw_path.split(b"?", 1)[0], - "root_path": self.root_path, - "scheme": scheme, - "query_string": query.encode(), - "headers": headers, - "client": self.client, - "server": [host, port], - "extensions": {"http.response.debug": {}}, - "state": self.app_state.copy(), - } - - request_complete = False - response_started = False - response_complete: anyio.Event - raw_kwargs: dict[str, Any] = {"stream": io.BytesIO()} - template = None - context = None - - async def receive() -> Message: - nonlocal request_complete - - if request_complete: - if not response_complete.is_set(): - await response_complete.wait() - return {"type": "http.disconnect"} - - body = request.read() - if isinstance(body, str): - body_bytes: bytes = body.encode("utf-8") # pragma: no cover - elif body is None: - body_bytes = b"" # pragma: no cover - elif isinstance(body, GeneratorType): - try: # pragma: no cover - chunk = body.send(None) - if isinstance(chunk, str): - chunk = chunk.encode("utf-8") - return {"type": "http.request", "body": chunk, "more_body": True} - except StopIteration: # pragma: no cover - request_complete = True - return {"type": "http.request", "body": b""} - else: - body_bytes = body - - request_complete = True - return {"type": "http.request", "body": body_bytes} - - async def send(message: Message) -> None: - nonlocal raw_kwargs, response_started, template, context - - if message["type"] == "http.response.start": - assert not response_started, 'Received multiple "http.response.start" messages.' - raw_kwargs["status_code"] = message["status"] - raw_kwargs["headers"] = [(key.decode(), value.decode()) for key, value in message.get("headers", [])] - response_started = True - elif message["type"] == "http.response.body": - assert response_started, 'Received "http.response.body" without "http.response.start".' - assert not response_complete.is_set(), 'Received "http.response.body" after response completed.' - body = message.get("body", b"") - more_body = message.get("more_body", False) - if request.method != "HEAD": - raw_kwargs["stream"].write(body) - if not more_body: - raw_kwargs["stream"].seek(0) - response_complete.set() - elif message["type"] == "http.response.debug": - template = message["info"]["template"] - context = message["info"]["context"] - - try: - with self.portal_factory() as portal: - response_complete = portal.call(anyio.Event) - portal.call(self.app, scope, receive, send) - except BaseException as exc: - if self.raise_server_exceptions: - raise exc - - if self.raise_server_exceptions: - assert response_started, "TestClient did not receive any response." - elif not response_started: - raw_kwargs = { - "status_code": 500, - "headers": [], - "stream": io.BytesIO(), - } - - raw_kwargs["stream"] = httpx.ByteStream(raw_kwargs["stream"].read()) - - response = httpx.Response(**raw_kwargs, request=request) - if template is not None: - response.template = template # type: ignore[attr-defined] - response.context = context # type: ignore[attr-defined] - return response - - -class TestClient(httpx.Client): - __test__ = False - task: Future[None] - portal: anyio.abc.BlockingPortal | None = None - - def __init__( - self, - app: ASGIApp, - base_url: str = "http://testserver", - raise_server_exceptions: bool = True, - root_path: str = "", - backend: Literal["asyncio", "trio"] = "asyncio", - backend_options: dict[str, Any] | None = None, - cookies: httpx._types.CookieTypes | None = None, - headers: dict[str, str] | None = None, - follow_redirects: bool = True, - client: tuple[str, int] = ("testclient", 50000), - ) -> None: - self.async_backend = _AsyncBackend(backend=backend, backend_options=backend_options or {}) - if _is_asgi3(app): - asgi_app = app - else: - app = cast(ASGI2App, app) # type: ignore[assignment] - asgi_app = _WrapASGI2(app) # type: ignore[arg-type] - self.app = asgi_app - self.app_state: dict[str, Any] = {} - transport = _TestClientTransport( - self.app, - portal_factory=self._portal_factory, - raise_server_exceptions=raise_server_exceptions, - root_path=root_path, - app_state=self.app_state, - client=client, - ) - if headers is None: - headers = {} - headers.setdefault("user-agent", "testclient") - super().__init__( - base_url=base_url, - headers=headers, - transport=transport, - follow_redirects=follow_redirects, - cookies=cookies, - ) - - @contextlib.contextmanager - def _portal_factory(self) -> Generator[anyio.abc.BlockingPortal, None, None]: - if self.portal is not None: - yield self.portal - else: - with anyio.from_thread.start_blocking_portal(**self.async_backend) as portal: - yield portal - - def request( # type: ignore[override] - self, - method: str, - url: httpx._types.URLTypes, - *, - content: httpx._types.RequestContent | None = None, - data: _RequestData | None = None, - files: httpx._types.RequestFiles | None = None, - json: Any = None, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - if timeout is not httpx.USE_CLIENT_DEFAULT: - warnings.warn( - "You should not use the 'timeout' argument with the TestClient. " - "See https://github.com/Kludex/starlette/issues/1108 for more information.", - StarletteDeprecationWarning, - stacklevel=2, - ) - url = self._merge_url(url) - return super().request( - method, - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def get( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().get( - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def options( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().options( - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def head( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().head( - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def post( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - content: httpx._types.RequestContent | None = None, - data: _RequestData | None = None, - files: httpx._types.RequestFiles | None = None, - json: Any = None, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().post( - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def put( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - content: httpx._types.RequestContent | None = None, - data: _RequestData | None = None, - files: httpx._types.RequestFiles | None = None, - json: Any = None, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().put( - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def patch( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - content: httpx._types.RequestContent | None = None, - data: _RequestData | None = None, - files: httpx._types.RequestFiles | None = None, - json: Any = None, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().patch( - url, - content=content, - data=data, - files=files, - json=json, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def delete( # type: ignore[override] - self, - url: httpx._types.URLTypes, - *, - params: httpx._types.QueryParamTypes | None = None, - headers: httpx._types.HeaderTypes | None = None, - cookies: httpx._types.CookieTypes | None = None, - auth: httpx._types.AuthTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - follow_redirects: bool | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - timeout: httpx._types.TimeoutTypes | httpx._client.UseClientDefault = httpx._client.USE_CLIENT_DEFAULT, - extensions: dict[str, Any] | None = None, - ) -> httpx.Response: - return super().delete( - url, - params=params, - headers=headers, - cookies=cookies, - auth=auth, - follow_redirects=follow_redirects, - timeout=timeout, - extensions=extensions, - ) - - def websocket_connect( - self, - url: str, - subprotocols: Sequence[str] | None = None, - **kwargs: Any, - ) -> WebSocketTestSession: - url = urljoin("ws://testserver", url) - headers = kwargs.get("headers", {}) - headers.setdefault("connection", "upgrade") - headers.setdefault("sec-websocket-key", "testserver==") - headers.setdefault("sec-websocket-version", "13") - if subprotocols is not None: - headers.setdefault("sec-websocket-protocol", ", ".join(subprotocols)) - kwargs["headers"] = headers - try: - super().request("GET", url, **kwargs) - except _Upgrade as exc: - session = exc.session - else: - raise RuntimeError("Expected WebSocket upgrade") # pragma: no cover - - return session - - def __enter__(self) -> Self: - with contextlib.ExitStack() as stack: - self.portal = portal = stack.enter_context(anyio.from_thread.start_blocking_portal(**self.async_backend)) - - @stack.callback - def reset_portal() -> None: - self.portal = None - - send: anyio.create_memory_object_stream[MutableMapping[str, Any] | None] = ( - anyio.create_memory_object_stream(math.inf) - ) - receive: anyio.create_memory_object_stream[MutableMapping[str, Any]] = anyio.create_memory_object_stream( - math.inf - ) - for channel in (*send, *receive): - stack.callback(channel.close) - self.stream_send = StapledObjectStream(*send) - self.stream_receive = StapledObjectStream(*receive) - self.task = portal.start_task_soon(self.lifespan) - portal.call(self.wait_startup) - - @stack.callback - def wait_shutdown() -> None: - portal.call(self.wait_shutdown) - - self.exit_stack = stack.pop_all() - - return self - - def __exit__(self, *args: Any) -> None: - self.exit_stack.close() - - async def lifespan(self) -> None: - scope = {"type": "lifespan", "state": self.app_state} - try: - await self.app(scope, self.stream_receive.receive, self.stream_send.send) - finally: - await self.stream_send.send(None) - - async def wait_startup(self) -> None: - await self.stream_receive.send({"type": "lifespan.startup"}) - - async def receive() -> Any: - message = await self.stream_send.receive() - if message is None: - self.task.result() - return message - - message = await receive() - assert message["type"] in ( - "lifespan.startup.complete", - "lifespan.startup.failed", - ) - if message["type"] == "lifespan.startup.failed": - await receive() - - async def wait_shutdown(self) -> None: - async def receive() -> Any: - message = await self.stream_send.receive() - if message is None: - self.task.result() - return message - - await self.stream_receive.send({"type": "lifespan.shutdown"}) - message = await receive() - assert message["type"] in ( - "lifespan.shutdown.complete", - "lifespan.shutdown.failed", - ) - if message["type"] == "lifespan.shutdown.failed": - await receive() diff --git a/functions/python/data-processor/starlette/types.py b/functions/python/data-processor/starlette/types.py deleted file mode 100644 index b767ae159..000000000 --- a/functions/python/data-processor/starlette/types.py +++ /dev/null @@ -1,26 +0,0 @@ -from collections.abc import Awaitable, Callable, Mapping, MutableMapping -from contextlib import AbstractAsyncContextManager -from typing import TYPE_CHECKING, Any, TypeVar - -if TYPE_CHECKING: - from starlette.requests import Request - from starlette.responses import Response - from starlette.websockets import WebSocket - -AppType = TypeVar("AppType") - -Scope = MutableMapping[str, Any] -Message = MutableMapping[str, Any] - -Receive = Callable[[], Awaitable[Message]] -Send = Callable[[Message], Awaitable[None]] - -ASGIApp = Callable[[Scope, Receive, Send], Awaitable[None]] - -StatelessLifespan = Callable[[AppType], AbstractAsyncContextManager[None]] -StatefulLifespan = Callable[[AppType], AbstractAsyncContextManager[Mapping[str, Any]]] -Lifespan = StatelessLifespan[AppType] | StatefulLifespan[AppType] - -HTTPExceptionHandler = Callable[["Request", Exception], "Response | Awaitable[Response]"] -WebSocketExceptionHandler = Callable[["WebSocket", Exception], Awaitable[None]] -ExceptionHandler = HTTPExceptionHandler | WebSocketExceptionHandler diff --git a/functions/python/data-processor/starlette/websockets.py b/functions/python/data-processor/starlette/websockets.py deleted file mode 100644 index 3ab709a30..000000000 --- a/functions/python/data-processor/starlette/websockets.py +++ /dev/null @@ -1,196 +0,0 @@ -from __future__ import annotations - -import enum -import json -from collections.abc import AsyncIterator, Iterable -from typing import Any, cast - -from starlette.requests import HTTPConnection, StateT -from starlette.responses import Response -from starlette.types import Message, Receive, Scope, Send - - -class WebSocketState(enum.Enum): - CONNECTING = 0 - CONNECTED = 1 - DISCONNECTED = 2 - RESPONSE = 3 - - -class WebSocketDisconnect(Exception): - def __init__(self, code: int = 1000, reason: str | None = None) -> None: - self.code = code - self.reason = reason or "" - - -class WebSocket(HTTPConnection[StateT]): - def __init__(self, scope: Scope, receive: Receive, send: Send) -> None: - super().__init__(scope) - assert scope["type"] == "websocket" - self._receive = receive - self._send = send - self.client_state = WebSocketState.CONNECTING - self.application_state = WebSocketState.CONNECTING - - async def receive(self) -> Message: - """ - Receive ASGI websocket messages, ensuring valid state transitions. - """ - if self.client_state == WebSocketState.CONNECTING: - message = await self._receive() - message_type = message["type"] - if message_type != "websocket.connect": - raise RuntimeError(f'Expected ASGI message "websocket.connect", but got {message_type!r}') - self.client_state = WebSocketState.CONNECTED - return message - elif self.client_state == WebSocketState.CONNECTED: - message = await self._receive() - message_type = message["type"] - if message_type not in {"websocket.receive", "websocket.disconnect"}: - raise RuntimeError( - f'Expected ASGI message "websocket.receive" or "websocket.disconnect", but got {message_type!r}' - ) - if message_type == "websocket.disconnect": - self.client_state = WebSocketState.DISCONNECTED - return message - else: - raise RuntimeError('Cannot call "receive" once a disconnect message has been received.') - - async def send(self, message: Message) -> None: - """ - Send ASGI websocket messages, ensuring valid state transitions. - """ - if self.application_state == WebSocketState.CONNECTING: - message_type = message["type"] - if message_type not in {"websocket.accept", "websocket.close", "websocket.http.response.start"}: - raise RuntimeError( - 'Expected ASGI message "websocket.accept", "websocket.close" or "websocket.http.response.start", ' - f"but got {message_type!r}" - ) - if message_type == "websocket.close": - self.application_state = WebSocketState.DISCONNECTED - elif message_type == "websocket.http.response.start": - self.application_state = WebSocketState.RESPONSE - else: - self.application_state = WebSocketState.CONNECTED - await self._send(message) - elif self.application_state == WebSocketState.CONNECTED: - message_type = message["type"] - if message_type not in {"websocket.send", "websocket.close"}: - raise RuntimeError( - f'Expected ASGI message "websocket.send" or "websocket.close", but got {message_type!r}' - ) - if message_type == "websocket.close": - self.application_state = WebSocketState.DISCONNECTED - try: - await self._send(message) - except OSError: - self.application_state = WebSocketState.DISCONNECTED - raise WebSocketDisconnect(code=1006) - elif self.application_state == WebSocketState.RESPONSE: - message_type = message["type"] - if message_type != "websocket.http.response.body": - raise RuntimeError(f'Expected ASGI message "websocket.http.response.body", but got {message_type!r}') - if not message.get("more_body", False): - self.application_state = WebSocketState.DISCONNECTED - await self._send(message) - else: - raise RuntimeError('Cannot call "send" once a close message has been sent.') - - async def accept( - self, - subprotocol: str | None = None, - headers: Iterable[tuple[bytes, bytes]] | None = None, - ) -> None: - headers = headers or [] - - if self.client_state == WebSocketState.CONNECTING: # pragma: no branch - # If we haven't yet seen the 'connect' message, then wait for it first. - await self.receive() - await self.send({"type": "websocket.accept", "subprotocol": subprotocol, "headers": headers}) - - def _raise_on_disconnect(self, message: Message) -> None: - if message["type"] == "websocket.disconnect": - raise WebSocketDisconnect(message["code"], message.get("reason")) - - async def receive_text(self) -> str: - if self.application_state != WebSocketState.CONNECTED: - raise RuntimeError('WebSocket is not connected. Need to call "accept" first.') - message = await self.receive() - self._raise_on_disconnect(message) - return cast(str, message["text"]) - - async def receive_bytes(self) -> bytes: - if self.application_state != WebSocketState.CONNECTED: - raise RuntimeError('WebSocket is not connected. Need to call "accept" first.') - message = await self.receive() - self._raise_on_disconnect(message) - return cast(bytes, message["bytes"]) - - async def receive_json(self, mode: str = "text") -> Any: - if mode not in {"text", "binary"}: - raise RuntimeError('The "mode" argument should be "text" or "binary".') - if self.application_state != WebSocketState.CONNECTED: - raise RuntimeError('WebSocket is not connected. Need to call "accept" first.') - message = await self.receive() - self._raise_on_disconnect(message) - - if mode == "text": - text = message["text"] - else: - text = message["bytes"].decode("utf-8") - return json.loads(text) - - async def iter_text(self) -> AsyncIterator[str]: - try: - while True: - yield await self.receive_text() - except WebSocketDisconnect: - pass - - async def iter_bytes(self) -> AsyncIterator[bytes]: - try: - while True: - yield await self.receive_bytes() - except WebSocketDisconnect: - pass - - async def iter_json(self) -> AsyncIterator[Any]: - try: - while True: - yield await self.receive_json() - except WebSocketDisconnect: - pass - - async def send_text(self, data: str) -> None: - await self.send({"type": "websocket.send", "text": data}) - - async def send_bytes(self, data: bytes) -> None: - await self.send({"type": "websocket.send", "bytes": data}) - - async def send_json(self, data: Any, mode: str = "text") -> None: - if mode not in {"text", "binary"}: - raise RuntimeError('The "mode" argument should be "text" or "binary".') - text = json.dumps(data, separators=(",", ":"), ensure_ascii=False) - if mode == "text": - await self.send({"type": "websocket.send", "text": text}) - else: - await self.send({"type": "websocket.send", "bytes": text.encode("utf-8")}) - - async def close(self, code: int = 1000, reason: str | None = None) -> None: - await self.send({"type": "websocket.close", "code": code, "reason": reason or ""}) - - async def send_denial_response(self, response: Response) -> None: - if "websocket.http.response" in self.scope.get("extensions", {}): - await response(self.scope, self.receive, self.send) - else: - raise RuntimeError("The server doesn't support the Websocket Denial Response extension.") - - -class WebSocketClose: - def __init__(self, code: int = 1000, reason: str | None = None) -> None: - self.code = code - self.reason = reason or "" - - async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: - await send({"type": "websocket.close", "code": self.code, "reason": self.reason}) diff --git a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/INSTALLER b/functions/python/data-processor/typing_extensions-4.15.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/METADATA b/functions/python/data-processor/typing_extensions-4.15.0.dist-info/METADATA deleted file mode 100644 index b09cb50e1..000000000 --- a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/METADATA +++ /dev/null @@ -1,72 +0,0 @@ -Metadata-Version: 2.4 -Name: typing_extensions -Version: 4.15.0 -Summary: Backported and Experimental Type Hints for Python 3.9+ -Keywords: annotations,backport,checker,checking,function,hinting,hints,type,typechecking,typehinting,typehints,typing -Author-email: "Guido van Rossum, Jukka Lehtosalo, Łukasz Langa, Michael Lee" -Requires-Python: >=3.9 -Description-Content-Type: text/markdown -License-Expression: PSF-2.0 -Classifier: Development Status :: 5 - Production/Stable -Classifier: Environment :: Console -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Topic :: Software Development -License-File: LICENSE -Project-URL: Bug Tracker, https://github.com/python/typing_extensions/issues -Project-URL: Changes, https://github.com/python/typing_extensions/blob/main/CHANGELOG.md -Project-URL: Documentation, https://typing-extensions.readthedocs.io/ -Project-URL: Home, https://github.com/python/typing_extensions -Project-URL: Q & A, https://github.com/python/typing/discussions -Project-URL: Repository, https://github.com/python/typing_extensions - -# Typing Extensions - -[![Chat at https://gitter.im/python/typing](https://badges.gitter.im/python/typing.svg)](https://gitter.im/python/typing) - -[Documentation](https://typing-extensions.readthedocs.io/en/latest/#) – -[PyPI](https://pypi.org/project/typing-extensions/) - -## Overview - -The `typing_extensions` module serves two related purposes: - -- Enable use of new type system features on older Python versions. For example, - `typing.TypeGuard` is new in Python 3.10, but `typing_extensions` allows - users on previous Python versions to use it too. -- Enable experimentation with new type system PEPs before they are accepted and - added to the `typing` module. - -`typing_extensions` is treated specially by static type checkers such as -mypy and pyright. Objects defined in `typing_extensions` are treated the same -way as equivalent forms in `typing`. - -`typing_extensions` uses -[Semantic Versioning](https://semver.org/). The -major version will be incremented only for backwards-incompatible changes. -Therefore, it's safe to depend -on `typing_extensions` like this: `typing_extensions ~=x.y`, -where `x.y` is the first version that includes all features you need. -[This](https://packaging.python.org/en/latest/specifications/version-specifiers/#compatible-release) -is equivalent to `typing_extensions >=x.y, <(x+1)`. Do not depend on `~= x.y.z` -unless you really know what you're doing; that defeats the purpose of -semantic versioning. - -## Included items - -See [the documentation](https://typing-extensions.readthedocs.io/en/latest/#) for a -complete listing of module contents. - -## Contributing - -See [CONTRIBUTING.md](https://github.com/python/typing_extensions/blob/main/CONTRIBUTING.md) -for how to contribute to `typing_extensions`. - diff --git a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/RECORD b/functions/python/data-processor/typing_extensions-4.15.0.dist-info/RECORD deleted file mode 100644 index f262b3581..000000000 --- a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/RECORD +++ /dev/null @@ -1,7 +0,0 @@ -__pycache__/typing_extensions.cpython-314.pyc,, -typing_extensions-4.15.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -typing_extensions-4.15.0.dist-info/METADATA,sha256=wTg3j-jxiTSsmd4GBTXFPsbBOu7WXpTDJkHafuMZKnI,3259 -typing_extensions-4.15.0.dist-info/RECORD,, -typing_extensions-4.15.0.dist-info/WHEEL,sha256=G2gURzTEtmeR8nrdXUJfNiB3VYVxigPQ-bEQujpNiNs,82 -typing_extensions-4.15.0.dist-info/licenses/LICENSE,sha256=Oy-B_iHRgcSZxZolbI4ZaEVdZonSaaqFNzv7avQdo78,13936 -typing_extensions.py,sha256=Qz0R0XDTok0usGXrwb_oSM6n49fOaFZ6tSvqLUwvftg,160429 diff --git a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/WHEEL b/functions/python/data-processor/typing_extensions-4.15.0.dist-info/WHEEL deleted file mode 100644 index d8b9936da..000000000 --- a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: flit 3.12.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/licenses/LICENSE b/functions/python/data-processor/typing_extensions-4.15.0.dist-info/licenses/LICENSE deleted file mode 100644 index f26bcf4d2..000000000 --- a/functions/python/data-processor/typing_extensions-4.15.0.dist-info/licenses/LICENSE +++ /dev/null @@ -1,279 +0,0 @@ -A. HISTORY OF THE SOFTWARE -========================== - -Python was created in the early 1990s by Guido van Rossum at Stichting -Mathematisch Centrum (CWI, see https://www.cwi.nl) in the Netherlands -as a successor of a language called ABC. Guido remains Python's -principal author, although it includes many contributions from others. - -In 1995, Guido continued his work on Python at the Corporation for -National Research Initiatives (CNRI, see https://www.cnri.reston.va.us) -in Reston, Virginia where he released several versions of the -software. - -In May 2000, Guido and the Python core development team moved to -BeOpen.com to form the BeOpen PythonLabs team. In October of the same -year, the PythonLabs team moved to Digital Creations, which became -Zope Corporation. In 2001, the Python Software Foundation (PSF, see -https://www.python.org/psf/) was formed, a non-profit organization -created specifically to own Python-related Intellectual Property. -Zope Corporation was a sponsoring member of the PSF. - -All Python releases are Open Source (see https://opensource.org for -the Open Source Definition). Historically, most, but not all, Python -releases have also been GPL-compatible; the table below summarizes -the various releases. - - Release Derived Year Owner GPL- - from compatible? (1) - - 0.9.0 thru 1.2 1991-1995 CWI yes - 1.3 thru 1.5.2 1.2 1995-1999 CNRI yes - 1.6 1.5.2 2000 CNRI no - 2.0 1.6 2000 BeOpen.com no - 1.6.1 1.6 2001 CNRI yes (2) - 2.1 2.0+1.6.1 2001 PSF no - 2.0.1 2.0+1.6.1 2001 PSF yes - 2.1.1 2.1+2.0.1 2001 PSF yes - 2.1.2 2.1.1 2002 PSF yes - 2.1.3 2.1.2 2002 PSF yes - 2.2 and above 2.1.1 2001-now PSF yes - -Footnotes: - -(1) GPL-compatible doesn't mean that we're distributing Python under - the GPL. All Python licenses, unlike the GPL, let you distribute - a modified version without making your changes open source. The - GPL-compatible licenses make it possible to combine Python with - other software that is released under the GPL; the others don't. - -(2) According to Richard Stallman, 1.6.1 is not GPL-compatible, - because its license has a choice of law clause. According to - CNRI, however, Stallman's lawyer has told CNRI's lawyer that 1.6.1 - is "not incompatible" with the GPL. - -Thanks to the many outside volunteers who have worked under Guido's -direction to make these releases possible. - - -B. TERMS AND CONDITIONS FOR ACCESSING OR OTHERWISE USING PYTHON -=============================================================== - -Python software and documentation are licensed under the -Python Software Foundation License Version 2. - -Starting with Python 3.8.6, examples, recipes, and other code in -the documentation are dual licensed under the PSF License Version 2 -and the Zero-Clause BSD license. - -Some software incorporated into Python is under different licenses. -The licenses are listed with code falling under that license. - - -PYTHON SOFTWARE FOUNDATION LICENSE VERSION 2 --------------------------------------------- - -1. This LICENSE AGREEMENT is between the Python Software Foundation -("PSF"), and the Individual or Organization ("Licensee") accessing and -otherwise using this software ("Python") in source or binary form and -its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, PSF hereby -grants Licensee a nonexclusive, royalty-free, world-wide license to reproduce, -analyze, test, perform and/or display publicly, prepare derivative works, -distribute, and otherwise use Python alone or in any derivative version, -provided, however, that PSF's License Agreement and PSF's notice of copyright, -i.e., "Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009, 2010, -2011, 2012, 2013, 2014, 2015, 2016, 2017, 2018, 2019, 2020, 2021, 2022, 2023 Python Software Foundation; -All Rights Reserved" are retained in Python alone or in any derivative version -prepared by Licensee. - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python. - -4. PSF is making Python available to Licensee on an "AS IS" -basis. PSF MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, PSF MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. PSF SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. Nothing in this License Agreement shall be deemed to create any -relationship of agency, partnership, or joint venture between PSF and -Licensee. This License Agreement does not grant permission to use PSF -trademarks or trade name in a trademark sense to endorse or promote -products or services of Licensee, or any third party. - -8. By copying, installing or otherwise using Python, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -BEOPEN.COM LICENSE AGREEMENT FOR PYTHON 2.0 -------------------------------------------- - -BEOPEN PYTHON OPEN SOURCE LICENSE AGREEMENT VERSION 1 - -1. This LICENSE AGREEMENT is between BeOpen.com ("BeOpen"), having an -office at 160 Saratoga Avenue, Santa Clara, CA 95051, and the -Individual or Organization ("Licensee") accessing and otherwise using -this software in source or binary form and its associated -documentation ("the Software"). - -2. Subject to the terms and conditions of this BeOpen Python License -Agreement, BeOpen hereby grants Licensee a non-exclusive, -royalty-free, world-wide license to reproduce, analyze, test, perform -and/or display publicly, prepare derivative works, distribute, and -otherwise use the Software alone or in any derivative version, -provided, however, that the BeOpen Python License is retained in the -Software, alone or in any derivative version prepared by Licensee. - -3. BeOpen is making the Software available to Licensee on an "AS IS" -basis. BEOPEN MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, BEOPEN MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF THE SOFTWARE WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -4. BEOPEN SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF THE -SOFTWARE FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS -AS A RESULT OF USING, MODIFYING OR DISTRIBUTING THE SOFTWARE, OR ANY -DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -5. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -6. This License Agreement shall be governed by and interpreted in all -respects by the law of the State of California, excluding conflict of -law provisions. Nothing in this License Agreement shall be deemed to -create any relationship of agency, partnership, or joint venture -between BeOpen and Licensee. This License Agreement does not grant -permission to use BeOpen trademarks or trade names in a trademark -sense to endorse or promote products or services of Licensee, or any -third party. As an exception, the "BeOpen Python" logos available at -http://www.pythonlabs.com/logos.html may be used according to the -permissions granted on that web page. - -7. By copying, installing or otherwise using the software, Licensee -agrees to be bound by the terms and conditions of this License -Agreement. - - -CNRI LICENSE AGREEMENT FOR PYTHON 1.6.1 ---------------------------------------- - -1. This LICENSE AGREEMENT is between the Corporation for National -Research Initiatives, having an office at 1895 Preston White Drive, -Reston, VA 20191 ("CNRI"), and the Individual or Organization -("Licensee") accessing and otherwise using Python 1.6.1 software in -source or binary form and its associated documentation. - -2. Subject to the terms and conditions of this License Agreement, CNRI -hereby grants Licensee a nonexclusive, royalty-free, world-wide -license to reproduce, analyze, test, perform and/or display publicly, -prepare derivative works, distribute, and otherwise use Python 1.6.1 -alone or in any derivative version, provided, however, that CNRI's -License Agreement and CNRI's notice of copyright, i.e., "Copyright (c) -1995-2001 Corporation for National Research Initiatives; All Rights -Reserved" are retained in Python 1.6.1 alone or in any derivative -version prepared by Licensee. Alternately, in lieu of CNRI's License -Agreement, Licensee may substitute the following text (omitting the -quotes): "Python 1.6.1 is made available subject to the terms and -conditions in CNRI's License Agreement. This Agreement together with -Python 1.6.1 may be located on the internet using the following -unique, persistent identifier (known as a handle): 1895.22/1013. This -Agreement may also be obtained from a proxy server on the internet -using the following URL: http://hdl.handle.net/1895.22/1013". - -3. In the event Licensee prepares a derivative work that is based on -or incorporates Python 1.6.1 or any part thereof, and wants to make -the derivative work available to others as provided herein, then -Licensee hereby agrees to include in any such work a brief summary of -the changes made to Python 1.6.1. - -4. CNRI is making Python 1.6.1 available to Licensee on an "AS IS" -basis. CNRI MAKES NO REPRESENTATIONS OR WARRANTIES, EXPRESS OR -IMPLIED. BY WAY OF EXAMPLE, BUT NOT LIMITATION, CNRI MAKES NO AND -DISCLAIMS ANY REPRESENTATION OR WARRANTY OF MERCHANTABILITY OR FITNESS -FOR ANY PARTICULAR PURPOSE OR THAT THE USE OF PYTHON 1.6.1 WILL NOT -INFRINGE ANY THIRD PARTY RIGHTS. - -5. CNRI SHALL NOT BE LIABLE TO LICENSEE OR ANY OTHER USERS OF PYTHON -1.6.1 FOR ANY INCIDENTAL, SPECIAL, OR CONSEQUENTIAL DAMAGES OR LOSS AS -A RESULT OF MODIFYING, DISTRIBUTING, OR OTHERWISE USING PYTHON 1.6.1, -OR ANY DERIVATIVE THEREOF, EVEN IF ADVISED OF THE POSSIBILITY THEREOF. - -6. This License Agreement will automatically terminate upon a material -breach of its terms and conditions. - -7. This License Agreement shall be governed by the federal -intellectual property law of the United States, including without -limitation the federal copyright law, and, to the extent such -U.S. federal law does not apply, by the law of the Commonwealth of -Virginia, excluding Virginia's conflict of law provisions. -Notwithstanding the foregoing, with regard to derivative works based -on Python 1.6.1 that incorporate non-separable material that was -previously distributed under the GNU General Public License (GPL), the -law of the Commonwealth of Virginia shall govern this License -Agreement only as to issues arising under or with respect to -Paragraphs 4, 5, and 7 of this License Agreement. Nothing in this -License Agreement shall be deemed to create any relationship of -agency, partnership, or joint venture between CNRI and Licensee. This -License Agreement does not grant permission to use CNRI trademarks or -trade name in a trademark sense to endorse or promote products or -services of Licensee, or any third party. - -8. By clicking on the "ACCEPT" button where indicated, or by copying, -installing or otherwise using Python 1.6.1, Licensee agrees to be -bound by the terms and conditions of this License Agreement. - - ACCEPT - - -CWI LICENSE AGREEMENT FOR PYTHON 0.9.0 THROUGH 1.2 --------------------------------------------------- - -Copyright (c) 1991 - 1995, Stichting Mathematisch Centrum Amsterdam, -The Netherlands. All rights reserved. - -Permission to use, copy, modify, and distribute this software and its -documentation for any purpose and without fee is hereby granted, -provided that the above copyright notice appear in all copies and that -both that copyright notice and this permission notice appear in -supporting documentation, and that the name of Stichting Mathematisch -Centrum or CWI not be used in advertising or publicity pertaining to -distribution of the software without specific, written prior -permission. - -STICHTING MATHEMATISCH CENTRUM DISCLAIMS ALL WARRANTIES WITH REGARD TO -THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND -FITNESS, IN NO EVENT SHALL STICHTING MATHEMATISCH CENTRUM BE LIABLE -FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES -WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN -ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT -OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. - -ZERO-CLAUSE BSD LICENSE FOR CODE IN THE PYTHON DOCUMENTATION ----------------------------------------------------------------------- - -Permission to use, copy, modify, and/or distribute this software for any -purpose with or without fee is hereby granted. - -THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH -REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY -AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY SPECIAL, DIRECT, -INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM -LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR -OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR -PERFORMANCE OF THIS SOFTWARE. diff --git a/functions/python/data-processor/typing_extensions.py b/functions/python/data-processor/typing_extensions.py deleted file mode 100644 index 77f33e161..000000000 --- a/functions/python/data-processor/typing_extensions.py +++ /dev/null @@ -1,4317 +0,0 @@ -import abc -import builtins -import collections -import collections.abc -import contextlib -import enum -import functools -import inspect -import io -import keyword -import operator -import sys -import types as _types -import typing -import warnings - -# Breakpoint: https://github.com/python/cpython/pull/119891 -if sys.version_info >= (3, 14): - import annotationlib - -__all__ = [ - # Super-special typing primitives. - 'Any', - 'ClassVar', - 'Concatenate', - 'Final', - 'LiteralString', - 'ParamSpec', - 'ParamSpecArgs', - 'ParamSpecKwargs', - 'Self', - 'Type', - 'TypeVar', - 'TypeVarTuple', - 'Unpack', - - # ABCs (from collections.abc). - 'Awaitable', - 'AsyncIterator', - 'AsyncIterable', - 'Coroutine', - 'AsyncGenerator', - 'AsyncContextManager', - 'Buffer', - 'ChainMap', - - # Concrete collection types. - 'ContextManager', - 'Counter', - 'Deque', - 'DefaultDict', - 'NamedTuple', - 'OrderedDict', - 'TypedDict', - - # Structural checks, a.k.a. protocols. - 'SupportsAbs', - 'SupportsBytes', - 'SupportsComplex', - 'SupportsFloat', - 'SupportsIndex', - 'SupportsInt', - 'SupportsRound', - 'Reader', - 'Writer', - - # One-off things. - 'Annotated', - 'assert_never', - 'assert_type', - 'clear_overloads', - 'dataclass_transform', - 'deprecated', - 'disjoint_base', - 'Doc', - 'evaluate_forward_ref', - 'get_overloads', - 'final', - 'Format', - 'get_annotations', - 'get_args', - 'get_origin', - 'get_original_bases', - 'get_protocol_members', - 'get_type_hints', - 'IntVar', - 'is_protocol', - 'is_typeddict', - 'Literal', - 'NewType', - 'overload', - 'override', - 'Protocol', - 'Sentinel', - 'reveal_type', - 'runtime', - 'runtime_checkable', - 'Text', - 'TypeAlias', - 'TypeAliasType', - 'TypeForm', - 'TypeGuard', - 'TypeIs', - 'TYPE_CHECKING', - 'type_repr', - 'Never', - 'NoReturn', - 'ReadOnly', - 'Required', - 'NotRequired', - 'NoDefault', - 'NoExtraItems', - - # Pure aliases, have always been in typing - 'AbstractSet', - 'AnyStr', - 'BinaryIO', - 'Callable', - 'Collection', - 'Container', - 'Dict', - 'ForwardRef', - 'FrozenSet', - 'Generator', - 'Generic', - 'Hashable', - 'IO', - 'ItemsView', - 'Iterable', - 'Iterator', - 'KeysView', - 'List', - 'Mapping', - 'MappingView', - 'Match', - 'MutableMapping', - 'MutableSequence', - 'MutableSet', - 'Optional', - 'Pattern', - 'Reversible', - 'Sequence', - 'Set', - 'Sized', - 'TextIO', - 'Tuple', - 'Union', - 'ValuesView', - 'cast', - 'no_type_check', - 'no_type_check_decorator', -] - -# for backward compatibility -PEP_560 = True -GenericMeta = type -# Breakpoint: https://github.com/python/cpython/pull/116129 -_PEP_696_IMPLEMENTED = sys.version_info >= (3, 13, 0, "beta") - -# Added with bpo-45166 to 3.10.1+ and some 3.9 versions -_FORWARD_REF_HAS_CLASS = "__forward_is_class__" in typing.ForwardRef.__slots__ - -# The functions below are modified copies of typing internal helpers. -# They are needed by _ProtocolMeta and they provide support for PEP 646. - - -class _Sentinel: - def __repr__(self): - return "" - - -_marker = _Sentinel() - - -# Breakpoint: https://github.com/python/cpython/pull/27342 -if sys.version_info >= (3, 10): - def _should_collect_from_parameters(t): - return isinstance( - t, (typing._GenericAlias, _types.GenericAlias, _types.UnionType) - ) -else: - def _should_collect_from_parameters(t): - return isinstance(t, (typing._GenericAlias, _types.GenericAlias)) - - -NoReturn = typing.NoReturn - -# Some unconstrained type variables. These are used by the container types. -# (These are not for export.) -T = typing.TypeVar('T') # Any type. -KT = typing.TypeVar('KT') # Key type. -VT = typing.TypeVar('VT') # Value type. -T_co = typing.TypeVar('T_co', covariant=True) # Any type covariant containers. -T_contra = typing.TypeVar('T_contra', contravariant=True) # Ditto contravariant. - - -# Breakpoint: https://github.com/python/cpython/pull/31841 -if sys.version_info >= (3, 11): - from typing import Any -else: - - class _AnyMeta(type): - def __instancecheck__(self, obj): - if self is Any: - raise TypeError("typing_extensions.Any cannot be used with isinstance()") - return super().__instancecheck__(obj) - - def __repr__(self): - if self is Any: - return "typing_extensions.Any" - return super().__repr__() - - class Any(metaclass=_AnyMeta): - """Special type indicating an unconstrained type. - - Any is compatible with every type. - - Any assumed to have all methods. - - All values assumed to be instances of Any. - Note that all the above statements are true from the point of view of - static type checkers. At runtime, Any should not be used with instance - checks. - """ - def __new__(cls, *args, **kwargs): - if cls is Any: - raise TypeError("Any cannot be instantiated") - return super().__new__(cls, *args, **kwargs) - - -ClassVar = typing.ClassVar - -# Vendored from cpython typing._SpecialFrom -# Having a separate class means that instances will not be rejected by -# typing._type_check. -class _SpecialForm(typing._Final, _root=True): - __slots__ = ('_name', '__doc__', '_getitem') - - def __init__(self, getitem): - self._getitem = getitem - self._name = getitem.__name__ - self.__doc__ = getitem.__doc__ - - def __getattr__(self, item): - if item in {'__name__', '__qualname__'}: - return self._name - - raise AttributeError(item) - - def __mro_entries__(self, bases): - raise TypeError(f"Cannot subclass {self!r}") - - def __repr__(self): - return f'typing_extensions.{self._name}' - - def __reduce__(self): - return self._name - - def __call__(self, *args, **kwds): - raise TypeError(f"Cannot instantiate {self!r}") - - def __or__(self, other): - return typing.Union[self, other] - - def __ror__(self, other): - return typing.Union[other, self] - - def __instancecheck__(self, obj): - raise TypeError(f"{self} cannot be used with isinstance()") - - def __subclasscheck__(self, cls): - raise TypeError(f"{self} cannot be used with issubclass()") - - @typing._tp_cache - def __getitem__(self, parameters): - return self._getitem(self, parameters) - - -# Note that inheriting from this class means that the object will be -# rejected by typing._type_check, so do not use it if the special form -# is arguably valid as a type by itself. -class _ExtensionsSpecialForm(typing._SpecialForm, _root=True): - def __repr__(self): - return 'typing_extensions.' + self._name - - -Final = typing.Final - -# Breakpoint: https://github.com/python/cpython/pull/30530 -if sys.version_info >= (3, 11): - final = typing.final -else: - # @final exists in 3.8+, but we backport it for all versions - # before 3.11 to keep support for the __final__ attribute. - # See https://bugs.python.org/issue46342 - def final(f): - """This decorator can be used to indicate to type checkers that - the decorated method cannot be overridden, and decorated class - cannot be subclassed. For example: - - class Base: - @final - def done(self) -> None: - ... - class Sub(Base): - def done(self) -> None: # Error reported by type checker - ... - @final - class Leaf: - ... - class Other(Leaf): # Error reported by type checker - ... - - There is no runtime checking of these properties. The decorator - sets the ``__final__`` attribute to ``True`` on the decorated object - to allow runtime introspection. - """ - try: - f.__final__ = True - except (AttributeError, TypeError): - # Skip the attribute silently if it is not writable. - # AttributeError happens if the object has __slots__ or a - # read-only property, TypeError if it's a builtin class. - pass - return f - - -if hasattr(typing, "disjoint_base"): # 3.15 - disjoint_base = typing.disjoint_base -else: - def disjoint_base(cls): - """This decorator marks a class as a disjoint base. - - Child classes of a disjoint base cannot inherit from other disjoint bases that are - not parent classes of the disjoint base. - - For example: - - @disjoint_base - class Disjoint1: pass - - @disjoint_base - class Disjoint2: pass - - class Disjoint3(Disjoint1, Disjoint2): pass # Type checker error - - Type checkers can use knowledge of disjoint bases to detect unreachable code - and determine when two types can overlap. - - See PEP 800.""" - cls.__disjoint_base__ = True - return cls - - -def IntVar(name): - return typing.TypeVar(name) - - -# A Literal bug was fixed in 3.11.0, 3.10.1 and 3.9.8 -# Breakpoint: https://github.com/python/cpython/pull/29334 -if sys.version_info >= (3, 10, 1): - Literal = typing.Literal -else: - def _flatten_literal_params(parameters): - """An internal helper for Literal creation: flatten Literals among parameters""" - params = [] - for p in parameters: - if isinstance(p, _LiteralGenericAlias): - params.extend(p.__args__) - else: - params.append(p) - return tuple(params) - - def _value_and_type_iter(params): - for p in params: - yield p, type(p) - - class _LiteralGenericAlias(typing._GenericAlias, _root=True): - def __eq__(self, other): - if not isinstance(other, _LiteralGenericAlias): - return NotImplemented - these_args_deduped = set(_value_and_type_iter(self.__args__)) - other_args_deduped = set(_value_and_type_iter(other.__args__)) - return these_args_deduped == other_args_deduped - - def __hash__(self): - return hash(frozenset(_value_and_type_iter(self.__args__))) - - class _LiteralForm(_ExtensionsSpecialForm, _root=True): - def __init__(self, doc: str): - self._name = 'Literal' - self._doc = self.__doc__ = doc - - def __getitem__(self, parameters): - if not isinstance(parameters, tuple): - parameters = (parameters,) - - parameters = _flatten_literal_params(parameters) - - val_type_pairs = list(_value_and_type_iter(parameters)) - try: - deduped_pairs = set(val_type_pairs) - except TypeError: - # unhashable parameters - pass - else: - # similar logic to typing._deduplicate on Python 3.9+ - if len(deduped_pairs) < len(val_type_pairs): - new_parameters = [] - for pair in val_type_pairs: - if pair in deduped_pairs: - new_parameters.append(pair[0]) - deduped_pairs.remove(pair) - assert not deduped_pairs, deduped_pairs - parameters = tuple(new_parameters) - - return _LiteralGenericAlias(self, parameters) - - Literal = _LiteralForm(doc="""\ - A type that can be used to indicate to type checkers - that the corresponding value has a value literally equivalent - to the provided parameter. For example: - - var: Literal[4] = 4 - - The type checker understands that 'var' is literally equal to - the value 4 and no other value. - - Literal[...] cannot be subclassed. There is no runtime - checking verifying that the parameter is actually a value - instead of a type.""") - - -_overload_dummy = typing._overload_dummy - - -if hasattr(typing, "get_overloads"): # 3.11+ - overload = typing.overload - get_overloads = typing.get_overloads - clear_overloads = typing.clear_overloads -else: - # {module: {qualname: {firstlineno: func}}} - _overload_registry = collections.defaultdict( - functools.partial(collections.defaultdict, dict) - ) - - def overload(func): - """Decorator for overloaded functions/methods. - - In a stub file, place two or more stub definitions for the same - function in a row, each decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - - In a non-stub file (i.e. a regular .py file), do the same but - follow it with an implementation. The implementation should *not* - be decorated with @overload. For example: - - @overload - def utf8(value: None) -> None: ... - @overload - def utf8(value: bytes) -> bytes: ... - @overload - def utf8(value: str) -> bytes: ... - def utf8(value): - # implementation goes here - - The overloads for a function can be retrieved at runtime using the - get_overloads() function. - """ - # classmethod and staticmethod - f = getattr(func, "__func__", func) - try: - _overload_registry[f.__module__][f.__qualname__][ - f.__code__.co_firstlineno - ] = func - except AttributeError: - # Not a normal function; ignore. - pass - return _overload_dummy - - def get_overloads(func): - """Return all defined overloads for *func* as a sequence.""" - # classmethod and staticmethod - f = getattr(func, "__func__", func) - if f.__module__ not in _overload_registry: - return [] - mod_dict = _overload_registry[f.__module__] - if f.__qualname__ not in mod_dict: - return [] - return list(mod_dict[f.__qualname__].values()) - - def clear_overloads(): - """Clear all overloads in the registry.""" - _overload_registry.clear() - - -# This is not a real generic class. Don't use outside annotations. -Type = typing.Type - -# Various ABCs mimicking those in collections.abc. -# A few are simply re-exported for completeness. -Awaitable = typing.Awaitable -Coroutine = typing.Coroutine -AsyncIterable = typing.AsyncIterable -AsyncIterator = typing.AsyncIterator -Deque = typing.Deque -DefaultDict = typing.DefaultDict -OrderedDict = typing.OrderedDict -Counter = typing.Counter -ChainMap = typing.ChainMap -Text = typing.Text -TYPE_CHECKING = typing.TYPE_CHECKING - - -# Breakpoint: https://github.com/python/cpython/pull/118681 -if sys.version_info >= (3, 13, 0, "beta"): - from typing import AsyncContextManager, AsyncGenerator, ContextManager, Generator -else: - def _is_dunder(attr): - return attr.startswith('__') and attr.endswith('__') - - - class _SpecialGenericAlias(typing._SpecialGenericAlias, _root=True): - def __init__(self, origin, nparams, *, inst=True, name=None, defaults=()): - super().__init__(origin, nparams, inst=inst, name=name) - self._defaults = defaults - - def __setattr__(self, attr, val): - allowed_attrs = {'_name', '_inst', '_nparams', '_defaults'} - if _is_dunder(attr) or attr in allowed_attrs: - object.__setattr__(self, attr, val) - else: - setattr(self.__origin__, attr, val) - - @typing._tp_cache - def __getitem__(self, params): - if not isinstance(params, tuple): - params = (params,) - msg = "Parameters to generic types must be types." - params = tuple(typing._type_check(p, msg) for p in params) - if ( - self._defaults - and len(params) < self._nparams - and len(params) + len(self._defaults) >= self._nparams - ): - params = (*params, *self._defaults[len(params) - self._nparams:]) - actual_len = len(params) - - if actual_len != self._nparams: - if self._defaults: - expected = f"at least {self._nparams - len(self._defaults)}" - else: - expected = str(self._nparams) - if not self._nparams: - raise TypeError(f"{self} is not a generic class") - raise TypeError( - f"Too {'many' if actual_len > self._nparams else 'few'}" - f" arguments for {self};" - f" actual {actual_len}, expected {expected}" - ) - return self.copy_with(params) - - _NoneType = type(None) - Generator = _SpecialGenericAlias( - collections.abc.Generator, 3, defaults=(_NoneType, _NoneType) - ) - AsyncGenerator = _SpecialGenericAlias( - collections.abc.AsyncGenerator, 2, defaults=(_NoneType,) - ) - ContextManager = _SpecialGenericAlias( - contextlib.AbstractContextManager, - 2, - name="ContextManager", - defaults=(typing.Optional[bool],) - ) - AsyncContextManager = _SpecialGenericAlias( - contextlib.AbstractAsyncContextManager, - 2, - name="AsyncContextManager", - defaults=(typing.Optional[bool],) - ) - - -_PROTO_ALLOWLIST = { - 'collections.abc': [ - 'Callable', 'Awaitable', 'Iterable', 'Iterator', 'AsyncIterable', - 'Hashable', 'Sized', 'Container', 'Collection', 'Reversible', 'Buffer', - ], - 'contextlib': ['AbstractContextManager', 'AbstractAsyncContextManager'], - 'typing_extensions': ['Buffer'], -} - - -_EXCLUDED_ATTRS = frozenset(typing.EXCLUDED_ATTRIBUTES) | { - "__match_args__", "__protocol_attrs__", "__non_callable_proto_members__", - "__final__", -} - - -def _get_protocol_attrs(cls): - attrs = set() - for base in cls.__mro__[:-1]: # without object - if base.__name__ in {'Protocol', 'Generic'}: - continue - annotations = getattr(base, '__annotations__', {}) - for attr in (*base.__dict__, *annotations): - if (not attr.startswith('_abc_') and attr not in _EXCLUDED_ATTRS): - attrs.add(attr) - return attrs - - -def _caller(depth=1, default='__main__'): - try: - return sys._getframemodulename(depth + 1) or default - except AttributeError: # For platforms without _getframemodulename() - pass - try: - return sys._getframe(depth + 1).f_globals.get('__name__', default) - except (AttributeError, ValueError): # For platforms without _getframe() - pass - return None - - -# `__match_args__` attribute was removed from protocol members in 3.13, -# we want to backport this change to older Python versions. -# Breakpoint: https://github.com/python/cpython/pull/110683 -if sys.version_info >= (3, 13): - Protocol = typing.Protocol -else: - def _allow_reckless_class_checks(depth=2): - """Allow instance and class checks for special stdlib modules. - The abc and functools modules indiscriminately call isinstance() and - issubclass() on the whole MRO of a user class, which may contain protocols. - """ - return _caller(depth) in {'abc', 'functools', None} - - def _no_init(self, *args, **kwargs): - if type(self)._is_protocol: - raise TypeError('Protocols cannot be instantiated') - - def _type_check_issubclass_arg_1(arg): - """Raise TypeError if `arg` is not an instance of `type` - in `issubclass(arg, )`. - - In most cases, this is verified by type.__subclasscheck__. - Checking it again unnecessarily would slow down issubclass() checks, - so, we don't perform this check unless we absolutely have to. - - For various error paths, however, - we want to ensure that *this* error message is shown to the user - where relevant, rather than a typing.py-specific error message. - """ - if not isinstance(arg, type): - # Same error message as for issubclass(1, int). - raise TypeError('issubclass() arg 1 must be a class') - - # Inheriting from typing._ProtocolMeta isn't actually desirable, - # but is necessary to allow typing.Protocol and typing_extensions.Protocol - # to mix without getting TypeErrors about "metaclass conflict" - class _ProtocolMeta(type(typing.Protocol)): - # This metaclass is somewhat unfortunate, - # but is necessary for several reasons... - # - # NOTE: DO NOT call super() in any methods in this class - # That would call the methods on typing._ProtocolMeta on Python <=3.11 - # and those are slow - def __new__(mcls, name, bases, namespace, **kwargs): - if name == "Protocol" and len(bases) < 2: - pass - elif {Protocol, typing.Protocol} & set(bases): - for base in bases: - if not ( - base in {object, typing.Generic, Protocol, typing.Protocol} - or base.__name__ in _PROTO_ALLOWLIST.get(base.__module__, []) - or is_protocol(base) - ): - raise TypeError( - f"Protocols can only inherit from other protocols, " - f"got {base!r}" - ) - return abc.ABCMeta.__new__(mcls, name, bases, namespace, **kwargs) - - def __init__(cls, *args, **kwargs): - abc.ABCMeta.__init__(cls, *args, **kwargs) - if getattr(cls, "_is_protocol", False): - cls.__protocol_attrs__ = _get_protocol_attrs(cls) - - def __subclasscheck__(cls, other): - if cls is Protocol: - return type.__subclasscheck__(cls, other) - if ( - getattr(cls, '_is_protocol', False) - and not _allow_reckless_class_checks() - ): - if not getattr(cls, '_is_runtime_protocol', False): - _type_check_issubclass_arg_1(other) - raise TypeError( - "Instance and class checks can only be used with " - "@runtime_checkable protocols" - ) - if ( - # this attribute is set by @runtime_checkable: - cls.__non_callable_proto_members__ - and cls.__dict__.get("__subclasshook__") is _proto_hook - ): - _type_check_issubclass_arg_1(other) - non_method_attrs = sorted(cls.__non_callable_proto_members__) - raise TypeError( - "Protocols with non-method members don't support issubclass()." - f" Non-method members: {str(non_method_attrs)[1:-1]}." - ) - return abc.ABCMeta.__subclasscheck__(cls, other) - - def __instancecheck__(cls, instance): - # We need this method for situations where attributes are - # assigned in __init__. - if cls is Protocol: - return type.__instancecheck__(cls, instance) - if not getattr(cls, "_is_protocol", False): - # i.e., it's a concrete subclass of a protocol - return abc.ABCMeta.__instancecheck__(cls, instance) - - if ( - not getattr(cls, '_is_runtime_protocol', False) and - not _allow_reckless_class_checks() - ): - raise TypeError("Instance and class checks can only be used with" - " @runtime_checkable protocols") - - if abc.ABCMeta.__instancecheck__(cls, instance): - return True - - for attr in cls.__protocol_attrs__: - try: - val = inspect.getattr_static(instance, attr) - except AttributeError: - break - # this attribute is set by @runtime_checkable: - if val is None and attr not in cls.__non_callable_proto_members__: - break - else: - return True - - return False - - def __eq__(cls, other): - # Hack so that typing.Generic.__class_getitem__ - # treats typing_extensions.Protocol - # as equivalent to typing.Protocol - if abc.ABCMeta.__eq__(cls, other) is True: - return True - return cls is Protocol and other is typing.Protocol - - # This has to be defined, or the abc-module cache - # complains about classes with this metaclass being unhashable, - # if we define only __eq__! - def __hash__(cls) -> int: - return type.__hash__(cls) - - @classmethod - def _proto_hook(cls, other): - if not cls.__dict__.get('_is_protocol', False): - return NotImplemented - - for attr in cls.__protocol_attrs__: - for base in other.__mro__: - # Check if the members appears in the class dictionary... - if attr in base.__dict__: - if base.__dict__[attr] is None: - return NotImplemented - break - - # ...or in annotations, if it is a sub-protocol. - annotations = getattr(base, '__annotations__', {}) - if ( - isinstance(annotations, collections.abc.Mapping) - and attr in annotations - and is_protocol(other) - ): - break - else: - return NotImplemented - return True - - class Protocol(typing.Generic, metaclass=_ProtocolMeta): - __doc__ = typing.Protocol.__doc__ - __slots__ = () - _is_protocol = True - _is_runtime_protocol = False - - def __init_subclass__(cls, *args, **kwargs): - super().__init_subclass__(*args, **kwargs) - - # Determine if this is a protocol or a concrete subclass. - if not cls.__dict__.get('_is_protocol', False): - cls._is_protocol = any(b is Protocol for b in cls.__bases__) - - # Set (or override) the protocol subclass hook. - if '__subclasshook__' not in cls.__dict__: - cls.__subclasshook__ = _proto_hook - - # Prohibit instantiation for protocol classes - if cls._is_protocol and cls.__init__ is Protocol.__init__: - cls.__init__ = _no_init - - -# Breakpoint: https://github.com/python/cpython/pull/113401 -if sys.version_info >= (3, 13): - runtime_checkable = typing.runtime_checkable -else: - def runtime_checkable(cls): - """Mark a protocol class as a runtime protocol. - - Such protocol can be used with isinstance() and issubclass(). - Raise TypeError if applied to a non-protocol class. - This allows a simple-minded structural check very similar to - one trick ponies in collections.abc such as Iterable. - - For example:: - - @runtime_checkable - class Closable(Protocol): - def close(self): ... - - assert isinstance(open('/some/file'), Closable) - - Warning: this will check only the presence of the required methods, - not their type signatures! - """ - if not issubclass(cls, typing.Generic) or not getattr(cls, '_is_protocol', False): - raise TypeError(f'@runtime_checkable can be only applied to protocol classes,' - f' got {cls!r}') - cls._is_runtime_protocol = True - - # typing.Protocol classes on <=3.11 break if we execute this block, - # because typing.Protocol classes on <=3.11 don't have a - # `__protocol_attrs__` attribute, and this block relies on the - # `__protocol_attrs__` attribute. Meanwhile, typing.Protocol classes on 3.12.2+ - # break if we *don't* execute this block, because *they* assume that all - # protocol classes have a `__non_callable_proto_members__` attribute - # (which this block sets) - if isinstance(cls, _ProtocolMeta) or sys.version_info >= (3, 12, 2): - # PEP 544 prohibits using issubclass() - # with protocols that have non-method members. - # See gh-113320 for why we compute this attribute here, - # rather than in `_ProtocolMeta.__init__` - cls.__non_callable_proto_members__ = set() - for attr in cls.__protocol_attrs__: - try: - is_callable = callable(getattr(cls, attr, None)) - except Exception as e: - raise TypeError( - f"Failed to determine whether protocol member {attr!r} " - "is a method member" - ) from e - else: - if not is_callable: - cls.__non_callable_proto_members__.add(attr) - - return cls - - -# The "runtime" alias exists for backwards compatibility. -runtime = runtime_checkable - - -# Our version of runtime-checkable protocols is faster on Python <=3.11 -# Breakpoint: https://github.com/python/cpython/pull/112717 -if sys.version_info >= (3, 12): - SupportsInt = typing.SupportsInt - SupportsFloat = typing.SupportsFloat - SupportsComplex = typing.SupportsComplex - SupportsBytes = typing.SupportsBytes - SupportsIndex = typing.SupportsIndex - SupportsAbs = typing.SupportsAbs - SupportsRound = typing.SupportsRound -else: - @runtime_checkable - class SupportsInt(Protocol): - """An ABC with one abstract method __int__.""" - __slots__ = () - - @abc.abstractmethod - def __int__(self) -> int: - pass - - @runtime_checkable - class SupportsFloat(Protocol): - """An ABC with one abstract method __float__.""" - __slots__ = () - - @abc.abstractmethod - def __float__(self) -> float: - pass - - @runtime_checkable - class SupportsComplex(Protocol): - """An ABC with one abstract method __complex__.""" - __slots__ = () - - @abc.abstractmethod - def __complex__(self) -> complex: - pass - - @runtime_checkable - class SupportsBytes(Protocol): - """An ABC with one abstract method __bytes__.""" - __slots__ = () - - @abc.abstractmethod - def __bytes__(self) -> bytes: - pass - - @runtime_checkable - class SupportsIndex(Protocol): - __slots__ = () - - @abc.abstractmethod - def __index__(self) -> int: - pass - - @runtime_checkable - class SupportsAbs(Protocol[T_co]): - """ - An ABC with one abstract method __abs__ that is covariant in its return type. - """ - __slots__ = () - - @abc.abstractmethod - def __abs__(self) -> T_co: - pass - - @runtime_checkable - class SupportsRound(Protocol[T_co]): - """ - An ABC with one abstract method __round__ that is covariant in its return type. - """ - __slots__ = () - - @abc.abstractmethod - def __round__(self, ndigits: int = 0) -> T_co: - pass - - -if hasattr(io, "Reader") and hasattr(io, "Writer"): - Reader = io.Reader - Writer = io.Writer -else: - @runtime_checkable - class Reader(Protocol[T_co]): - """Protocol for simple I/O reader instances. - - This protocol only supports blocking I/O. - """ - - __slots__ = () - - @abc.abstractmethod - def read(self, size: int = ..., /) -> T_co: - """Read data from the input stream and return it. - - If *size* is specified, at most *size* items (bytes/characters) will be - read. - """ - - @runtime_checkable - class Writer(Protocol[T_contra]): - """Protocol for simple I/O writer instances. - - This protocol only supports blocking I/O. - """ - - __slots__ = () - - @abc.abstractmethod - def write(self, data: T_contra, /) -> int: - """Write *data* to the output stream and return the number of items written.""" # noqa: E501 - - -_NEEDS_SINGLETONMETA = ( - not hasattr(typing, "NoDefault") or not hasattr(typing, "NoExtraItems") -) - -if _NEEDS_SINGLETONMETA: - class SingletonMeta(type): - def __setattr__(cls, attr, value): - # TypeError is consistent with the behavior of NoneType - raise TypeError( - f"cannot set {attr!r} attribute of immutable type {cls.__name__!r}" - ) - - -if hasattr(typing, "NoDefault"): - NoDefault = typing.NoDefault -else: - class NoDefaultType(metaclass=SingletonMeta): - """The type of the NoDefault singleton.""" - - __slots__ = () - - def __new__(cls): - return globals().get("NoDefault") or object.__new__(cls) - - def __repr__(self): - return "typing_extensions.NoDefault" - - def __reduce__(self): - return "NoDefault" - - NoDefault = NoDefaultType() - del NoDefaultType - -if hasattr(typing, "NoExtraItems"): - NoExtraItems = typing.NoExtraItems -else: - class NoExtraItemsType(metaclass=SingletonMeta): - """The type of the NoExtraItems singleton.""" - - __slots__ = () - - def __new__(cls): - return globals().get("NoExtraItems") or object.__new__(cls) - - def __repr__(self): - return "typing_extensions.NoExtraItems" - - def __reduce__(self): - return "NoExtraItems" - - NoExtraItems = NoExtraItemsType() - del NoExtraItemsType - -if _NEEDS_SINGLETONMETA: - del SingletonMeta - - -# Update this to something like >=3.13.0b1 if and when -# PEP 728 is implemented in CPython -_PEP_728_IMPLEMENTED = False - -if _PEP_728_IMPLEMENTED: - # The standard library TypedDict in Python 3.9.0/1 does not honour the "total" - # keyword with old-style TypedDict(). See https://bugs.python.org/issue42059 - # The standard library TypedDict below Python 3.11 does not store runtime - # information about optional and required keys when using Required or NotRequired. - # Generic TypedDicts are also impossible using typing.TypedDict on Python <3.11. - # Aaaand on 3.12 we add __orig_bases__ to TypedDict - # to enable better runtime introspection. - # On 3.13 we deprecate some odd ways of creating TypedDicts. - # Also on 3.13, PEP 705 adds the ReadOnly[] qualifier. - # PEP 728 (still pending) makes more changes. - TypedDict = typing.TypedDict - _TypedDictMeta = typing._TypedDictMeta - is_typeddict = typing.is_typeddict -else: - # 3.10.0 and later - _TAKES_MODULE = "module" in inspect.signature(typing._type_check).parameters - - def _get_typeddict_qualifiers(annotation_type): - while True: - annotation_origin = get_origin(annotation_type) - if annotation_origin is Annotated: - annotation_args = get_args(annotation_type) - if annotation_args: - annotation_type = annotation_args[0] - else: - break - elif annotation_origin is Required: - yield Required - annotation_type, = get_args(annotation_type) - elif annotation_origin is NotRequired: - yield NotRequired - annotation_type, = get_args(annotation_type) - elif annotation_origin is ReadOnly: - yield ReadOnly - annotation_type, = get_args(annotation_type) - else: - break - - class _TypedDictMeta(type): - - def __new__(cls, name, bases, ns, *, total=True, closed=None, - extra_items=NoExtraItems): - """Create new typed dict class object. - - This method is called when TypedDict is subclassed, - or when TypedDict is instantiated. This way - TypedDict supports all three syntax forms described in its docstring. - Subclasses and instances of TypedDict return actual dictionaries. - """ - for base in bases: - if type(base) is not _TypedDictMeta and base is not typing.Generic: - raise TypeError('cannot inherit from both a TypedDict type ' - 'and a non-TypedDict base class') - if closed is not None and extra_items is not NoExtraItems: - raise TypeError(f"Cannot combine closed={closed!r} and extra_items") - - if any(issubclass(b, typing.Generic) for b in bases): - generic_base = (typing.Generic,) - else: - generic_base = () - - ns_annotations = ns.pop('__annotations__', None) - - # typing.py generally doesn't let you inherit from plain Generic, unless - # the name of the class happens to be "Protocol" - tp_dict = type.__new__(_TypedDictMeta, "Protocol", (*generic_base, dict), ns) - tp_dict.__name__ = name - if tp_dict.__qualname__ == "Protocol": - tp_dict.__qualname__ = name - - if not hasattr(tp_dict, '__orig_bases__'): - tp_dict.__orig_bases__ = bases - - annotations = {} - own_annotate = None - if ns_annotations is not None: - own_annotations = ns_annotations - elif sys.version_info >= (3, 14): - if hasattr(annotationlib, "get_annotate_from_class_namespace"): - own_annotate = annotationlib.get_annotate_from_class_namespace(ns) - else: - # 3.14.0a7 and earlier - own_annotate = ns.get("__annotate__") - if own_annotate is not None: - own_annotations = annotationlib.call_annotate_function( - own_annotate, Format.FORWARDREF, owner=tp_dict - ) - else: - own_annotations = {} - else: - own_annotations = {} - msg = "TypedDict('Name', {f0: t0, f1: t1, ...}); each t must be a type" - if _TAKES_MODULE: - own_checked_annotations = { - n: typing._type_check(tp, msg, module=tp_dict.__module__) - for n, tp in own_annotations.items() - } - else: - own_checked_annotations = { - n: typing._type_check(tp, msg) - for n, tp in own_annotations.items() - } - required_keys = set() - optional_keys = set() - readonly_keys = set() - mutable_keys = set() - extra_items_type = extra_items - - for base in bases: - base_dict = base.__dict__ - - if sys.version_info <= (3, 14): - annotations.update(base_dict.get('__annotations__', {})) - required_keys.update(base_dict.get('__required_keys__', ())) - optional_keys.update(base_dict.get('__optional_keys__', ())) - readonly_keys.update(base_dict.get('__readonly_keys__', ())) - mutable_keys.update(base_dict.get('__mutable_keys__', ())) - - # This was specified in an earlier version of PEP 728. Support - # is retained for backwards compatibility, but only for Python - # 3.13 and lower. - if (closed and sys.version_info < (3, 14) - and "__extra_items__" in own_checked_annotations): - annotation_type = own_checked_annotations.pop("__extra_items__") - qualifiers = set(_get_typeddict_qualifiers(annotation_type)) - if Required in qualifiers: - raise TypeError( - "Special key __extra_items__ does not support " - "Required" - ) - if NotRequired in qualifiers: - raise TypeError( - "Special key __extra_items__ does not support " - "NotRequired" - ) - extra_items_type = annotation_type - - annotations.update(own_checked_annotations) - for annotation_key, annotation_type in own_checked_annotations.items(): - qualifiers = set(_get_typeddict_qualifiers(annotation_type)) - - if Required in qualifiers: - required_keys.add(annotation_key) - elif NotRequired in qualifiers: - optional_keys.add(annotation_key) - elif total: - required_keys.add(annotation_key) - else: - optional_keys.add(annotation_key) - if ReadOnly in qualifiers: - mutable_keys.discard(annotation_key) - readonly_keys.add(annotation_key) - else: - mutable_keys.add(annotation_key) - readonly_keys.discard(annotation_key) - - # Breakpoint: https://github.com/python/cpython/pull/119891 - if sys.version_info >= (3, 14): - def __annotate__(format): - annos = {} - for base in bases: - if base is Generic: - continue - base_annotate = base.__annotate__ - if base_annotate is None: - continue - base_annos = annotationlib.call_annotate_function( - base_annotate, format, owner=base) - annos.update(base_annos) - if own_annotate is not None: - own = annotationlib.call_annotate_function( - own_annotate, format, owner=tp_dict) - if format != Format.STRING: - own = { - n: typing._type_check(tp, msg, module=tp_dict.__module__) - for n, tp in own.items() - } - elif format == Format.STRING: - own = annotationlib.annotations_to_string(own_annotations) - elif format in (Format.FORWARDREF, Format.VALUE): - own = own_checked_annotations - else: - raise NotImplementedError(format) - annos.update(own) - return annos - - tp_dict.__annotate__ = __annotate__ - else: - tp_dict.__annotations__ = annotations - tp_dict.__required_keys__ = frozenset(required_keys) - tp_dict.__optional_keys__ = frozenset(optional_keys) - tp_dict.__readonly_keys__ = frozenset(readonly_keys) - tp_dict.__mutable_keys__ = frozenset(mutable_keys) - tp_dict.__total__ = total - tp_dict.__closed__ = closed - tp_dict.__extra_items__ = extra_items_type - return tp_dict - - __call__ = dict # static method - - def __subclasscheck__(cls, other): - # Typed dicts are only for static structural subtyping. - raise TypeError('TypedDict does not support instance and class checks') - - __instancecheck__ = __subclasscheck__ - - _TypedDict = type.__new__(_TypedDictMeta, 'TypedDict', (), {}) - - def _create_typeddict( - typename, - fields, - /, - *, - typing_is_inline, - total, - closed, - extra_items, - **kwargs, - ): - if fields is _marker or fields is None: - if fields is _marker: - deprecated_thing = ( - "Failing to pass a value for the 'fields' parameter" - ) - else: - deprecated_thing = "Passing `None` as the 'fields' parameter" - - example = f"`{typename} = TypedDict({typename!r}, {{}})`" - deprecation_msg = ( - f"{deprecated_thing} is deprecated and will be disallowed in " - "Python 3.15. To create a TypedDict class with 0 fields " - "using the functional syntax, pass an empty dictionary, e.g. " - ) + example + "." - warnings.warn(deprecation_msg, DeprecationWarning, stacklevel=2) - # Support a field called "closed" - if closed is not False and closed is not True and closed is not None: - kwargs["closed"] = closed - closed = None - # Or "extra_items" - if extra_items is not NoExtraItems: - kwargs["extra_items"] = extra_items - extra_items = NoExtraItems - fields = kwargs - elif kwargs: - raise TypeError("TypedDict takes either a dict or keyword arguments," - " but not both") - if kwargs: - # Breakpoint: https://github.com/python/cpython/pull/104891 - if sys.version_info >= (3, 13): - raise TypeError("TypedDict takes no keyword arguments") - warnings.warn( - "The kwargs-based syntax for TypedDict definitions is deprecated " - "in Python 3.11, will be removed in Python 3.13, and may not be " - "understood by third-party type checkers.", - DeprecationWarning, - stacklevel=2, - ) - - ns = {'__annotations__': dict(fields)} - module = _caller(depth=4 if typing_is_inline else 2) - if module is not None: - # Setting correct module is necessary to make typed dict classes - # pickleable. - ns['__module__'] = module - - td = _TypedDictMeta(typename, (), ns, total=total, closed=closed, - extra_items=extra_items) - td.__orig_bases__ = (TypedDict,) - return td - - class _TypedDictSpecialForm(_SpecialForm, _root=True): - def __call__( - self, - typename, - fields=_marker, - /, - *, - total=True, - closed=None, - extra_items=NoExtraItems, - **kwargs - ): - return _create_typeddict( - typename, - fields, - typing_is_inline=False, - total=total, - closed=closed, - extra_items=extra_items, - **kwargs, - ) - - def __mro_entries__(self, bases): - return (_TypedDict,) - - @_TypedDictSpecialForm - def TypedDict(self, args): - """A simple typed namespace. At runtime it is equivalent to a plain dict. - - TypedDict creates a dictionary type such that a type checker will expect all - instances to have a certain set of keys, where each key is - associated with a value of a consistent type. This expectation - is not checked at runtime. - - Usage:: - - class Point2D(TypedDict): - x: int - y: int - label: str - - a: Point2D = {'x': 1, 'y': 2, 'label': 'good'} # OK - b: Point2D = {'z': 3, 'label': 'bad'} # Fails type check - - assert Point2D(x=1, y=2, label='first') == dict(x=1, y=2, label='first') - - The type info can be accessed via the Point2D.__annotations__ dict, and - the Point2D.__required_keys__ and Point2D.__optional_keys__ frozensets. - TypedDict supports an additional equivalent form:: - - Point2D = TypedDict('Point2D', {'x': int, 'y': int, 'label': str}) - - By default, all keys must be present in a TypedDict. It is possible - to override this by specifying totality:: - - class Point2D(TypedDict, total=False): - x: int - y: int - - This means that a Point2D TypedDict can have any of the keys omitted. A type - checker is only expected to support a literal False or True as the value of - the total argument. True is the default, and makes all items defined in the - class body be required. - - The Required and NotRequired special forms can also be used to mark - individual keys as being required or not required:: - - class Point2D(TypedDict): - x: int # the "x" key must always be present (Required is the default) - y: NotRequired[int] # the "y" key can be omitted - - See PEP 655 for more details on Required and NotRequired. - """ - # This runs when creating inline TypedDicts: - if not isinstance(args, dict): - raise TypeError( - "TypedDict[...] should be used with a single dict argument" - ) - - return _create_typeddict( - "", - args, - typing_is_inline=True, - total=True, - closed=True, - extra_items=NoExtraItems, - ) - - _TYPEDDICT_TYPES = (typing._TypedDictMeta, _TypedDictMeta) - - def is_typeddict(tp): - """Check if an annotation is a TypedDict class - - For example:: - class Film(TypedDict): - title: str - year: int - - is_typeddict(Film) # => True - is_typeddict(Union[list, str]) # => False - """ - return isinstance(tp, _TYPEDDICT_TYPES) - - -if hasattr(typing, "assert_type"): - assert_type = typing.assert_type - -else: - def assert_type(val, typ, /): - """Assert (to the type checker) that the value is of the given type. - - When the type checker encounters a call to assert_type(), it - emits an error if the value is not of the specified type:: - - def greet(name: str) -> None: - assert_type(name, str) # ok - assert_type(name, int) # type checker error - - At runtime this returns the first argument unchanged and otherwise - does nothing. - """ - return val - - -if hasattr(typing, "ReadOnly"): # 3.13+ - get_type_hints = typing.get_type_hints -else: # <=3.13 - # replaces _strip_annotations() - def _strip_extras(t): - """Strips Annotated, Required and NotRequired from a given type.""" - if isinstance(t, typing._AnnotatedAlias): - return _strip_extras(t.__origin__) - if hasattr(t, "__origin__") and t.__origin__ in (Required, NotRequired, ReadOnly): - return _strip_extras(t.__args__[0]) - if isinstance(t, typing._GenericAlias): - stripped_args = tuple(_strip_extras(a) for a in t.__args__) - if stripped_args == t.__args__: - return t - return t.copy_with(stripped_args) - if hasattr(_types, "GenericAlias") and isinstance(t, _types.GenericAlias): - stripped_args = tuple(_strip_extras(a) for a in t.__args__) - if stripped_args == t.__args__: - return t - return _types.GenericAlias(t.__origin__, stripped_args) - if hasattr(_types, "UnionType") and isinstance(t, _types.UnionType): - stripped_args = tuple(_strip_extras(a) for a in t.__args__) - if stripped_args == t.__args__: - return t - return functools.reduce(operator.or_, stripped_args) - - return t - - def get_type_hints(obj, globalns=None, localns=None, include_extras=False): - """Return type hints for an object. - - This is often the same as obj.__annotations__, but it handles - forward references encoded as string literals, adds Optional[t] if a - default value equal to None is set and recursively replaces all - 'Annotated[T, ...]', 'Required[T]' or 'NotRequired[T]' with 'T' - (unless 'include_extras=True'). - - The argument may be a module, class, method, or function. The annotations - are returned as a dictionary. For classes, annotations include also - inherited members. - - TypeError is raised if the argument is not of a type that can contain - annotations, and an empty dictionary is returned if no annotations are - present. - - BEWARE -- the behavior of globalns and localns is counterintuitive - (unless you are familiar with how eval() and exec() work). The - search order is locals first, then globals. - - - If no dict arguments are passed, an attempt is made to use the - globals from obj (or the respective module's globals for classes), - and these are also used as the locals. If the object does not appear - to have globals, an empty dictionary is used. - - - If one dict argument is passed, it is used for both globals and - locals. - - - If two dict arguments are passed, they specify globals and - locals, respectively. - """ - hint = typing.get_type_hints( - obj, globalns=globalns, localns=localns, include_extras=True - ) - # Breakpoint: https://github.com/python/cpython/pull/30304 - if sys.version_info < (3, 11): - _clean_optional(obj, hint, globalns, localns) - if include_extras: - return hint - return {k: _strip_extras(t) for k, t in hint.items()} - - _NoneType = type(None) - - def _could_be_inserted_optional(t): - """detects Union[..., None] pattern""" - if not isinstance(t, typing._UnionGenericAlias): - return False - # Assume if last argument is not None they are user defined - if t.__args__[-1] is not _NoneType: - return False - return True - - # < 3.11 - def _clean_optional(obj, hints, globalns=None, localns=None): - # reverts injected Union[..., None] cases from typing.get_type_hints - # when a None default value is used. - # see https://github.com/python/typing_extensions/issues/310 - if not hints or isinstance(obj, type): - return - defaults = typing._get_defaults(obj) # avoid accessing __annotations___ - if not defaults: - return - original_hints = obj.__annotations__ - for name, value in hints.items(): - # Not a Union[..., None] or replacement conditions not fullfilled - if (not _could_be_inserted_optional(value) - or name not in defaults - or defaults[name] is not None - ): - continue - original_value = original_hints[name] - # value=NoneType should have caused a skip above but check for safety - if original_value is None: - original_value = _NoneType - # Forward reference - if isinstance(original_value, str): - if globalns is None: - if isinstance(obj, _types.ModuleType): - globalns = obj.__dict__ - else: - nsobj = obj - # Find globalns for the unwrapped object. - while hasattr(nsobj, '__wrapped__'): - nsobj = nsobj.__wrapped__ - globalns = getattr(nsobj, '__globals__', {}) - if localns is None: - localns = globalns - elif localns is None: - localns = globalns - - original_value = ForwardRef( - original_value, - is_argument=not isinstance(obj, _types.ModuleType) - ) - original_evaluated = typing._eval_type(original_value, globalns, localns) - # Compare if values differ. Note that even if equal - # value might be cached by typing._tp_cache contrary to original_evaluated - if original_evaluated != value or ( - # 3.10: ForwardRefs of UnionType might be turned into _UnionGenericAlias - hasattr(_types, "UnionType") - and isinstance(original_evaluated, _types.UnionType) - and not isinstance(value, _types.UnionType) - ): - hints[name] = original_evaluated - -# Python 3.9 has get_origin() and get_args() but those implementations don't support -# ParamSpecArgs and ParamSpecKwargs, so only Python 3.10's versions will do. -# Breakpoint: https://github.com/python/cpython/pull/25298 -if sys.version_info >= (3, 10): - get_origin = typing.get_origin - get_args = typing.get_args -# 3.9 -else: - def get_origin(tp): - """Get the unsubscripted version of a type. - - This supports generic types, Callable, Tuple, Union, Literal, Final, ClassVar - and Annotated. Return None for unsupported types. Examples:: - - get_origin(Literal[42]) is Literal - get_origin(int) is None - get_origin(ClassVar[int]) is ClassVar - get_origin(Generic) is Generic - get_origin(Generic[T]) is Generic - get_origin(Union[T, int]) is Union - get_origin(List[Tuple[T, T]][int]) == list - get_origin(P.args) is P - """ - if isinstance(tp, typing._AnnotatedAlias): - return Annotated - if isinstance(tp, (typing._BaseGenericAlias, _types.GenericAlias, - ParamSpecArgs, ParamSpecKwargs)): - return tp.__origin__ - if tp is typing.Generic: - return typing.Generic - return None - - def get_args(tp): - """Get type arguments with all substitutions performed. - - For unions, basic simplifications used by Union constructor are performed. - Examples:: - get_args(Dict[str, int]) == (str, int) - get_args(int) == () - get_args(Union[int, Union[T, int], str][int]) == (int, str) - get_args(Union[int, Tuple[T, int]][str]) == (int, Tuple[str, int]) - get_args(Callable[[], T][int]) == ([], int) - """ - if isinstance(tp, typing._AnnotatedAlias): - return (tp.__origin__, *tp.__metadata__) - if isinstance(tp, (typing._GenericAlias, _types.GenericAlias)): - res = tp.__args__ - if get_origin(tp) is collections.abc.Callable and res[0] is not Ellipsis: - res = (list(res[:-1]), res[-1]) - return res - return () - - -# 3.10+ -if hasattr(typing, 'TypeAlias'): - TypeAlias = typing.TypeAlias -# 3.9 -else: - @_ExtensionsSpecialForm - def TypeAlias(self, parameters): - """Special marker indicating that an assignment should - be recognized as a proper type alias definition by type - checkers. - - For example:: - - Predicate: TypeAlias = Callable[..., bool] - - It's invalid when used anywhere except as in the example above. - """ - raise TypeError(f"{self} is not subscriptable") - - -def _set_default(type_param, default): - type_param.has_default = lambda: default is not NoDefault - type_param.__default__ = default - - -def _set_module(typevarlike): - # for pickling: - def_mod = _caller(depth=2) - if def_mod != 'typing_extensions': - typevarlike.__module__ = def_mod - - -class _DefaultMixin: - """Mixin for TypeVarLike defaults.""" - - __slots__ = () - __init__ = _set_default - - -# Classes using this metaclass must provide a _backported_typevarlike ClassVar -class _TypeVarLikeMeta(type): - def __instancecheck__(cls, __instance: Any) -> bool: - return isinstance(__instance, cls._backported_typevarlike) - - -if _PEP_696_IMPLEMENTED: - from typing import TypeVar -else: - # Add default and infer_variance parameters from PEP 696 and 695 - class TypeVar(metaclass=_TypeVarLikeMeta): - """Type variable.""" - - _backported_typevarlike = typing.TypeVar - - def __new__(cls, name, *constraints, bound=None, - covariant=False, contravariant=False, - default=NoDefault, infer_variance=False): - if hasattr(typing, "TypeAliasType"): - # PEP 695 implemented (3.12+), can pass infer_variance to typing.TypeVar - typevar = typing.TypeVar(name, *constraints, bound=bound, - covariant=covariant, contravariant=contravariant, - infer_variance=infer_variance) - else: - typevar = typing.TypeVar(name, *constraints, bound=bound, - covariant=covariant, contravariant=contravariant) - if infer_variance and (covariant or contravariant): - raise ValueError("Variance cannot be specified with infer_variance.") - typevar.__infer_variance__ = infer_variance - - _set_default(typevar, default) - _set_module(typevar) - - def _tvar_prepare_subst(alias, args): - if ( - typevar.has_default() - and alias.__parameters__.index(typevar) == len(args) - ): - args += (typevar.__default__,) - return args - - typevar.__typing_prepare_subst__ = _tvar_prepare_subst - return typevar - - def __init_subclass__(cls) -> None: - raise TypeError(f"type '{__name__}.TypeVar' is not an acceptable base type") - - -# Python 3.10+ has PEP 612 -if hasattr(typing, 'ParamSpecArgs'): - ParamSpecArgs = typing.ParamSpecArgs - ParamSpecKwargs = typing.ParamSpecKwargs -# 3.9 -else: - class _Immutable: - """Mixin to indicate that object should not be copied.""" - __slots__ = () - - def __copy__(self): - return self - - def __deepcopy__(self, memo): - return self - - class ParamSpecArgs(_Immutable): - """The args for a ParamSpec object. - - Given a ParamSpec object P, P.args is an instance of ParamSpecArgs. - - ParamSpecArgs objects have a reference back to their ParamSpec: - - P.args.__origin__ is P - - This type is meant for runtime introspection and has no special meaning to - static type checkers. - """ - def __init__(self, origin): - self.__origin__ = origin - - def __repr__(self): - return f"{self.__origin__.__name__}.args" - - def __eq__(self, other): - if not isinstance(other, ParamSpecArgs): - return NotImplemented - return self.__origin__ == other.__origin__ - - class ParamSpecKwargs(_Immutable): - """The kwargs for a ParamSpec object. - - Given a ParamSpec object P, P.kwargs is an instance of ParamSpecKwargs. - - ParamSpecKwargs objects have a reference back to their ParamSpec: - - P.kwargs.__origin__ is P - - This type is meant for runtime introspection and has no special meaning to - static type checkers. - """ - def __init__(self, origin): - self.__origin__ = origin - - def __repr__(self): - return f"{self.__origin__.__name__}.kwargs" - - def __eq__(self, other): - if not isinstance(other, ParamSpecKwargs): - return NotImplemented - return self.__origin__ == other.__origin__ - - -if _PEP_696_IMPLEMENTED: - from typing import ParamSpec - -# 3.10+ -elif hasattr(typing, 'ParamSpec'): - - # Add default parameter - PEP 696 - class ParamSpec(metaclass=_TypeVarLikeMeta): - """Parameter specification.""" - - _backported_typevarlike = typing.ParamSpec - - def __new__(cls, name, *, bound=None, - covariant=False, contravariant=False, - infer_variance=False, default=NoDefault): - if hasattr(typing, "TypeAliasType"): - # PEP 695 implemented, can pass infer_variance to typing.TypeVar - paramspec = typing.ParamSpec(name, bound=bound, - covariant=covariant, - contravariant=contravariant, - infer_variance=infer_variance) - else: - paramspec = typing.ParamSpec(name, bound=bound, - covariant=covariant, - contravariant=contravariant) - paramspec.__infer_variance__ = infer_variance - - _set_default(paramspec, default) - _set_module(paramspec) - - def _paramspec_prepare_subst(alias, args): - params = alias.__parameters__ - i = params.index(paramspec) - if i == len(args) and paramspec.has_default(): - args = [*args, paramspec.__default__] - if i >= len(args): - raise TypeError(f"Too few arguments for {alias}") - # Special case where Z[[int, str, bool]] == Z[int, str, bool] in PEP 612. - if len(params) == 1 and not typing._is_param_expr(args[0]): - assert i == 0 - args = (args,) - # Convert lists to tuples to help other libraries cache the results. - elif isinstance(args[i], list): - args = (*args[:i], tuple(args[i]), *args[i + 1:]) - return args - - paramspec.__typing_prepare_subst__ = _paramspec_prepare_subst - return paramspec - - def __init_subclass__(cls) -> None: - raise TypeError(f"type '{__name__}.ParamSpec' is not an acceptable base type") - -# 3.9 -else: - - # Inherits from list as a workaround for Callable checks in Python < 3.9.2. - class ParamSpec(list, _DefaultMixin): - """Parameter specification variable. - - Usage:: - - P = ParamSpec('P') - - Parameter specification variables exist primarily for the benefit of static - type checkers. They are used to forward the parameter types of one - callable to another callable, a pattern commonly found in higher order - functions and decorators. They are only valid when used in ``Concatenate``, - or s the first argument to ``Callable``. In Python 3.10 and higher, - they are also supported in user-defined Generics at runtime. - See class Generic for more information on generic types. An - example for annotating a decorator:: - - T = TypeVar('T') - P = ParamSpec('P') - - def add_logging(f: Callable[P, T]) -> Callable[P, T]: - '''A type-safe decorator to add logging to a function.''' - def inner(*args: P.args, **kwargs: P.kwargs) -> T: - logging.info(f'{f.__name__} was called') - return f(*args, **kwargs) - return inner - - @add_logging - def add_two(x: float, y: float) -> float: - '''Add two numbers together.''' - return x + y - - Parameter specification variables defined with covariant=True or - contravariant=True can be used to declare covariant or contravariant - generic types. These keyword arguments are valid, but their actual semantics - are yet to be decided. See PEP 612 for details. - - Parameter specification variables can be introspected. e.g.: - - P.__name__ == 'T' - P.__bound__ == None - P.__covariant__ == False - P.__contravariant__ == False - - Note that only parameter specification variables defined in global scope can - be pickled. - """ - - # Trick Generic __parameters__. - __class__ = typing.TypeVar - - @property - def args(self): - return ParamSpecArgs(self) - - @property - def kwargs(self): - return ParamSpecKwargs(self) - - def __init__(self, name, *, bound=None, covariant=False, contravariant=False, - infer_variance=False, default=NoDefault): - list.__init__(self, [self]) - self.__name__ = name - self.__covariant__ = bool(covariant) - self.__contravariant__ = bool(contravariant) - self.__infer_variance__ = bool(infer_variance) - if bound: - self.__bound__ = typing._type_check(bound, 'Bound must be a type.') - else: - self.__bound__ = None - _DefaultMixin.__init__(self, default) - - # for pickling: - def_mod = _caller() - if def_mod != 'typing_extensions': - self.__module__ = def_mod - - def __repr__(self): - if self.__infer_variance__: - prefix = '' - elif self.__covariant__: - prefix = '+' - elif self.__contravariant__: - prefix = '-' - else: - prefix = '~' - return prefix + self.__name__ - - def __hash__(self): - return object.__hash__(self) - - def __eq__(self, other): - return self is other - - def __reduce__(self): - return self.__name__ - - # Hack to get typing._type_check to pass. - def __call__(self, *args, **kwargs): - pass - - -# 3.9 -if not hasattr(typing, 'Concatenate'): - # Inherits from list as a workaround for Callable checks in Python < 3.9.2. - - # 3.9.0-1 - if not hasattr(typing, '_type_convert'): - def _type_convert(arg, module=None, *, allow_special_forms=False): - """For converting None to type(None), and strings to ForwardRef.""" - if arg is None: - return type(None) - if isinstance(arg, str): - if sys.version_info <= (3, 9, 6): - return ForwardRef(arg) - if sys.version_info <= (3, 9, 7): - return ForwardRef(arg, module=module) - return ForwardRef(arg, module=module, is_class=allow_special_forms) - return arg - else: - _type_convert = typing._type_convert - - class _ConcatenateGenericAlias(list): - - # Trick Generic into looking into this for __parameters__. - __class__ = typing._GenericAlias - - def __init__(self, origin, args): - super().__init__(args) - self.__origin__ = origin - self.__args__ = args - - def __repr__(self): - _type_repr = typing._type_repr - return (f'{_type_repr(self.__origin__)}' - f'[{", ".join(_type_repr(arg) for arg in self.__args__)}]') - - def __hash__(self): - return hash((self.__origin__, self.__args__)) - - # Hack to get typing._type_check to pass in Generic. - def __call__(self, *args, **kwargs): - pass - - @property - def __parameters__(self): - return tuple( - tp for tp in self.__args__ if isinstance(tp, (typing.TypeVar, ParamSpec)) - ) - - # 3.9 used by __getitem__ below - def copy_with(self, params): - if isinstance(params[-1], _ConcatenateGenericAlias): - params = (*params[:-1], *params[-1].__args__) - elif isinstance(params[-1], (list, tuple)): - return (*params[:-1], *params[-1]) - elif (not (params[-1] is ... or isinstance(params[-1], ParamSpec))): - raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable or ellipsis.") - return self.__class__(self.__origin__, params) - - # 3.9; accessed during GenericAlias.__getitem__ when substituting - def __getitem__(self, args): - if self.__origin__ in (Generic, Protocol): - # Can't subscript Generic[...] or Protocol[...]. - raise TypeError(f"Cannot subscript already-subscripted {self}") - if not self.__parameters__: - raise TypeError(f"{self} is not a generic class") - - if not isinstance(args, tuple): - args = (args,) - args = _unpack_args(*(_type_convert(p) for p in args)) - params = self.__parameters__ - for param in params: - prepare = getattr(param, "__typing_prepare_subst__", None) - if prepare is not None: - args = prepare(self, args) - # 3.9 & typing.ParamSpec - elif isinstance(param, ParamSpec): - i = params.index(param) - if ( - i == len(args) - and getattr(param, '__default__', NoDefault) is not NoDefault - ): - args = [*args, param.__default__] - if i >= len(args): - raise TypeError(f"Too few arguments for {self}") - # Special case for Z[[int, str, bool]] == Z[int, str, bool] - if len(params) == 1 and not _is_param_expr(args[0]): - assert i == 0 - args = (args,) - elif ( - isinstance(args[i], list) - # 3.9 - # This class inherits from list do not convert - and not isinstance(args[i], _ConcatenateGenericAlias) - ): - args = (*args[:i], tuple(args[i]), *args[i + 1:]) - - alen = len(args) - plen = len(params) - if alen != plen: - raise TypeError( - f"Too {'many' if alen > plen else 'few'} arguments for {self};" - f" actual {alen}, expected {plen}" - ) - - subst = dict(zip(self.__parameters__, args)) - # determine new args - new_args = [] - for arg in self.__args__: - if isinstance(arg, type): - new_args.append(arg) - continue - if isinstance(arg, TypeVar): - arg = subst[arg] - if ( - (isinstance(arg, typing._GenericAlias) and _is_unpack(arg)) - or ( - hasattr(_types, "GenericAlias") - and isinstance(arg, _types.GenericAlias) - and getattr(arg, "__unpacked__", False) - ) - ): - raise TypeError(f"{arg} is not valid as type argument") - - elif isinstance(arg, - typing._GenericAlias - if not hasattr(_types, "GenericAlias") else - (typing._GenericAlias, _types.GenericAlias) - ): - subparams = arg.__parameters__ - if subparams: - subargs = tuple(subst[x] for x in subparams) - arg = arg[subargs] - new_args.append(arg) - return self.copy_with(tuple(new_args)) - -# 3.10+ -else: - _ConcatenateGenericAlias = typing._ConcatenateGenericAlias - - # 3.10 - if sys.version_info < (3, 11): - - class _ConcatenateGenericAlias(typing._ConcatenateGenericAlias, _root=True): - # needed for checks in collections.abc.Callable to accept this class - __module__ = "typing" - - def copy_with(self, params): - if isinstance(params[-1], (list, tuple)): - return (*params[:-1], *params[-1]) - if isinstance(params[-1], typing._ConcatenateGenericAlias): - params = (*params[:-1], *params[-1].__args__) - elif not (params[-1] is ... or isinstance(params[-1], ParamSpec)): - raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable or ellipsis.") - return super(typing._ConcatenateGenericAlias, self).copy_with(params) - - def __getitem__(self, args): - value = super().__getitem__(args) - if isinstance(value, tuple) and any(_is_unpack(t) for t in value): - return tuple(_unpack_args(*(n for n in value))) - return value - - -# 3.9.2 -class _EllipsisDummy: ... - - -# <=3.10 -def _create_concatenate_alias(origin, parameters): - if parameters[-1] is ... and sys.version_info < (3, 9, 2): - # Hack: Arguments must be types, replace it with one. - parameters = (*parameters[:-1], _EllipsisDummy) - if sys.version_info >= (3, 10, 3): - concatenate = _ConcatenateGenericAlias(origin, parameters, - _typevar_types=(TypeVar, ParamSpec), - _paramspec_tvars=True) - else: - concatenate = _ConcatenateGenericAlias(origin, parameters) - if parameters[-1] is not _EllipsisDummy: - return concatenate - # Remove dummy again - concatenate.__args__ = tuple(p if p is not _EllipsisDummy else ... - for p in concatenate.__args__) - if sys.version_info < (3, 10): - # backport needs __args__ adjustment only - return concatenate - concatenate.__parameters__ = tuple(p for p in concatenate.__parameters__ - if p is not _EllipsisDummy) - return concatenate - - -# <=3.10 -@typing._tp_cache -def _concatenate_getitem(self, parameters): - if parameters == (): - raise TypeError("Cannot take a Concatenate of no types.") - if not isinstance(parameters, tuple): - parameters = (parameters,) - if not (parameters[-1] is ... or isinstance(parameters[-1], ParamSpec)): - raise TypeError("The last parameter to Concatenate should be a " - "ParamSpec variable or ellipsis.") - msg = "Concatenate[arg, ...]: each arg must be a type." - parameters = (*(typing._type_check(p, msg) for p in parameters[:-1]), - parameters[-1]) - return _create_concatenate_alias(self, parameters) - - -# 3.11+; Concatenate does not accept ellipsis in 3.10 -# Breakpoint: https://github.com/python/cpython/pull/30969 -if sys.version_info >= (3, 11): - Concatenate = typing.Concatenate -# <=3.10 -else: - @_ExtensionsSpecialForm - def Concatenate(self, parameters): - """Used in conjunction with ``ParamSpec`` and ``Callable`` to represent a - higher order function which adds, removes or transforms parameters of a - callable. - - For example:: - - Callable[Concatenate[int, P], int] - - See PEP 612 for detailed information. - """ - return _concatenate_getitem(self, parameters) - - -# 3.10+ -if hasattr(typing, 'TypeGuard'): - TypeGuard = typing.TypeGuard -# 3.9 -else: - @_ExtensionsSpecialForm - def TypeGuard(self, parameters): - """Special typing form used to annotate the return type of a user-defined - type guard function. ``TypeGuard`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. - - ``TypeGuard`` aims to benefit *type narrowing* -- a technique used by static - type checkers to determine a more precise type of an expression within a - program's code flow. Usually type narrowing is done by analyzing - conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". - - Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeGuard[...]`` as its - return type to alert static type checkers to this intention. - - Using ``-> TypeGuard`` tells the static type checker that for a given - function: - - 1. The return value is a boolean. - 2. If the return value is ``True``, the type of its argument - is the type inside ``TypeGuard``. - - For example:: - - def is_str(val: Union[str, float]): - # "isinstance" type guard - if isinstance(val, str): - # Type of ``val`` is narrowed to ``str`` - ... - else: - # Else, type of ``val`` is narrowed to ``float``. - ... - - Strict type narrowing is not enforced -- ``TypeB`` need not be a narrower - form of ``TypeA`` (it can even be a wider form) and this may lead to - type-unsafe results. The main reason is to allow for things like - narrowing ``List[object]`` to ``List[str]`` even though the latter is not - a subtype of the former, since ``List`` is invariant. The responsibility of - writing type-safe type guards is left to the user. - - ``TypeGuard`` also works with type variables. For more information, see - PEP 647 (User-Defined Type Guards). - """ - item = typing._type_check(parameters, f'{self} accepts only a single type.') - return typing._GenericAlias(self, (item,)) - - -# 3.13+ -if hasattr(typing, 'TypeIs'): - TypeIs = typing.TypeIs -# <=3.12 -else: - @_ExtensionsSpecialForm - def TypeIs(self, parameters): - """Special typing form used to annotate the return type of a user-defined - type narrower function. ``TypeIs`` only accepts a single type argument. - At runtime, functions marked this way should return a boolean. - - ``TypeIs`` aims to benefit *type narrowing* -- a technique used by static - type checkers to determine a more precise type of an expression within a - program's code flow. Usually type narrowing is done by analyzing - conditional code flow and applying the narrowing to a block of code. The - conditional expression here is sometimes referred to as a "type guard". - - Sometimes it would be convenient to use a user-defined boolean function - as a type guard. Such a function should use ``TypeIs[...]`` as its - return type to alert static type checkers to this intention. - - Using ``-> TypeIs`` tells the static type checker that for a given - function: - - 1. The return value is a boolean. - 2. If the return value is ``True``, the type of its argument - is the intersection of the type inside ``TypeIs`` and the argument's - previously known type. - - For example:: - - def is_awaitable(val: object) -> TypeIs[Awaitable[Any]]: - return hasattr(val, '__await__') - - def f(val: Union[int, Awaitable[int]]) -> int: - if is_awaitable(val): - assert_type(val, Awaitable[int]) - else: - assert_type(val, int) - - ``TypeIs`` also works with type variables. For more information, see - PEP 742 (Narrowing types with TypeIs). - """ - item = typing._type_check(parameters, f'{self} accepts only a single type.') - return typing._GenericAlias(self, (item,)) - - -# 3.14+? -if hasattr(typing, 'TypeForm'): - TypeForm = typing.TypeForm -# <=3.13 -else: - class _TypeFormForm(_ExtensionsSpecialForm, _root=True): - # TypeForm(X) is equivalent to X but indicates to the type checker - # that the object is a TypeForm. - def __call__(self, obj, /): - return obj - - @_TypeFormForm - def TypeForm(self, parameters): - """A special form representing the value that results from the evaluation - of a type expression. This value encodes the information supplied in the - type expression, and it represents the type described by that type expression. - - When used in a type expression, TypeForm describes a set of type form objects. - It accepts a single type argument, which must be a valid type expression. - ``TypeForm[T]`` describes the set of all type form objects that represent - the type T or types that are assignable to T. - - Usage: - - def cast[T](typ: TypeForm[T], value: Any) -> T: ... - - reveal_type(cast(int, "x")) # int - - See PEP 747 for more information. - """ - item = typing._type_check(parameters, f'{self} accepts only a single type.') - return typing._GenericAlias(self, (item,)) - - - - -if hasattr(typing, "LiteralString"): # 3.11+ - LiteralString = typing.LiteralString -else: - @_SpecialForm - def LiteralString(self, params): - """Represents an arbitrary literal string. - - Example:: - - from typing_extensions import LiteralString - - def query(sql: LiteralString) -> ...: - ... - - query("SELECT * FROM table") # ok - query(f"SELECT * FROM {input()}") # not ok - - See PEP 675 for details. - - """ - raise TypeError(f"{self} is not subscriptable") - - -if hasattr(typing, "Self"): # 3.11+ - Self = typing.Self -else: - @_SpecialForm - def Self(self, params): - """Used to spell the type of "self" in classes. - - Example:: - - from typing import Self - - class ReturnsSelf: - def parse(self, data: bytes) -> Self: - ... - return self - - """ - - raise TypeError(f"{self} is not subscriptable") - - -if hasattr(typing, "Never"): # 3.11+ - Never = typing.Never -else: - @_SpecialForm - def Never(self, params): - """The bottom type, a type that has no members. - - This can be used to define a function that should never be - called, or a function that never returns:: - - from typing_extensions import Never - - def never_call_me(arg: Never) -> None: - pass - - def int_or_str(arg: int | str) -> None: - never_call_me(arg) # type checker error - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _: - never_call_me(arg) # ok, arg is of type Never - - """ - - raise TypeError(f"{self} is not subscriptable") - - -if hasattr(typing, 'Required'): # 3.11+ - Required = typing.Required - NotRequired = typing.NotRequired -else: # <=3.10 - @_ExtensionsSpecialForm - def Required(self, parameters): - """A special typing construct to mark a key of a total=False TypedDict - as required. For example: - - class Movie(TypedDict, total=False): - title: Required[str] - year: int - - m = Movie( - title='The Matrix', # typechecker error if key is omitted - year=1999, - ) - - There is no runtime checking that a required key is actually provided - when instantiating a related TypedDict. - """ - item = typing._type_check(parameters, f'{self._name} accepts only a single type.') - return typing._GenericAlias(self, (item,)) - - @_ExtensionsSpecialForm - def NotRequired(self, parameters): - """A special typing construct to mark a key of a TypedDict as - potentially missing. For example: - - class Movie(TypedDict): - title: str - year: NotRequired[int] - - m = Movie( - title='The Matrix', # typechecker error if key is omitted - year=1999, - ) - """ - item = typing._type_check(parameters, f'{self._name} accepts only a single type.') - return typing._GenericAlias(self, (item,)) - - -if hasattr(typing, 'ReadOnly'): - ReadOnly = typing.ReadOnly -else: # <=3.12 - @_ExtensionsSpecialForm - def ReadOnly(self, parameters): - """A special typing construct to mark an item of a TypedDict as read-only. - - For example: - - class Movie(TypedDict): - title: ReadOnly[str] - year: int - - def mutate_movie(m: Movie) -> None: - m["year"] = 1992 # allowed - m["title"] = "The Matrix" # typechecker error - - There is no runtime checking for this property. - """ - item = typing._type_check(parameters, f'{self._name} accepts only a single type.') - return typing._GenericAlias(self, (item,)) - - -_UNPACK_DOC = """\ -Type unpack operator. - -The type unpack operator takes the child types from some container type, -such as `tuple[int, str]` or a `TypeVarTuple`, and 'pulls them out'. For -example: - - # For some generic class `Foo`: - Foo[Unpack[tuple[int, str]]] # Equivalent to Foo[int, str] - - Ts = TypeVarTuple('Ts') - # Specifies that `Bar` is generic in an arbitrary number of types. - # (Think of `Ts` as a tuple of an arbitrary number of individual - # `TypeVar`s, which the `Unpack` is 'pulling out' directly into the - # `Generic[]`.) - class Bar(Generic[Unpack[Ts]]): ... - Bar[int] # Valid - Bar[int, str] # Also valid - -From Python 3.11, this can also be done using the `*` operator: - - Foo[*tuple[int, str]] - class Bar(Generic[*Ts]): ... - -The operator can also be used along with a `TypedDict` to annotate -`**kwargs` in a function signature. For instance: - - class Movie(TypedDict): - name: str - year: int - - # This function expects two keyword arguments - *name* of type `str` and - # *year* of type `int`. - def foo(**kwargs: Unpack[Movie]): ... - -Note that there is only some runtime checking of this operator. Not -everything the runtime allows may be accepted by static type checkers. - -For more information, see PEP 646 and PEP 692. -""" - - -# PEP 692 changed the repr of Unpack[] -# Breakpoint: https://github.com/python/cpython/pull/104048 -if sys.version_info >= (3, 12): - Unpack = typing.Unpack - - def _is_unpack(obj): - return get_origin(obj) is Unpack - -else: # <=3.11 - class _UnpackSpecialForm(_ExtensionsSpecialForm, _root=True): - def __init__(self, getitem): - super().__init__(getitem) - self.__doc__ = _UNPACK_DOC - - class _UnpackAlias(typing._GenericAlias, _root=True): - if sys.version_info < (3, 11): - # needed for compatibility with Generic[Unpack[Ts]] - __class__ = typing.TypeVar - - @property - def __typing_unpacked_tuple_args__(self): - assert self.__origin__ is Unpack - assert len(self.__args__) == 1 - arg, = self.__args__ - if isinstance(arg, (typing._GenericAlias, _types.GenericAlias)): - if arg.__origin__ is not tuple: - raise TypeError("Unpack[...] must be used with a tuple type") - return arg.__args__ - return None - - @property - def __typing_is_unpacked_typevartuple__(self): - assert self.__origin__ is Unpack - assert len(self.__args__) == 1 - return isinstance(self.__args__[0], TypeVarTuple) - - def __getitem__(self, args): - if self.__typing_is_unpacked_typevartuple__: - return args - return super().__getitem__(args) - - @_UnpackSpecialForm - def Unpack(self, parameters): - item = typing._type_check(parameters, f'{self._name} accepts only a single type.') - return _UnpackAlias(self, (item,)) - - def _is_unpack(obj): - return isinstance(obj, _UnpackAlias) - - -def _unpack_args(*args): - newargs = [] - for arg in args: - subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) - if subargs is not None and (not (subargs and subargs[-1] is ...)): - newargs.extend(subargs) - else: - newargs.append(arg) - return newargs - - -if _PEP_696_IMPLEMENTED: - from typing import TypeVarTuple - -elif hasattr(typing, "TypeVarTuple"): # 3.11+ - - # Add default parameter - PEP 696 - class TypeVarTuple(metaclass=_TypeVarLikeMeta): - """Type variable tuple.""" - - _backported_typevarlike = typing.TypeVarTuple - - def __new__(cls, name, *, default=NoDefault): - tvt = typing.TypeVarTuple(name) - _set_default(tvt, default) - _set_module(tvt) - - def _typevartuple_prepare_subst(alias, args): - params = alias.__parameters__ - typevartuple_index = params.index(tvt) - for param in params[typevartuple_index + 1:]: - if isinstance(param, TypeVarTuple): - raise TypeError( - f"More than one TypeVarTuple parameter in {alias}" - ) - - alen = len(args) - plen = len(params) - left = typevartuple_index - right = plen - typevartuple_index - 1 - var_tuple_index = None - fillarg = None - for k, arg in enumerate(args): - if not isinstance(arg, type): - subargs = getattr(arg, '__typing_unpacked_tuple_args__', None) - if subargs and len(subargs) == 2 and subargs[-1] is ...: - if var_tuple_index is not None: - raise TypeError( - "More than one unpacked " - "arbitrary-length tuple argument" - ) - var_tuple_index = k - fillarg = subargs[0] - if var_tuple_index is not None: - left = min(left, var_tuple_index) - right = min(right, alen - var_tuple_index - 1) - elif left + right > alen: - raise TypeError(f"Too few arguments for {alias};" - f" actual {alen}, expected at least {plen - 1}") - if left == alen - right and tvt.has_default(): - replacement = _unpack_args(tvt.__default__) - else: - replacement = args[left: alen - right] - - return ( - *args[:left], - *([fillarg] * (typevartuple_index - left)), - replacement, - *([fillarg] * (plen - right - left - typevartuple_index - 1)), - *args[alen - right:], - ) - - tvt.__typing_prepare_subst__ = _typevartuple_prepare_subst - return tvt - - def __init_subclass__(self, *args, **kwds): - raise TypeError("Cannot subclass special typing classes") - -else: # <=3.10 - class TypeVarTuple(_DefaultMixin): - """Type variable tuple. - - Usage:: - - Ts = TypeVarTuple('Ts') - - In the same way that a normal type variable is a stand-in for a single - type such as ``int``, a type variable *tuple* is a stand-in for a *tuple* - type such as ``Tuple[int, str]``. - - Type variable tuples can be used in ``Generic`` declarations. - Consider the following example:: - - class Array(Generic[*Ts]): ... - - The ``Ts`` type variable tuple here behaves like ``tuple[T1, T2]``, - where ``T1`` and ``T2`` are type variables. To use these type variables - as type parameters of ``Array``, we must *unpack* the type variable tuple using - the star operator: ``*Ts``. The signature of ``Array`` then behaves - as if we had simply written ``class Array(Generic[T1, T2]): ...``. - In contrast to ``Generic[T1, T2]``, however, ``Generic[*Shape]`` allows - us to parameterise the class with an *arbitrary* number of type parameters. - - Type variable tuples can be used anywhere a normal ``TypeVar`` can. - This includes class definitions, as shown above, as well as function - signatures and variable annotations:: - - class Array(Generic[*Ts]): - - def __init__(self, shape: Tuple[*Ts]): - self._shape: Tuple[*Ts] = shape - - def get_shape(self) -> Tuple[*Ts]: - return self._shape - - shape = (Height(480), Width(640)) - x: Array[Height, Width] = Array(shape) - y = abs(x) # Inferred type is Array[Height, Width] - z = x + x # ... is Array[Height, Width] - x.get_shape() # ... is tuple[Height, Width] - - """ - - # Trick Generic __parameters__. - __class__ = typing.TypeVar - - def __iter__(self): - yield self.__unpacked__ - - def __init__(self, name, *, default=NoDefault): - self.__name__ = name - _DefaultMixin.__init__(self, default) - - # for pickling: - def_mod = _caller() - if def_mod != 'typing_extensions': - self.__module__ = def_mod - - self.__unpacked__ = Unpack[self] - - def __repr__(self): - return self.__name__ - - def __hash__(self): - return object.__hash__(self) - - def __eq__(self, other): - return self is other - - def __reduce__(self): - return self.__name__ - - def __init_subclass__(self, *args, **kwds): - if '_root' not in kwds: - raise TypeError("Cannot subclass special typing classes") - - -if hasattr(typing, "reveal_type"): # 3.11+ - reveal_type = typing.reveal_type -else: # <=3.10 - def reveal_type(obj: T, /) -> T: - """Reveal the inferred type of a variable. - - When a static type checker encounters a call to ``reveal_type()``, - it will emit the inferred type of the argument:: - - x: int = 1 - reveal_type(x) - - Running a static type checker (e.g., ``mypy``) on this example - will produce output similar to 'Revealed type is "builtins.int"'. - - At runtime, the function prints the runtime type of the - argument and returns it unchanged. - - """ - print(f"Runtime type is {type(obj).__name__!r}", file=sys.stderr) - return obj - - -if hasattr(typing, "_ASSERT_NEVER_REPR_MAX_LENGTH"): # 3.11+ - _ASSERT_NEVER_REPR_MAX_LENGTH = typing._ASSERT_NEVER_REPR_MAX_LENGTH -else: # <=3.10 - _ASSERT_NEVER_REPR_MAX_LENGTH = 100 - - -if hasattr(typing, "assert_never"): # 3.11+ - assert_never = typing.assert_never -else: # <=3.10 - def assert_never(arg: Never, /) -> Never: - """Assert to the type checker that a line of code is unreachable. - - Example:: - - def int_or_str(arg: int | str) -> None: - match arg: - case int(): - print("It's an int") - case str(): - print("It's a str") - case _: - assert_never(arg) - - If a type checker finds that a call to assert_never() is - reachable, it will emit an error. - - At runtime, this throws an exception when called. - - """ - value = repr(arg) - if len(value) > _ASSERT_NEVER_REPR_MAX_LENGTH: - value = value[:_ASSERT_NEVER_REPR_MAX_LENGTH] + '...' - raise AssertionError(f"Expected code to be unreachable, but got: {value}") - - -# dataclass_transform exists in 3.11 but lacks the frozen_default parameter -# Breakpoint: https://github.com/python/cpython/pull/99958 -if sys.version_info >= (3, 12): # 3.12+ - dataclass_transform = typing.dataclass_transform -else: # <=3.11 - def dataclass_transform( - *, - eq_default: bool = True, - order_default: bool = False, - kw_only_default: bool = False, - frozen_default: bool = False, - field_specifiers: typing.Tuple[ - typing.Union[typing.Type[typing.Any], typing.Callable[..., typing.Any]], - ... - ] = (), - **kwargs: typing.Any, - ) -> typing.Callable[[T], T]: - """Decorator that marks a function, class, or metaclass as providing - dataclass-like behavior. - - Example: - - from typing_extensions import dataclass_transform - - _T = TypeVar("_T") - - # Used on a decorator function - @dataclass_transform() - def create_model(cls: type[_T]) -> type[_T]: - ... - return cls - - @create_model - class CustomerModel: - id: int - name: str - - # Used on a base class - @dataclass_transform() - class ModelBase: ... - - class CustomerModel(ModelBase): - id: int - name: str - - # Used on a metaclass - @dataclass_transform() - class ModelMeta(type): ... - - class ModelBase(metaclass=ModelMeta): ... - - class CustomerModel(ModelBase): - id: int - name: str - - Each of the ``CustomerModel`` classes defined in this example will now - behave similarly to a dataclass created with the ``@dataclasses.dataclass`` - decorator. For example, the type checker will synthesize an ``__init__`` - method. - - The arguments to this decorator can be used to customize this behavior: - - ``eq_default`` indicates whether the ``eq`` parameter is assumed to be - True or False if it is omitted by the caller. - - ``order_default`` indicates whether the ``order`` parameter is - assumed to be True or False if it is omitted by the caller. - - ``kw_only_default`` indicates whether the ``kw_only`` parameter is - assumed to be True or False if it is omitted by the caller. - - ``frozen_default`` indicates whether the ``frozen`` parameter is - assumed to be True or False if it is omitted by the caller. - - ``field_specifiers`` specifies a static list of supported classes - or functions that describe fields, similar to ``dataclasses.field()``. - - At runtime, this decorator records its arguments in the - ``__dataclass_transform__`` attribute on the decorated object. - - See PEP 681 for details. - - """ - def decorator(cls_or_fn): - cls_or_fn.__dataclass_transform__ = { - "eq_default": eq_default, - "order_default": order_default, - "kw_only_default": kw_only_default, - "frozen_default": frozen_default, - "field_specifiers": field_specifiers, - "kwargs": kwargs, - } - return cls_or_fn - return decorator - - -if hasattr(typing, "override"): # 3.12+ - override = typing.override -else: # <=3.11 - _F = typing.TypeVar("_F", bound=typing.Callable[..., typing.Any]) - - def override(arg: _F, /) -> _F: - """Indicate that a method is intended to override a method in a base class. - - Usage: - - class Base: - def method(self) -> None: - pass - - class Child(Base): - @override - def method(self) -> None: - super().method() - - When this decorator is applied to a method, the type checker will - validate that it overrides a method with the same name on a base class. - This helps prevent bugs that may occur when a base class is changed - without an equivalent change to a child class. - - There is no runtime checking of these properties. The decorator - sets the ``__override__`` attribute to ``True`` on the decorated object - to allow runtime introspection. - - See PEP 698 for details. - - """ - try: - arg.__override__ = True - except (AttributeError, TypeError): - # Skip the attribute silently if it is not writable. - # AttributeError happens if the object has __slots__ or a - # read-only property, TypeError if it's a builtin class. - pass - return arg - - -# Python 3.13.3+ contains a fix for the wrapped __new__ -# Breakpoint: https://github.com/python/cpython/pull/132160 -if sys.version_info >= (3, 13, 3): - deprecated = warnings.deprecated -else: - _T = typing.TypeVar("_T") - - class deprecated: - """Indicate that a class, function or overload is deprecated. - - When this decorator is applied to an object, the type checker - will generate a diagnostic on usage of the deprecated object. - - Usage: - - @deprecated("Use B instead") - class A: - pass - - @deprecated("Use g instead") - def f(): - pass - - @overload - @deprecated("int support is deprecated") - def g(x: int) -> int: ... - @overload - def g(x: str) -> int: ... - - The warning specified by *category* will be emitted at runtime - on use of deprecated objects. For functions, that happens on calls; - for classes, on instantiation and on creation of subclasses. - If the *category* is ``None``, no warning is emitted at runtime. - The *stacklevel* determines where the - warning is emitted. If it is ``1`` (the default), the warning - is emitted at the direct caller of the deprecated object; if it - is higher, it is emitted further up the stack. - Static type checker behavior is not affected by the *category* - and *stacklevel* arguments. - - The deprecation message passed to the decorator is saved in the - ``__deprecated__`` attribute on the decorated object. - If applied to an overload, the decorator - must be after the ``@overload`` decorator for the attribute to - exist on the overload as returned by ``get_overloads()``. - - See PEP 702 for details. - - """ - def __init__( - self, - message: str, - /, - *, - category: typing.Optional[typing.Type[Warning]] = DeprecationWarning, - stacklevel: int = 1, - ) -> None: - if not isinstance(message, str): - raise TypeError( - "Expected an object of type str for 'message', not " - f"{type(message).__name__!r}" - ) - self.message = message - self.category = category - self.stacklevel = stacklevel - - def __call__(self, arg: _T, /) -> _T: - # Make sure the inner functions created below don't - # retain a reference to self. - msg = self.message - category = self.category - stacklevel = self.stacklevel - if category is None: - arg.__deprecated__ = msg - return arg - elif isinstance(arg, type): - import functools - from types import MethodType - - original_new = arg.__new__ - - @functools.wraps(original_new) - def __new__(cls, /, *args, **kwargs): - if cls is arg: - warnings.warn(msg, category=category, stacklevel=stacklevel + 1) - if original_new is not object.__new__: - return original_new(cls, *args, **kwargs) - # Mirrors a similar check in object.__new__. - elif cls.__init__ is object.__init__ and (args or kwargs): - raise TypeError(f"{cls.__name__}() takes no arguments") - else: - return original_new(cls) - - arg.__new__ = staticmethod(__new__) - - original_init_subclass = arg.__init_subclass__ - # We need slightly different behavior if __init_subclass__ - # is a bound method (likely if it was implemented in Python) - if isinstance(original_init_subclass, MethodType): - original_init_subclass = original_init_subclass.__func__ - - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): - warnings.warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = classmethod(__init_subclass__) - # Or otherwise, which likely means it's a builtin such as - # object's implementation of __init_subclass__. - else: - @functools.wraps(original_init_subclass) - def __init_subclass__(*args, **kwargs): - warnings.warn(msg, category=category, stacklevel=stacklevel + 1) - return original_init_subclass(*args, **kwargs) - - arg.__init_subclass__ = __init_subclass__ - - arg.__deprecated__ = __new__.__deprecated__ = msg - __init_subclass__.__deprecated__ = msg - return arg - elif callable(arg): - import asyncio.coroutines - import functools - import inspect - - @functools.wraps(arg) - def wrapper(*args, **kwargs): - warnings.warn(msg, category=category, stacklevel=stacklevel + 1) - return arg(*args, **kwargs) - - if asyncio.coroutines.iscoroutinefunction(arg): - # Breakpoint: https://github.com/python/cpython/pull/99247 - if sys.version_info >= (3, 12): - wrapper = inspect.markcoroutinefunction(wrapper) - else: - wrapper._is_coroutine = asyncio.coroutines._is_coroutine - - arg.__deprecated__ = wrapper.__deprecated__ = msg - return wrapper - else: - raise TypeError( - "@deprecated decorator with non-None category must be applied to " - f"a class or callable, not {arg!r}" - ) - -# Breakpoint: https://github.com/python/cpython/pull/23702 -if sys.version_info < (3, 10): - def _is_param_expr(arg): - return arg is ... or isinstance( - arg, (tuple, list, ParamSpec, _ConcatenateGenericAlias) - ) -else: - def _is_param_expr(arg): - return arg is ... or isinstance( - arg, - ( - tuple, - list, - ParamSpec, - _ConcatenateGenericAlias, - typing._ConcatenateGenericAlias, - ), - ) - - -# We have to do some monkey patching to deal with the dual nature of -# Unpack/TypeVarTuple: -# - We want Unpack to be a kind of TypeVar so it gets accepted in -# Generic[Unpack[Ts]] -# - We want it to *not* be treated as a TypeVar for the purposes of -# counting generic parameters, so that when we subscript a generic, -# the runtime doesn't try to substitute the Unpack with the subscripted type. -if not hasattr(typing, "TypeVarTuple"): - def _check_generic(cls, parameters, elen=_marker): - """Check correct count for parameters of a generic cls (internal helper). - - This gives a nice error message in case of count mismatch. - """ - # If substituting a single ParamSpec with multiple arguments - # we do not check the count - if (inspect.isclass(cls) and issubclass(cls, typing.Generic) - and len(cls.__parameters__) == 1 - and isinstance(cls.__parameters__[0], ParamSpec) - and parameters - and not _is_param_expr(parameters[0]) - ): - # Generic modifies parameters variable, but here we cannot do this - return - - if not elen: - raise TypeError(f"{cls} is not a generic class") - if elen is _marker: - if not hasattr(cls, "__parameters__") or not cls.__parameters__: - raise TypeError(f"{cls} is not a generic class") - elen = len(cls.__parameters__) - alen = len(parameters) - if alen != elen: - expect_val = elen - if hasattr(cls, "__parameters__"): - parameters = [p for p in cls.__parameters__ if not _is_unpack(p)] - num_tv_tuples = sum(isinstance(p, TypeVarTuple) for p in parameters) - if (num_tv_tuples > 0) and (alen >= elen - num_tv_tuples): - return - - # deal with TypeVarLike defaults - # required TypeVarLikes cannot appear after a defaulted one. - if alen < elen: - # since we validate TypeVarLike default in _collect_type_vars - # or _collect_parameters we can safely check parameters[alen] - if ( - getattr(parameters[alen], '__default__', NoDefault) - is not NoDefault - ): - return - - num_default_tv = sum(getattr(p, '__default__', NoDefault) - is not NoDefault for p in parameters) - - elen -= num_default_tv - - expect_val = f"at least {elen}" - - # Breakpoint: https://github.com/python/cpython/pull/27515 - things = "arguments" if sys.version_info >= (3, 10) else "parameters" - raise TypeError(f"Too {'many' if alen > elen else 'few'} {things}" - f" for {cls}; actual {alen}, expected {expect_val}") -else: - # Python 3.11+ - - def _check_generic(cls, parameters, elen): - """Check correct count for parameters of a generic cls (internal helper). - - This gives a nice error message in case of count mismatch. - """ - if not elen: - raise TypeError(f"{cls} is not a generic class") - alen = len(parameters) - if alen != elen: - expect_val = elen - if hasattr(cls, "__parameters__"): - parameters = [p for p in cls.__parameters__ if not _is_unpack(p)] - - # deal with TypeVarLike defaults - # required TypeVarLikes cannot appear after a defaulted one. - if alen < elen: - # since we validate TypeVarLike default in _collect_type_vars - # or _collect_parameters we can safely check parameters[alen] - if ( - getattr(parameters[alen], '__default__', NoDefault) - is not NoDefault - ): - return - - num_default_tv = sum(getattr(p, '__default__', NoDefault) - is not NoDefault for p in parameters) - - elen -= num_default_tv - - expect_val = f"at least {elen}" - - raise TypeError(f"Too {'many' if alen > elen else 'few'} arguments" - f" for {cls}; actual {alen}, expected {expect_val}") - -if not _PEP_696_IMPLEMENTED: - typing._check_generic = _check_generic - - -def _has_generic_or_protocol_as_origin() -> bool: - try: - frame = sys._getframe(2) - # - Catch AttributeError: not all Python implementations have sys._getframe() - # - Catch ValueError: maybe we're called from an unexpected module - # and the call stack isn't deep enough - except (AttributeError, ValueError): - return False # err on the side of leniency - else: - # If we somehow get invoked from outside typing.py, - # also err on the side of leniency - if frame.f_globals.get("__name__") != "typing": - return False - origin = frame.f_locals.get("origin") - # Cannot use "in" because origin may be an object with a buggy __eq__ that - # throws an error. - return origin is typing.Generic or origin is Protocol or origin is typing.Protocol - - -_TYPEVARTUPLE_TYPES = {TypeVarTuple, getattr(typing, "TypeVarTuple", None)} - - -def _is_unpacked_typevartuple(x) -> bool: - if get_origin(x) is not Unpack: - return False - args = get_args(x) - return ( - bool(args) - and len(args) == 1 - and type(args[0]) in _TYPEVARTUPLE_TYPES - ) - - -# Python 3.11+ _collect_type_vars was renamed to _collect_parameters -if hasattr(typing, '_collect_type_vars'): - def _collect_type_vars(types, typevar_types=None): - """Collect all type variable contained in types in order of - first appearance (lexicographic order). For example:: - - _collect_type_vars((T, List[S, T])) == (T, S) - """ - if typevar_types is None: - typevar_types = typing.TypeVar - tvars = [] - - # A required TypeVarLike cannot appear after a TypeVarLike with a default - # if it was a direct call to `Generic[]` or `Protocol[]` - enforce_default_ordering = _has_generic_or_protocol_as_origin() - default_encountered = False - - # Also, a TypeVarLike with a default cannot appear after a TypeVarTuple - type_var_tuple_encountered = False - - for t in types: - if _is_unpacked_typevartuple(t): - type_var_tuple_encountered = True - elif ( - isinstance(t, typevar_types) and not isinstance(t, _UnpackAlias) - and t not in tvars - ): - if enforce_default_ordering: - has_default = getattr(t, '__default__', NoDefault) is not NoDefault - if has_default: - if type_var_tuple_encountered: - raise TypeError('Type parameter with a default' - ' follows TypeVarTuple') - default_encountered = True - elif default_encountered: - raise TypeError(f'Type parameter {t!r} without a default' - ' follows type parameter with a default') - - tvars.append(t) - if _should_collect_from_parameters(t): - tvars.extend([t for t in t.__parameters__ if t not in tvars]) - elif isinstance(t, tuple): - # Collect nested type_vars - # tuple wrapped by _prepare_paramspec_params(cls, params) - for x in t: - for collected in _collect_type_vars([x]): - if collected not in tvars: - tvars.append(collected) - return tuple(tvars) - - typing._collect_type_vars = _collect_type_vars -else: - def _collect_parameters(args): - """Collect all type variables and parameter specifications in args - in order of first appearance (lexicographic order). - - For example:: - - assert _collect_parameters((T, Callable[P, T])) == (T, P) - """ - parameters = [] - - # A required TypeVarLike cannot appear after a TypeVarLike with default - # if it was a direct call to `Generic[]` or `Protocol[]` - enforce_default_ordering = _has_generic_or_protocol_as_origin() - default_encountered = False - - # Also, a TypeVarLike with a default cannot appear after a TypeVarTuple - type_var_tuple_encountered = False - - for t in args: - if isinstance(t, type): - # We don't want __parameters__ descriptor of a bare Python class. - pass - elif isinstance(t, tuple): - # `t` might be a tuple, when `ParamSpec` is substituted with - # `[T, int]`, or `[int, *Ts]`, etc. - for x in t: - for collected in _collect_parameters([x]): - if collected not in parameters: - parameters.append(collected) - elif hasattr(t, '__typing_subst__'): - if t not in parameters: - if enforce_default_ordering: - has_default = ( - getattr(t, '__default__', NoDefault) is not NoDefault - ) - - if type_var_tuple_encountered and has_default: - raise TypeError('Type parameter with a default' - ' follows TypeVarTuple') - - if has_default: - default_encountered = True - elif default_encountered: - raise TypeError(f'Type parameter {t!r} without a default' - ' follows type parameter with a default') - - parameters.append(t) - else: - if _is_unpacked_typevartuple(t): - type_var_tuple_encountered = True - for x in getattr(t, '__parameters__', ()): - if x not in parameters: - parameters.append(x) - - return tuple(parameters) - - if not _PEP_696_IMPLEMENTED: - typing._collect_parameters = _collect_parameters - -# Backport typing.NamedTuple as it exists in Python 3.13. -# In 3.11, the ability to define generic `NamedTuple`s was supported. -# This was explicitly disallowed in 3.9-3.10, and only half-worked in <=3.8. -# On 3.12, we added __orig_bases__ to call-based NamedTuples -# On 3.13, we deprecated kwargs-based NamedTuples -# Breakpoint: https://github.com/python/cpython/pull/105609 -if sys.version_info >= (3, 13): - NamedTuple = typing.NamedTuple -else: - def _make_nmtuple(name, types, module, defaults=()): - fields = [n for n, t in types] - annotations = {n: typing._type_check(t, f"field {n} annotation must be a type") - for n, t in types} - nm_tpl = collections.namedtuple(name, fields, - defaults=defaults, module=module) - nm_tpl.__annotations__ = nm_tpl.__new__.__annotations__ = annotations - return nm_tpl - - _prohibited_namedtuple_fields = typing._prohibited - _special_namedtuple_fields = frozenset({'__module__', '__name__', '__annotations__'}) - - class _NamedTupleMeta(type): - def __new__(cls, typename, bases, ns): - assert _NamedTuple in bases - for base in bases: - if base is not _NamedTuple and base is not typing.Generic: - raise TypeError( - 'can only inherit from a NamedTuple type and Generic') - bases = tuple(tuple if base is _NamedTuple else base for base in bases) - if "__annotations__" in ns: - types = ns["__annotations__"] - elif "__annotate__" in ns: - # TODO: Use inspect.VALUE here, and make the annotations lazily evaluated - types = ns["__annotate__"](1) - else: - types = {} - default_names = [] - for field_name in types: - if field_name in ns: - default_names.append(field_name) - elif default_names: - raise TypeError(f"Non-default namedtuple field {field_name} " - f"cannot follow default field" - f"{'s' if len(default_names) > 1 else ''} " - f"{', '.join(default_names)}") - nm_tpl = _make_nmtuple( - typename, types.items(), - defaults=[ns[n] for n in default_names], - module=ns['__module__'] - ) - nm_tpl.__bases__ = bases - if typing.Generic in bases: - if hasattr(typing, '_generic_class_getitem'): # 3.12+ - nm_tpl.__class_getitem__ = classmethod(typing._generic_class_getitem) - else: - class_getitem = typing.Generic.__class_getitem__.__func__ - nm_tpl.__class_getitem__ = classmethod(class_getitem) - # update from user namespace without overriding special namedtuple attributes - for key, val in ns.items(): - if key in _prohibited_namedtuple_fields: - raise AttributeError("Cannot overwrite NamedTuple attribute " + key) - elif key not in _special_namedtuple_fields: - if key not in nm_tpl._fields: - setattr(nm_tpl, key, ns[key]) - try: - set_name = type(val).__set_name__ - except AttributeError: - pass - else: - try: - set_name(val, nm_tpl, key) - except BaseException as e: - msg = ( - f"Error calling __set_name__ on {type(val).__name__!r} " - f"instance {key!r} in {typename!r}" - ) - # BaseException.add_note() existed on py311, - # but the __set_name__ machinery didn't start - # using add_note() until py312. - # Making sure exceptions are raised in the same way - # as in "normal" classes seems most important here. - # Breakpoint: https://github.com/python/cpython/pull/95915 - if sys.version_info >= (3, 12): - e.add_note(msg) - raise - else: - raise RuntimeError(msg) from e - - if typing.Generic in bases: - nm_tpl.__init_subclass__() - return nm_tpl - - _NamedTuple = type.__new__(_NamedTupleMeta, 'NamedTuple', (), {}) - - def _namedtuple_mro_entries(bases): - assert NamedTuple in bases - return (_NamedTuple,) - - def NamedTuple(typename, fields=_marker, /, **kwargs): - """Typed version of namedtuple. - - Usage:: - - class Employee(NamedTuple): - name: str - id: int - - This is equivalent to:: - - Employee = collections.namedtuple('Employee', ['name', 'id']) - - The resulting class has an extra __annotations__ attribute, giving a - dict that maps field names to types. (The field names are also in - the _fields attribute, which is part of the namedtuple API.) - An alternative equivalent functional syntax is also accepted:: - - Employee = NamedTuple('Employee', [('name', str), ('id', int)]) - """ - if fields is _marker: - if kwargs: - deprecated_thing = "Creating NamedTuple classes using keyword arguments" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "Use the class-based or functional syntax instead." - ) - else: - deprecated_thing = "Failing to pass a value for the 'fields' parameter" - example = f"`{typename} = NamedTuple({typename!r}, [])`" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "To create a NamedTuple class with 0 fields " - "using the functional syntax, " - "pass an empty list, e.g. " - ) + example + "." - elif fields is None: - if kwargs: - raise TypeError( - "Cannot pass `None` as the 'fields' parameter " - "and also specify fields using keyword arguments" - ) - else: - deprecated_thing = "Passing `None` as the 'fields' parameter" - example = f"`{typename} = NamedTuple({typename!r}, [])`" - deprecation_msg = ( - "{name} is deprecated and will be disallowed in Python {remove}. " - "To create a NamedTuple class with 0 fields " - "using the functional syntax, " - "pass an empty list, e.g. " - ) + example + "." - elif kwargs: - raise TypeError("Either list of fields or keywords" - " can be provided to NamedTuple, not both") - if fields is _marker or fields is None: - warnings.warn( - deprecation_msg.format(name=deprecated_thing, remove="3.15"), - DeprecationWarning, - stacklevel=2, - ) - fields = kwargs.items() - nt = _make_nmtuple(typename, fields, module=_caller()) - nt.__orig_bases__ = (NamedTuple,) - return nt - - NamedTuple.__mro_entries__ = _namedtuple_mro_entries - - -if hasattr(collections.abc, "Buffer"): - Buffer = collections.abc.Buffer -else: - class Buffer(abc.ABC): # noqa: B024 - """Base class for classes that implement the buffer protocol. - - The buffer protocol allows Python objects to expose a low-level - memory buffer interface. Before Python 3.12, it is not possible - to implement the buffer protocol in pure Python code, or even - to check whether a class implements the buffer protocol. In - Python 3.12 and higher, the ``__buffer__`` method allows access - to the buffer protocol from Python code, and the - ``collections.abc.Buffer`` ABC allows checking whether a class - implements the buffer protocol. - - To indicate support for the buffer protocol in earlier versions, - inherit from this ABC, either in a stub file or at runtime, - or use ABC registration. This ABC provides no methods, because - there is no Python-accessible methods shared by pre-3.12 buffer - classes. It is useful primarily for static checks. - - """ - - # As a courtesy, register the most common stdlib buffer classes. - Buffer.register(memoryview) - Buffer.register(bytearray) - Buffer.register(bytes) - - -# Backport of types.get_original_bases, available on 3.12+ in CPython -if hasattr(_types, "get_original_bases"): - get_original_bases = _types.get_original_bases -else: - def get_original_bases(cls, /): - """Return the class's "original" bases prior to modification by `__mro_entries__`. - - Examples:: - - from typing import TypeVar, Generic - from typing_extensions import NamedTuple, TypedDict - - T = TypeVar("T") - class Foo(Generic[T]): ... - class Bar(Foo[int], float): ... - class Baz(list[str]): ... - Eggs = NamedTuple("Eggs", [("a", int), ("b", str)]) - Spam = TypedDict("Spam", {"a": int, "b": str}) - - assert get_original_bases(Bar) == (Foo[int], float) - assert get_original_bases(Baz) == (list[str],) - assert get_original_bases(Eggs) == (NamedTuple,) - assert get_original_bases(Spam) == (TypedDict,) - assert get_original_bases(int) == (object,) - """ - try: - return cls.__dict__.get("__orig_bases__", cls.__bases__) - except AttributeError: - raise TypeError( - f'Expected an instance of type, not {type(cls).__name__!r}' - ) from None - - -# NewType is a class on Python 3.10+, making it pickleable -# The error message for subclassing instances of NewType was improved on 3.11+ -# Breakpoint: https://github.com/python/cpython/pull/30268 -if sys.version_info >= (3, 11): - NewType = typing.NewType -else: - class NewType: - """NewType creates simple unique types with almost zero - runtime overhead. NewType(name, tp) is considered a subtype of tp - by static type checkers. At runtime, NewType(name, tp) returns - a dummy callable that simply returns its argument. Usage:: - UserId = NewType('UserId', int) - def name_by_id(user_id: UserId) -> str: - ... - UserId('user') # Fails type check - name_by_id(42) # Fails type check - name_by_id(UserId(42)) # OK - num = UserId(5) + 1 # type: int - """ - - def __call__(self, obj, /): - return obj - - def __init__(self, name, tp): - self.__qualname__ = name - if '.' in name: - name = name.rpartition('.')[-1] - self.__name__ = name - self.__supertype__ = tp - def_mod = _caller() - if def_mod != 'typing_extensions': - self.__module__ = def_mod - - def __mro_entries__(self, bases): - # We defined __mro_entries__ to get a better error message - # if a user attempts to subclass a NewType instance. bpo-46170 - supercls_name = self.__name__ - - class Dummy: - def __init_subclass__(cls): - subcls_name = cls.__name__ - raise TypeError( - f"Cannot subclass an instance of NewType. " - f"Perhaps you were looking for: " - f"`{subcls_name} = NewType({subcls_name!r}, {supercls_name})`" - ) - - return (Dummy,) - - def __repr__(self): - return f'{self.__module__}.{self.__qualname__}' - - def __reduce__(self): - return self.__qualname__ - - # Breakpoint: https://github.com/python/cpython/pull/21515 - if sys.version_info >= (3, 10): - # PEP 604 methods - # It doesn't make sense to have these methods on Python <3.10 - - def __or__(self, other): - return typing.Union[self, other] - - def __ror__(self, other): - return typing.Union[other, self] - - -# Breakpoint: https://github.com/python/cpython/pull/124795 -if sys.version_info >= (3, 14): - TypeAliasType = typing.TypeAliasType -# <=3.13 -else: - # Breakpoint: https://github.com/python/cpython/pull/103764 - if sys.version_info >= (3, 12): - # 3.12-3.13 - def _is_unionable(obj): - """Corresponds to is_unionable() in unionobject.c in CPython.""" - return obj is None or isinstance(obj, ( - type, - _types.GenericAlias, - _types.UnionType, - typing.TypeAliasType, - TypeAliasType, - )) - else: - # <=3.11 - def _is_unionable(obj): - """Corresponds to is_unionable() in unionobject.c in CPython.""" - return obj is None or isinstance(obj, ( - type, - _types.GenericAlias, - _types.UnionType, - TypeAliasType, - )) - - if sys.version_info < (3, 10): - # Copied and pasted from https://github.com/python/cpython/blob/986a4e1b6fcae7fe7a1d0a26aea446107dd58dd2/Objects/genericaliasobject.c#L568-L582, - # so that we emulate the behaviour of `types.GenericAlias` - # on the latest versions of CPython - _ATTRIBUTE_DELEGATION_EXCLUSIONS = frozenset({ - "__class__", - "__bases__", - "__origin__", - "__args__", - "__unpacked__", - "__parameters__", - "__typing_unpacked_tuple_args__", - "__mro_entries__", - "__reduce_ex__", - "__reduce__", - "__copy__", - "__deepcopy__", - }) - - class _TypeAliasGenericAlias(typing._GenericAlias, _root=True): - def __getattr__(self, attr): - if attr in _ATTRIBUTE_DELEGATION_EXCLUSIONS: - return object.__getattr__(self, attr) - return getattr(self.__origin__, attr) - - - class TypeAliasType: - """Create named, parameterized type aliases. - - This provides a backport of the new `type` statement in Python 3.12: - - type ListOrSet[T] = list[T] | set[T] - - is equivalent to: - - T = TypeVar("T") - ListOrSet = TypeAliasType("ListOrSet", list[T] | set[T], type_params=(T,)) - - The name ListOrSet can then be used as an alias for the type it refers to. - - The type_params argument should contain all the type parameters used - in the value of the type alias. If the alias is not generic, this - argument is omitted. - - Static type checkers should only support type aliases declared using - TypeAliasType that follow these rules: - - - The first argument (the name) must be a string literal. - - The TypeAliasType instance must be immediately assigned to a variable - of the same name. (For example, 'X = TypeAliasType("Y", int)' is invalid, - as is 'X, Y = TypeAliasType("X", int), TypeAliasType("Y", int)'). - - """ - - def __init__(self, name: str, value, *, type_params=()): - if not isinstance(name, str): - raise TypeError("TypeAliasType name must be a string") - if not isinstance(type_params, tuple): - raise TypeError("type_params must be a tuple") - self.__value__ = value - self.__type_params__ = type_params - - default_value_encountered = False - parameters = [] - for type_param in type_params: - if ( - not isinstance(type_param, (TypeVar, TypeVarTuple, ParamSpec)) - # <=3.11 - # Unpack Backport passes isinstance(type_param, TypeVar) - or _is_unpack(type_param) - ): - raise TypeError(f"Expected a type param, got {type_param!r}") - has_default = ( - getattr(type_param, '__default__', NoDefault) is not NoDefault - ) - if default_value_encountered and not has_default: - raise TypeError(f"non-default type parameter '{type_param!r}'" - " follows default type parameter") - if has_default: - default_value_encountered = True - if isinstance(type_param, TypeVarTuple): - parameters.extend(type_param) - else: - parameters.append(type_param) - self.__parameters__ = tuple(parameters) - def_mod = _caller() - if def_mod != 'typing_extensions': - self.__module__ = def_mod - # Setting this attribute closes the TypeAliasType from further modification - self.__name__ = name - - def __setattr__(self, name: str, value: object, /) -> None: - if hasattr(self, "__name__"): - self._raise_attribute_error(name) - super().__setattr__(name, value) - - def __delattr__(self, name: str, /) -> Never: - self._raise_attribute_error(name) - - def _raise_attribute_error(self, name: str) -> Never: - # Match the Python 3.12 error messages exactly - if name == "__name__": - raise AttributeError("readonly attribute") - elif name in {"__value__", "__type_params__", "__parameters__", "__module__"}: - raise AttributeError( - f"attribute '{name}' of 'typing.TypeAliasType' objects " - "is not writable" - ) - else: - raise AttributeError( - f"'typing.TypeAliasType' object has no attribute '{name}'" - ) - - def __repr__(self) -> str: - return self.__name__ - - if sys.version_info < (3, 11): - def _check_single_param(self, param, recursion=0): - # Allow [], [int], [int, str], [int, ...], [int, T] - if param is ...: - return ... - if param is None: - return None - # Note in <= 3.9 _ConcatenateGenericAlias inherits from list - if isinstance(param, list) and recursion == 0: - return [self._check_single_param(arg, recursion+1) - for arg in param] - return typing._type_check( - param, f'Subscripting {self.__name__} requires a type.' - ) - - def _check_parameters(self, parameters): - if sys.version_info < (3, 11): - return tuple( - self._check_single_param(item) - for item in parameters - ) - return tuple(typing._type_check( - item, f'Subscripting {self.__name__} requires a type.' - ) - for item in parameters - ) - - def __getitem__(self, parameters): - if not self.__type_params__: - raise TypeError("Only generic type aliases are subscriptable") - if not isinstance(parameters, tuple): - parameters = (parameters,) - # Using 3.9 here will create problems with Concatenate - if sys.version_info >= (3, 10): - return _types.GenericAlias(self, parameters) - type_vars = _collect_type_vars(parameters) - parameters = self._check_parameters(parameters) - alias = _TypeAliasGenericAlias(self, parameters) - # alias.__parameters__ is not complete if Concatenate is present - # as it is converted to a list from which no parameters are extracted. - if alias.__parameters__ != type_vars: - alias.__parameters__ = type_vars - return alias - - def __reduce__(self): - return self.__name__ - - def __init_subclass__(cls, *args, **kwargs): - raise TypeError( - "type 'typing_extensions.TypeAliasType' is not an acceptable base type" - ) - - # The presence of this method convinces typing._type_check - # that TypeAliasTypes are types. - def __call__(self): - raise TypeError("Type alias is not callable") - - # Breakpoint: https://github.com/python/cpython/pull/21515 - if sys.version_info >= (3, 10): - def __or__(self, right): - # For forward compatibility with 3.12, reject Unions - # that are not accepted by the built-in Union. - if not _is_unionable(right): - return NotImplemented - return typing.Union[self, right] - - def __ror__(self, left): - if not _is_unionable(left): - return NotImplemented - return typing.Union[left, self] - - -if hasattr(typing, "is_protocol"): - is_protocol = typing.is_protocol - get_protocol_members = typing.get_protocol_members -else: - def is_protocol(tp: type, /) -> bool: - """Return True if the given type is a Protocol. - - Example:: - - >>> from typing_extensions import Protocol, is_protocol - >>> class P(Protocol): - ... def a(self) -> str: ... - ... b: int - >>> is_protocol(P) - True - >>> is_protocol(int) - False - """ - return ( - isinstance(tp, type) - and getattr(tp, '_is_protocol', False) - and tp is not Protocol - and tp is not typing.Protocol - ) - - def get_protocol_members(tp: type, /) -> typing.FrozenSet[str]: - """Return the set of members defined in a Protocol. - - Example:: - - >>> from typing_extensions import Protocol, get_protocol_members - >>> class P(Protocol): - ... def a(self) -> str: ... - ... b: int - >>> get_protocol_members(P) - frozenset({'a', 'b'}) - - Raise a TypeError for arguments that are not Protocols. - """ - if not is_protocol(tp): - raise TypeError(f'{tp!r} is not a Protocol') - if hasattr(tp, '__protocol_attrs__'): - return frozenset(tp.__protocol_attrs__) - return frozenset(_get_protocol_attrs(tp)) - - -if hasattr(typing, "Doc"): - Doc = typing.Doc -else: - class Doc: - """Define the documentation of a type annotation using ``Annotated``, to be - used in class attributes, function and method parameters, return values, - and variables. - - The value should be a positional-only string literal to allow static tools - like editors and documentation generators to use it. - - This complements docstrings. - - The string value passed is available in the attribute ``documentation``. - - Example:: - - >>> from typing_extensions import Annotated, Doc - >>> def hi(to: Annotated[str, Doc("Who to say hi to")]) -> None: ... - """ - def __init__(self, documentation: str, /) -> None: - self.documentation = documentation - - def __repr__(self) -> str: - return f"Doc({self.documentation!r})" - - def __hash__(self) -> int: - return hash(self.documentation) - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Doc): - return NotImplemented - return self.documentation == other.documentation - - -_CapsuleType = getattr(_types, "CapsuleType", None) - -if _CapsuleType is None: - try: - import _socket - except ImportError: - pass - else: - _CAPI = getattr(_socket, "CAPI", None) - if _CAPI is not None: - _CapsuleType = type(_CAPI) - -if _CapsuleType is not None: - CapsuleType = _CapsuleType - __all__.append("CapsuleType") - - -if sys.version_info >= (3, 14): - from annotationlib import Format, get_annotations -else: - # Available since Python 3.14.0a3 - # PR: https://github.com/python/cpython/pull/124415 - class Format(enum.IntEnum): - VALUE = 1 - VALUE_WITH_FAKE_GLOBALS = 2 - FORWARDREF = 3 - STRING = 4 - - # Available since Python 3.14.0a1 - # PR: https://github.com/python/cpython/pull/119891 - def get_annotations(obj, *, globals=None, locals=None, eval_str=False, - format=Format.VALUE): - """Compute the annotations dict for an object. - - obj may be a callable, class, or module. - Passing in an object of any other type raises TypeError. - - Returns a dict. get_annotations() returns a new dict every time - it's called; calling it twice on the same object will return two - different but equivalent dicts. - - This is a backport of `inspect.get_annotations`, which has been - in the standard library since Python 3.10. See the standard library - documentation for more: - - https://docs.python.org/3/library/inspect.html#inspect.get_annotations - - This backport adds the *format* argument introduced by PEP 649. The - three formats supported are: - * VALUE: the annotations are returned as-is. This is the default and - it is compatible with the behavior on previous Python versions. - * FORWARDREF: return annotations as-is if possible, but replace any - undefined names with ForwardRef objects. The implementation proposed by - PEP 649 relies on language changes that cannot be backported; the - typing-extensions implementation simply returns the same result as VALUE. - * STRING: return annotations as strings, in a format close to the original - source. Again, this behavior cannot be replicated directly in a backport. - As an approximation, typing-extensions retrieves the annotations under - VALUE semantics and then stringifies them. - - The purpose of this backport is to allow users who would like to use - FORWARDREF or STRING semantics once PEP 649 is implemented, but who also - want to support earlier Python versions, to simply write: - - typing_extensions.get_annotations(obj, format=Format.FORWARDREF) - - """ - format = Format(format) - if format is Format.VALUE_WITH_FAKE_GLOBALS: - raise ValueError( - "The VALUE_WITH_FAKE_GLOBALS format is for internal use only" - ) - - if eval_str and format is not Format.VALUE: - raise ValueError("eval_str=True is only supported with format=Format.VALUE") - - if isinstance(obj, type): - # class - obj_dict = getattr(obj, '__dict__', None) - if obj_dict and hasattr(obj_dict, 'get'): - ann = obj_dict.get('__annotations__', None) - if isinstance(ann, _types.GetSetDescriptorType): - ann = None - else: - ann = None - - obj_globals = None - module_name = getattr(obj, '__module__', None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - obj_globals = getattr(module, '__dict__', None) - obj_locals = dict(vars(obj)) - unwrap = obj - elif isinstance(obj, _types.ModuleType): - # module - ann = getattr(obj, '__annotations__', None) - obj_globals = obj.__dict__ - obj_locals = None - unwrap = None - elif callable(obj): - # this includes types.Function, types.BuiltinFunctionType, - # types.BuiltinMethodType, functools.partial, functools.singledispatch, - # "class funclike" from Lib/test/test_inspect... on and on it goes. - ann = getattr(obj, '__annotations__', None) - obj_globals = getattr(obj, '__globals__', None) - obj_locals = None - unwrap = obj - elif hasattr(obj, '__annotations__'): - ann = obj.__annotations__ - obj_globals = obj_locals = unwrap = None - else: - raise TypeError(f"{obj!r} is not a module, class, or callable.") - - if ann is None: - return {} - - if not isinstance(ann, dict): - raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") - - if not ann: - return {} - - if not eval_str: - if format is Format.STRING: - return { - key: value if isinstance(value, str) else typing._type_repr(value) - for key, value in ann.items() - } - return dict(ann) - - if unwrap is not None: - while True: - if hasattr(unwrap, '__wrapped__'): - unwrap = unwrap.__wrapped__ - continue - if isinstance(unwrap, functools.partial): - unwrap = unwrap.func - continue - break - if hasattr(unwrap, "__globals__"): - obj_globals = unwrap.__globals__ - - if globals is None: - globals = obj_globals - if locals is None: - locals = obj_locals or {} - - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - if type_params := getattr(obj, "__type_params__", ()): - locals = {param.__name__: param for param in type_params} | locals - - return_value = {key: - value if not isinstance(value, str) else eval(value, globals, locals) - for key, value in ann.items() } - return return_value - - -if hasattr(typing, "evaluate_forward_ref"): - evaluate_forward_ref = typing.evaluate_forward_ref -else: - # Implements annotationlib.ForwardRef.evaluate - def _eval_with_owner( - forward_ref, *, owner=None, globals=None, locals=None, type_params=None - ): - if forward_ref.__forward_evaluated__: - return forward_ref.__forward_value__ - if getattr(forward_ref, "__cell__", None) is not None: - try: - value = forward_ref.__cell__.cell_contents - except ValueError: - pass - else: - forward_ref.__forward_evaluated__ = True - forward_ref.__forward_value__ = value - return value - if owner is None: - owner = getattr(forward_ref, "__owner__", None) - - if ( - globals is None - and getattr(forward_ref, "__forward_module__", None) is not None - ): - globals = getattr( - sys.modules.get(forward_ref.__forward_module__, None), "__dict__", None - ) - if globals is None: - globals = getattr(forward_ref, "__globals__", None) - if globals is None: - if isinstance(owner, type): - module_name = getattr(owner, "__module__", None) - if module_name: - module = sys.modules.get(module_name, None) - if module: - globals = getattr(module, "__dict__", None) - elif isinstance(owner, _types.ModuleType): - globals = getattr(owner, "__dict__", None) - elif callable(owner): - globals = getattr(owner, "__globals__", None) - - # If we pass None to eval() below, the globals of this module are used. - if globals is None: - globals = {} - - if locals is None: - locals = {} - if isinstance(owner, type): - locals.update(vars(owner)) - - if type_params is None and owner is not None: - # "Inject" type parameters into the local namespace - # (unless they are shadowed by assignments *in* the local namespace), - # as a way of emulating annotation scopes when calling `eval()` - type_params = getattr(owner, "__type_params__", None) - - # Type parameters exist in their own scope, which is logically - # between the locals and the globals. We simulate this by adding - # them to the globals. - if type_params is not None: - globals = dict(globals) - for param in type_params: - globals[param.__name__] = param - - arg = forward_ref.__forward_arg__ - if arg.isidentifier() and not keyword.iskeyword(arg): - if arg in locals: - value = locals[arg] - elif arg in globals: - value = globals[arg] - elif hasattr(builtins, arg): - return getattr(builtins, arg) - else: - raise NameError(arg) - else: - code = forward_ref.__forward_code__ - value = eval(code, globals, locals) - forward_ref.__forward_evaluated__ = True - forward_ref.__forward_value__ = value - return value - - def evaluate_forward_ref( - forward_ref, - *, - owner=None, - globals=None, - locals=None, - type_params=None, - format=None, - _recursive_guard=frozenset(), - ): - """Evaluate a forward reference as a type hint. - - This is similar to calling the ForwardRef.evaluate() method, - but unlike that method, evaluate_forward_ref() also: - - * Recursively evaluates forward references nested within the type hint. - * Rejects certain objects that are not valid type hints. - * Replaces type hints that evaluate to None with types.NoneType. - * Supports the *FORWARDREF* and *STRING* formats. - - *forward_ref* must be an instance of ForwardRef. *owner*, if given, - should be the object that holds the annotations that the forward reference - derived from, such as a module, class object, or function. It is used to - infer the namespaces to use for looking up names. *globals* and *locals* - can also be explicitly given to provide the global and local namespaces. - *type_params* is a tuple of type parameters that are in scope when - evaluating the forward reference. This parameter must be provided (though - it may be an empty tuple) if *owner* is not given and the forward reference - does not already have an owner set. *format* specifies the format of the - annotation and is a member of the annotationlib.Format enum. - - """ - if format == Format.STRING: - return forward_ref.__forward_arg__ - if forward_ref.__forward_arg__ in _recursive_guard: - return forward_ref - - # Evaluate the forward reference - try: - value = _eval_with_owner( - forward_ref, - owner=owner, - globals=globals, - locals=locals, - type_params=type_params, - ) - except NameError: - if format == Format.FORWARDREF: - return forward_ref - else: - raise - - if isinstance(value, str): - value = ForwardRef(value) - - # Recursively evaluate the type - if isinstance(value, ForwardRef): - if getattr(value, "__forward_module__", True) is not None: - globals = None - return evaluate_forward_ref( - value, - globals=globals, - locals=locals, - type_params=type_params, owner=owner, - _recursive_guard=_recursive_guard, format=format - ) - if sys.version_info < (3, 12, 5) and type_params: - # Make use of type_params - locals = dict(locals) if locals else {} - for tvar in type_params: - if tvar.__name__ not in locals: # lets not overwrite something present - locals[tvar.__name__] = tvar - if sys.version_info < (3, 12, 5): - return typing._eval_type( - value, - globals, - locals, - recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, - ) - else: - return typing._eval_type( - value, - globals, - locals, - type_params, - recursive_guard=_recursive_guard | {forward_ref.__forward_arg__}, - ) - - -class Sentinel: - """Create a unique sentinel object. - - *name* should be the name of the variable to which the return value shall be assigned. - - *repr*, if supplied, will be used for the repr of the sentinel object. - If not provided, "" will be used. - """ - - def __init__( - self, - name: str, - repr: typing.Optional[str] = None, - ): - self._name = name - self._repr = repr if repr is not None else f'<{name}>' - - def __repr__(self): - return self._repr - - if sys.version_info < (3, 11): - # The presence of this method convinces typing._type_check - # that Sentinels are types. - def __call__(self, *args, **kwargs): - raise TypeError(f"{type(self).__name__!r} object is not callable") - - # Breakpoint: https://github.com/python/cpython/pull/21515 - if sys.version_info >= (3, 10): - def __or__(self, other): - return typing.Union[self, other] - - def __ror__(self, other): - return typing.Union[other, self] - - def __getstate__(self): - raise TypeError(f"Cannot pickle {type(self).__name__!r} object") - - -if sys.version_info >= (3, 14, 0, "beta"): - type_repr = annotationlib.type_repr -else: - def type_repr(value): - """Convert a Python value to a format suitable for use with the STRING format. - - This is intended as a helper for tools that support the STRING format but do - not have access to the code that originally produced the annotations. It uses - repr() for most objects. - - """ - if isinstance(value, (type, _types.FunctionType, _types.BuiltinFunctionType)): - if value.__module__ == "builtins": - return value.__qualname__ - return f"{value.__module__}.{value.__qualname__}" - if value is ...: - return "..." - return repr(value) - - -# Aliases for items that are in typing in all supported versions. -# We use hasattr() checks so this library will continue to import on -# future versions of Python that may remove these names. -_typing_names = [ - "AbstractSet", - "AnyStr", - "BinaryIO", - "Callable", - "Collection", - "Container", - "Dict", - "FrozenSet", - "Hashable", - "IO", - "ItemsView", - "Iterable", - "Iterator", - "KeysView", - "List", - "Mapping", - "MappingView", - "Match", - "MutableMapping", - "MutableSequence", - "MutableSet", - "Optional", - "Pattern", - "Reversible", - "Sequence", - "Set", - "Sized", - "TextIO", - "Tuple", - "Union", - "ValuesView", - "cast", - "no_type_check", - "no_type_check_decorator", - # This is private, but it was defined by typing_extensions for a long time - # and some users rely on it. - "_AnnotatedAlias", -] -globals().update( - {name: getattr(typing, name) for name in _typing_names if hasattr(typing, name)} -) -# These are defined unconditionally because they are used in -# typing-extensions itself. -Generic = typing.Generic -ForwardRef = typing.ForwardRef -Annotated = typing.Annotated diff --git a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/INSTALLER b/functions/python/data-processor/typing_inspection-0.4.2.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/METADATA b/functions/python/data-processor/typing_inspection-0.4.2.dist-info/METADATA deleted file mode 100644 index b0eba85d3..000000000 --- a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/METADATA +++ /dev/null @@ -1,49 +0,0 @@ -Metadata-Version: 2.4 -Name: typing-inspection -Version: 0.4.2 -Summary: Runtime typing introspection tools -Project-URL: Homepage, https://github.com/pydantic/typing-inspection -Project-URL: Documentation, https://pydantic.github.io/typing-inspection/dev/ -Project-URL: Source, https://github.com/pydantic/typing-inspection -Project-URL: Changelog, https://github.com/pydantic/typing-inspection/blob/main/HISTORY.md -Author-email: Victorien Plot -License-Expression: MIT -License-File: LICENSE -Classifier: Development Status :: 3 - Alpha -Classifier: Intended Audience :: Developers -Classifier: Programming Language :: Python -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3 :: Only -Classifier: Programming Language :: Python :: 3.9 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Topic :: Software Development :: Libraries :: Python Modules -Classifier: Typing :: Typed -Requires-Python: >=3.9 -Requires-Dist: typing-extensions>=4.12.0 -Description-Content-Type: text/markdown - -# typing-inspection - -[![CI](https://img.shields.io/github/actions/workflow/status/pydantic/typing-inspection/ci.yml?branch=main&logo=github&label=CI)](https://github.com/pydantic/typing-inspection/actions?query=event%3Apush+branch%3Amain+workflow%3ACI) -[![Coverage](https://coverage-badge.samuelcolvin.workers.dev/pydantic/typing-inspection.svg)](https://coverage-badge.samuelcolvin.workers.dev/redirect/pydantic/typing-inspection) -[![PyPI](https://img.shields.io/pypi/v/typing-inspection.svg)](https://pypi.org/project/typing-inspection/) -[![Versions](https://img.shields.io/pypi/pyversions/typing-inspection.svg)](https://github.com/pydantic/typing-inspection) -[![License](https://img.shields.io/github/license/pydantic/typing-inspection.svg)](https://github.com/pydantic/typing-inspection/blob/main/LICENSE) -[![Ruff](https://img.shields.io/endpoint?url=https://raw.githubusercontent.com/astral-sh/ruff/main/assets/badge/v2.json)](https://github.com/astral-sh/ruff) - -`typing-inspection` provides tools to inspect type annotations at runtime. - -## Installation - -From [PyPI](https://pypi.org/project/typing-inspection/): - -```bash -pip install typing-inspection -``` - -The library can be imported from the `typing_inspection` module. diff --git a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/RECORD b/functions/python/data-processor/typing_inspection-0.4.2.dist-info/RECORD deleted file mode 100644 index bd033dabb..000000000 --- a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/RECORD +++ /dev/null @@ -1,13 +0,0 @@ -typing_inspection-0.4.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -typing_inspection-0.4.2.dist-info/METADATA,sha256=YQls0L_jxwQLb5jCKwLRkP4Bk20P92FsUTT-0CiRlTo,2552 -typing_inspection-0.4.2.dist-info/RECORD,, -typing_inspection-0.4.2.dist-info/WHEEL,sha256=qtCwoSJWgHk21S1Kb4ihdzI2rlJ1ZKaIurTj_ngOhyQ,87 -typing_inspection-0.4.2.dist-info/licenses/LICENSE,sha256=gEtZsl8sMb0nj5ICoZrkmjlFqiZkOH4tChKMfKzGHsM,1090 -typing_inspection/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -typing_inspection/__pycache__/__init__.cpython-314.pyc,, -typing_inspection/__pycache__/introspection.cpython-314.pyc,, -typing_inspection/__pycache__/typing_objects.cpython-314.pyc,, -typing_inspection/introspection.py,sha256=dD5Ad4J6hAfF6UBzBO4sqSs1h2ybQVThkQofLWWVBP0,22534 -typing_inspection/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -typing_inspection/typing_objects.py,sha256=kajVgh8J7UZ7wTidVxzFMpjwSnFGBoDoTqfVAAvOHZ8,17166 -typing_inspection/typing_objects.pyi,sha256=u1NDpl_RJFnAUMAMx-WBd0SBKwVG7luLEc8ukSvRnZs,9401 diff --git a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/WHEEL b/functions/python/data-processor/typing_inspection-0.4.2.dist-info/WHEEL deleted file mode 100644 index 12228d414..000000000 --- a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.27.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/licenses/LICENSE b/functions/python/data-processor/typing_inspection-0.4.2.dist-info/licenses/LICENSE deleted file mode 100644 index e825ad516..000000000 --- a/functions/python/data-processor/typing_inspection-0.4.2.dist-info/licenses/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) Pydantic Services Inc. 2025 to present - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/functions/python/data-processor/typing_inspection/__init__.py b/functions/python/data-processor/typing_inspection/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/typing_inspection/introspection.py b/functions/python/data-processor/typing_inspection/introspection.py deleted file mode 100644 index d6c083e54..000000000 --- a/functions/python/data-processor/typing_inspection/introspection.py +++ /dev/null @@ -1,587 +0,0 @@ -"""High-level introspection utilities, used to inspect type annotations.""" - -from __future__ import annotations - -import sys -import types -from collections.abc import Generator -from dataclasses import InitVar -from enum import Enum, IntEnum, auto -from typing import Any, Literal, NamedTuple, cast - -from typing_extensions import TypeAlias, assert_never, get_args, get_origin - -from . import typing_objects - -__all__ = ( - 'AnnotationSource', - 'ForbiddenQualifier', - 'InspectedAnnotation', - 'Qualifier', - 'get_literal_values', - 'inspect_annotation', - 'is_union_origin', -) - -if sys.version_info >= (3, 14) or sys.version_info < (3, 10): - - def is_union_origin(obj: Any, /) -> bool: - """Return whether the provided origin is the union form. - - ```pycon - >>> is_union_origin(typing.Union) - True - >>> is_union_origin(get_origin(int | str)) - True - >>> is_union_origin(types.UnionType) - True - ``` - - !!! note - Since Python 3.14, both `Union[, , ...]` and ` | | ...` forms create instances - of the same [`typing.Union`][] class. As such, it is recommended to not use this function - anymore (provided that you only support Python 3.14 or greater), and instead use the - [`typing_objects.is_union()`][typing_inspection.typing_objects.is_union] function directly: - - ```python - from typing import Union, get_origin - - from typing_inspection import typing_objects - - typ = int | str # Or Union[int, str] - origin = get_origin(typ) - if typing_objects.is_union(origin): - ... - ``` - """ - return typing_objects.is_union(obj) - - -else: - - def is_union_origin(obj: Any, /) -> bool: - """Return whether the provided origin is the union form. - - ```pycon - >>> is_union_origin(typing.Union) - True - >>> is_union_origin(get_origin(int | str)) - True - >>> is_union_origin(types.UnionType) - True - ``` - - !!! note - Since Python 3.14, both `Union[, , ...]` and ` | | ...` forms create instances - of the same [`typing.Union`][] class. As such, it is recommended to not use this function - anymore (provided that you only support Python 3.14 or greater), and instead use the - [`typing_objects.is_union()`][typing_inspection.typing_objects.is_union] function directly: - - ```python - from typing import Union, get_origin - - from typing_inspection import typing_objects - - typ = int | str # Or Union[int, str] - origin = get_origin(typ) - if typing_objects.is_union(origin): - ... - ``` - """ - return typing_objects.is_union(obj) or obj is types.UnionType - - -def _literal_type_check(value: Any, /) -> None: - """Type check the provided literal value against the legal parameters.""" - if ( - not isinstance(value, (int, bytes, str, bool, Enum, typing_objects.NoneType)) - and value is not typing_objects.NoneType - ): - raise TypeError(f'{value} is not a valid literal value, must be one of: int, bytes, str, Enum, None.') - - -def get_literal_values( - annotation: Any, - /, - *, - type_check: bool = False, - unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'eager', -) -> Generator[Any]: - """Yield the values contained in the provided [`Literal`][typing.Literal] [special form][]. - - Args: - annotation: The [`Literal`][typing.Literal] [special form][] to unpack. - type_check: Whether to check if the literal values are [legal parameters][literal-legal-parameters]. - Raises a [`TypeError`][] otherwise. - unpack_type_aliases: What to do when encountering [PEP 695](https://peps.python.org/pep-0695/) - [type aliases][type-aliases]. Can be one of: - - - `'skip'`: Do not try to parse type aliases. Note that this can lead to incorrect results: - ```pycon - >>> type MyAlias = Literal[1, 2] - >>> list(get_literal_values(Literal[MyAlias, 3], unpack_type_aliases="skip")) - [MyAlias, 3] - ``` - - - `'lenient'`: Try to parse type aliases, and fallback to `'skip'` if the type alias can't be inspected - (because of an undefined forward reference). - - - `'eager'`: Parse type aliases and raise any encountered [`NameError`][] exceptions (the default): - ```pycon - >>> type MyAlias = Literal[1, 2] - >>> list(get_literal_values(Literal[MyAlias, 3], unpack_type_aliases="eager")) - [1, 2, 3] - ``` - - Note: - While `None` is [equivalent to][none] `type(None)`, the runtime implementation of [`Literal`][typing.Literal] - does not de-duplicate them. This function makes sure this de-duplication is applied: - - ```pycon - >>> list(get_literal_values(Literal[NoneType, None])) - [None] - ``` - - Example: - ```pycon - >>> type Ints = Literal[1, 2] - >>> list(get_literal_values(Literal[1, Ints], unpack_type_alias="skip")) - ["a", Ints] - >>> list(get_literal_values(Literal[1, Ints])) - [1, 2] - >>> list(get_literal_values(Literal[1.0], type_check=True)) - Traceback (most recent call last): - ... - TypeError: 1.0 is not a valid literal value, must be one of: int, bytes, str, Enum, None. - ``` - """ - # `literal` is guaranteed to be a `Literal[...]` special form, so use - # `__args__` directly instead of calling `get_args()`. - - if unpack_type_aliases == 'skip': - _has_none = False - # `Literal` parameters are already deduplicated, no need to do it ourselves. - # (we only check for `None` and `NoneType`, which should be considered as duplicates). - for arg in annotation.__args__: - if type_check: - _literal_type_check(arg) - if arg is None or arg is typing_objects.NoneType: - if not _has_none: - yield None - _has_none = True - else: - yield arg - else: - # We'll need to manually deduplicate parameters, see the `Literal` implementation in `typing`. - values_and_type: list[tuple[Any, type[Any]]] = [] - - for arg in annotation.__args__: - # Note: we could also check for generic aliases with a type alias as an origin. - # However, it is very unlikely that this happens as type variables can't appear in - # `Literal` forms, so the only valid (but unnecessary) use case would be something like: - # `type Test[T] = Literal['a']` (and then use `Test[SomeType]`). - if typing_objects.is_typealiastype(arg): - try: - alias_value = arg.__value__ - except NameError: - if unpack_type_aliases == 'eager': - raise - # unpack_type_aliases == "lenient": - if type_check: - _literal_type_check(arg) - values_and_type.append((arg, type(arg))) - else: - sub_args = get_literal_values( - alias_value, type_check=type_check, unpack_type_aliases=unpack_type_aliases - ) - values_and_type.extend((a, type(a)) for a in sub_args) # pyright: ignore[reportUnknownArgumentType] - else: - if type_check: - _literal_type_check(arg) - if arg is typing_objects.NoneType: - values_and_type.append((None, typing_objects.NoneType)) - else: - values_and_type.append((arg, type(arg))) # pyright: ignore[reportUnknownArgumentType] - - try: - dct = dict.fromkeys(values_and_type) - except TypeError: - # Unhashable parameters, the Python implementation allows them - yield from (p for p, _ in values_and_type) - else: - yield from (p for p, _ in dct) - - -Qualifier: TypeAlias = Literal['required', 'not_required', 'read_only', 'class_var', 'init_var', 'final'] -"""A [type qualifier][].""" - -_all_qualifiers: set[Qualifier] = set(get_args(Qualifier)) - - -# TODO at some point, we could switch to an enum flag, so that multiple sources -# can be combined. However, is there a need for this? -class AnnotationSource(IntEnum): - # TODO if/when https://peps.python.org/pep-0767/ is accepted, add 'read_only' - # to CLASS and NAMED_TUPLE (even though for named tuples it is redundant). - - """The source of an annotation, e.g. a class or a function. - - Depending on the source, different [type qualifiers][type qualifier] may be (dis)allowed. - """ - - ASSIGNMENT_OR_VARIABLE = auto() - """An annotation used in an assignment or variable annotation: - - ```python - x: Final[int] = 1 - y: Final[str] - ``` - - **Allowed type qualifiers:** [`Final`][typing.Final]. - """ - - CLASS = auto() - """An annotation used in the body of a class: - - ```python - class Test: - x: Final[int] = 1 - y: ClassVar[str] - ``` - - **Allowed type qualifiers:** [`ClassVar`][typing.ClassVar], [`Final`][typing.Final]. - """ - - DATACLASS = auto() - """An annotation used in the body of a dataclass: - - ```python - @dataclass - class Test: - x: Final[int] = 1 - y: InitVar[str] = 'test' - ``` - - **Allowed type qualifiers:** [`ClassVar`][typing.ClassVar], [`Final`][typing.Final], [`InitVar`][dataclasses.InitVar]. - """ # noqa: E501 - - TYPED_DICT = auto() - """An annotation used in the body of a [`TypedDict`][typing.TypedDict]: - - ```python - class TD(TypedDict): - x: Required[ReadOnly[int]] - y: ReadOnly[NotRequired[str]] - ``` - - **Allowed type qualifiers:** [`ReadOnly`][typing.ReadOnly], [`Required`][typing.Required], - [`NotRequired`][typing.NotRequired]. - """ - - NAMED_TUPLE = auto() - """An annotation used in the body of a [`NamedTuple`][typing.NamedTuple]. - - ```python - class NT(NamedTuple): - x: int - y: str - ``` - - **Allowed type qualifiers:** none. - """ - - FUNCTION = auto() - """An annotation used in a function, either for a parameter or the return value. - - ```python - def func(a: int) -> str: - ... - ``` - - **Allowed type qualifiers:** none. - """ - - ANY = auto() - """An annotation that might come from any source. - - **Allowed type qualifiers:** all. - """ - - BARE = auto() - """An annotation that is inspected as is. - - **Allowed type qualifiers:** none. - """ - - @property - def allowed_qualifiers(self) -> set[Qualifier]: - """The allowed [type qualifiers][type qualifier] for this annotation source.""" - # TODO use a match statement when Python 3.9 support is dropped. - if self is AnnotationSource.ASSIGNMENT_OR_VARIABLE: - return {'final'} - elif self is AnnotationSource.CLASS: - return {'final', 'class_var'} - elif self is AnnotationSource.DATACLASS: - return {'final', 'class_var', 'init_var'} - elif self is AnnotationSource.TYPED_DICT: - return {'required', 'not_required', 'read_only'} - elif self in (AnnotationSource.NAMED_TUPLE, AnnotationSource.FUNCTION, AnnotationSource.BARE): - return set() - elif self is AnnotationSource.ANY: - return _all_qualifiers - else: # pragma: no cover - assert_never(self) - - -class ForbiddenQualifier(Exception): - """The provided [type qualifier][] is forbidden.""" - - qualifier: Qualifier - """The forbidden qualifier.""" - - def __init__(self, qualifier: Qualifier, /) -> None: - self.qualifier = qualifier - - -class _UnknownTypeEnum(Enum): - UNKNOWN = auto() - - def __str__(self) -> str: - return 'UNKNOWN' - - def __repr__(self) -> str: - return '' - - -UNKNOWN = _UnknownTypeEnum.UNKNOWN -"""A sentinel value used when no [type expression][] is present.""" - -_UnkownType: TypeAlias = Literal[_UnknownTypeEnum.UNKNOWN] -"""The type of the [`UNKNOWN`][typing_inspection.introspection.UNKNOWN] sentinel value.""" - - -class InspectedAnnotation(NamedTuple): - """The result of the inspected annotation.""" - - type: Any | _UnkownType - """The final [type expression][], with [type qualifiers][type qualifier] and annotated metadata stripped. - - If no type expression is available, the [`UNKNOWN`][typing_inspection.introspection.UNKNOWN] sentinel - value is used instead. This is the case when a [type qualifier][] is used with no type annotation: - - ```python - ID: Final = 1 - - class C: - x: ClassVar = 'test' - ``` - """ - - qualifiers: set[Qualifier] - """The [type qualifiers][type qualifier] present on the annotation.""" - - metadata: list[Any] - """The annotated metadata.""" - - -def inspect_annotation( # noqa: PLR0915 - annotation: Any, - /, - *, - annotation_source: AnnotationSource, - unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'skip', -) -> InspectedAnnotation: - """Inspect an [annotation expression][], extracting any [type qualifier][] and metadata. - - An [annotation expression][] is a [type expression][] optionally surrounded by one or more - [type qualifiers][type qualifier] or by [`Annotated`][typing.Annotated]. This function will: - - - Unwrap the type expression, keeping track of the type qualifiers. - - Unwrap [`Annotated`][typing.Annotated] forms, keeping track of the annotated metadata. - - Args: - annotation: The annotation expression to be inspected. - annotation_source: The source of the annotation. Depending on the source (e.g. a class), different type - qualifiers may be (dis)allowed. To allow any type qualifier, use - [`AnnotationSource.ANY`][typing_inspection.introspection.AnnotationSource.ANY]. - unpack_type_aliases: What to do when encountering [PEP 695](https://peps.python.org/pep-0695/) - [type aliases][type-aliases]. Can be one of: - - - `'skip'`: Do not try to parse type aliases (the default): - ```pycon - >>> type MyInt = Annotated[int, 'meta'] - >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='skip') - InspectedAnnotation(type=MyInt, qualifiers={}, metadata=[]) - ``` - - - `'lenient'`: Try to parse type aliases, and fallback to `'skip'` if the type alias - can't be inspected (because of an undefined forward reference): - ```pycon - >>> type MyInt = Annotated[Undefined, 'meta'] - >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='lenient') - InspectedAnnotation(type=MyInt, qualifiers={}, metadata=[]) - >>> Undefined = int - >>> inspect_annotation(MyInt, annotation_source=AnnotationSource.BARE, unpack_type_aliases='lenient') - InspectedAnnotation(type=int, qualifiers={}, metadata=['meta']) - ``` - - - `'eager'`: Parse type aliases and raise any encountered [`NameError`][] exceptions. - - Returns: - The result of the inspected annotation, where the type expression, used qualifiers and metadata is stored. - - Example: - ```pycon - >>> inspect_annotation( - ... Final[Annotated[ClassVar[Annotated[int, 'meta_1']], 'meta_2']], - ... annotation_source=AnnotationSource.CLASS, - ... ) - ... - InspectedAnnotation(type=int, qualifiers={'class_var', 'final'}, metadata=['meta_1', 'meta_2']) - ``` - """ - allowed_qualifiers = annotation_source.allowed_qualifiers - qualifiers: set[Qualifier] = set() - metadata: list[Any] = [] - - while True: - annotation, _meta = _unpack_annotated(annotation, unpack_type_aliases=unpack_type_aliases) - if _meta: - metadata = _meta + metadata - continue - - origin = get_origin(annotation) - if origin is not None: - if typing_objects.is_classvar(origin): - if 'class_var' not in allowed_qualifiers: - raise ForbiddenQualifier('class_var') - qualifiers.add('class_var') - annotation = annotation.__args__[0] - elif typing_objects.is_final(origin): - if 'final' not in allowed_qualifiers: - raise ForbiddenQualifier('final') - qualifiers.add('final') - annotation = annotation.__args__[0] - elif typing_objects.is_required(origin): - if 'required' not in allowed_qualifiers: - raise ForbiddenQualifier('required') - qualifiers.add('required') - annotation = annotation.__args__[0] - elif typing_objects.is_notrequired(origin): - if 'not_required' not in allowed_qualifiers: - raise ForbiddenQualifier('not_required') - qualifiers.add('not_required') - annotation = annotation.__args__[0] - elif typing_objects.is_readonly(origin): - if 'read_only' not in allowed_qualifiers: - raise ForbiddenQualifier('not_required') - qualifiers.add('read_only') - annotation = annotation.__args__[0] - else: - # origin is not None but not a type qualifier nor `Annotated` (e.g. `list[int]`): - break - elif isinstance(annotation, InitVar): - if 'init_var' not in allowed_qualifiers: - raise ForbiddenQualifier('init_var') - qualifiers.add('init_var') - annotation = cast(Any, annotation.type) - else: - break - - # `Final`, `ClassVar` and `InitVar` are type qualifiers allowed to be used as a bare annotation: - if typing_objects.is_final(annotation): - if 'final' not in allowed_qualifiers: - raise ForbiddenQualifier('final') - qualifiers.add('final') - annotation = UNKNOWN - elif typing_objects.is_classvar(annotation): - if 'class_var' not in allowed_qualifiers: - raise ForbiddenQualifier('class_var') - qualifiers.add('class_var') - annotation = UNKNOWN - elif annotation is InitVar: - if 'init_var' not in allowed_qualifiers: - raise ForbiddenQualifier('init_var') - qualifiers.add('init_var') - annotation = UNKNOWN - - return InspectedAnnotation(annotation, qualifiers, metadata) - - -def _unpack_annotated_inner( - annotation: Any, unpack_type_aliases: Literal['lenient', 'eager'], check_annotated: bool -) -> tuple[Any, list[Any]]: - origin = get_origin(annotation) - if check_annotated and typing_objects.is_annotated(origin): - annotated_type = annotation.__origin__ - metadata = list(annotation.__metadata__) - - # The annotated type might be a PEP 695 type alias, so we need to recursively - # unpack it. Because Python already flattens `Annotated[Annotated[, ...], ...]` forms, - # we can skip the `is_annotated()` check in the next call: - annotated_type, sub_meta = _unpack_annotated_inner( - annotated_type, unpack_type_aliases=unpack_type_aliases, check_annotated=False - ) - metadata = sub_meta + metadata - return annotated_type, metadata - elif typing_objects.is_typealiastype(annotation): - try: - value = annotation.__value__ - except NameError: - if unpack_type_aliases == 'eager': - raise - else: - typ, metadata = _unpack_annotated_inner( - value, unpack_type_aliases=unpack_type_aliases, check_annotated=True - ) - if metadata: - # Having metadata means the type alias' `__value__` was an `Annotated` form - # (or, recursively, a type alias to an `Annotated` form). It is important to check - # for this, as we don't want to unpack other type aliases (e.g. `type MyInt = int`). - return typ, metadata - return annotation, [] - elif typing_objects.is_typealiastype(origin): - # When parameterized, PEP 695 type aliases become generic aliases - # (e.g. with `type MyList[T] = Annotated[list[T], ...]`, `MyList[int]` - # is a generic alias). - try: - value = origin.__value__ - except NameError: - if unpack_type_aliases == 'eager': - raise - else: - # While Python already handles type variable replacement for simple `Annotated` forms, - # we need to manually apply the same logic for PEP 695 type aliases: - # - With `MyList = Annotated[list[T], ...]`, `MyList[int] == Annotated[list[int], ...]` - # - With `type MyList[T] = Annotated[list[T], ...]`, `MyList[int].__value__ == Annotated[list[T], ...]`. - - try: - # To do so, we emulate the parameterization of the value with the arguments: - # with `type MyList[T] = Annotated[list[T], ...]`, to emulate `MyList[int]`, - # we do `Annotated[list[T], ...][int]` (which gives `Annotated[list[T], ...]`): - value = value[annotation.__args__] - except TypeError: - # Might happen if the type alias is parameterized, but its value doesn't have any - # type variables, e.g. `type MyInt[T] = int`. - pass - typ, metadata = _unpack_annotated_inner( - value, unpack_type_aliases=unpack_type_aliases, check_annotated=True - ) - if metadata: - return typ, metadata - return annotation, [] - - return annotation, [] - - -# This could eventually be made public: -def _unpack_annotated( - annotation: Any, /, *, unpack_type_aliases: Literal['skip', 'lenient', 'eager'] = 'eager' -) -> tuple[Any, list[Any]]: - if unpack_type_aliases == 'skip': - if typing_objects.is_annotated(get_origin(annotation)): - return annotation.__origin__, list(annotation.__metadata__) - else: - return annotation, [] - - return _unpack_annotated_inner(annotation, unpack_type_aliases=unpack_type_aliases, check_annotated=True) diff --git a/functions/python/data-processor/typing_inspection/py.typed b/functions/python/data-processor/typing_inspection/py.typed deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/typing_inspection/typing_objects.py b/functions/python/data-processor/typing_inspection/typing_objects.py deleted file mode 100644 index dc44ba98c..000000000 --- a/functions/python/data-processor/typing_inspection/typing_objects.py +++ /dev/null @@ -1,607 +0,0 @@ -"""Low-level introspection utilities for [`typing`][] members. - -The provided functions in this module check against both the [`typing`][] and [`typing_extensions`][] -variants, if they exists and are different. -""" -# ruff: noqa: UP006 - -import collections.abc -import contextlib -import re -import sys -import typing -import warnings -from textwrap import dedent -from types import FunctionType, GenericAlias -from typing import Any, Final - -import typing_extensions -from typing_extensions import LiteralString, TypeAliasType, TypeIs, deprecated - -__all__ = ( - 'DEPRECATED_ALIASES', - 'NoneType', - 'is_annotated', - 'is_any', - 'is_classvar', - 'is_concatenate', - 'is_deprecated', - 'is_final', - 'is_forwardref', - 'is_generic', - 'is_literal', - 'is_literalstring', - 'is_namedtuple', - 'is_never', - 'is_newtype', - 'is_nodefault', - 'is_noextraitems', - 'is_noreturn', - 'is_notrequired', - 'is_paramspec', - 'is_paramspecargs', - 'is_paramspeckwargs', - 'is_readonly', - 'is_required', - 'is_self', - 'is_typealias', - 'is_typealiastype', - 'is_typeguard', - 'is_typeis', - 'is_typevar', - 'is_typevartuple', - 'is_union', - 'is_unpack', -) - -_IS_PY310 = sys.version_info[:2] == (3, 10) - - -def _compile_identity_check_function(member: LiteralString, function_name: LiteralString) -> FunctionType: - """Create a function checking that the function argument is the (unparameterized) typing `member`. - - The function will make sure to check against both the `typing` and `typing_extensions` - variants as depending on the Python version, the `typing_extensions` variant might be different. - For instance, on Python 3.9: - - ```pycon - >>> from typing import Literal as t_Literal - >>> from typing_extensions import Literal as te_Literal, get_origin - - >>> t_Literal is te_Literal - False - >>> get_origin(t_Literal[1]) - typing.Literal - >>> get_origin(te_Literal[1]) - typing_extensions.Literal - ``` - """ - in_typing = hasattr(typing, member) - in_typing_extensions = hasattr(typing_extensions, member) - - if in_typing and in_typing_extensions: - if getattr(typing, member) is getattr(typing_extensions, member): - check_code = f'obj is typing.{member}' - else: - check_code = f'obj is typing.{member} or obj is typing_extensions.{member}' - elif in_typing and not in_typing_extensions: - check_code = f'obj is typing.{member}' - elif not in_typing and in_typing_extensions: - check_code = f'obj is typing_extensions.{member}' - else: - check_code = 'False' - - func_code = dedent(f""" - def {function_name}(obj: Any, /) -> bool: - return {check_code} - """) - - locals_: dict[str, Any] = {} - globals_: dict[str, Any] = {'Any': Any, 'typing': typing, 'typing_extensions': typing_extensions} - exec(func_code, globals_, locals_) - return locals_[function_name] - - -def _compile_isinstance_check_function(member: LiteralString, function_name: LiteralString) -> FunctionType: - """Create a function checking that the function is an instance of the typing `member`. - - The function will make sure to check against both the `typing` and `typing_extensions` - variants as depending on the Python version, the `typing_extensions` variant might be different. - """ - in_typing = hasattr(typing, member) - in_typing_extensions = hasattr(typing_extensions, member) - - if in_typing and in_typing_extensions: - if getattr(typing, member) is getattr(typing_extensions, member): - check_code = f'isinstance(obj, typing.{member})' - else: - check_code = f'isinstance(obj, (typing.{member}, typing_extensions.{member}))' - elif in_typing and not in_typing_extensions: - check_code = f'isinstance(obj, typing.{member})' - elif not in_typing and in_typing_extensions: - check_code = f'isinstance(obj, typing_extensions.{member})' - else: - check_code = 'False' - - func_code = dedent(f""" - def {function_name}(obj: Any, /) -> 'TypeIs[{member}]': - return {check_code} - """) - - locals_: dict[str, Any] = {} - globals_: dict[str, Any] = {'Any': Any, 'typing': typing, 'typing_extensions': typing_extensions} - exec(func_code, globals_, locals_) - return locals_[function_name] - - -if sys.version_info >= (3, 10): - from types import NoneType -else: - NoneType = type(None) - -# Keep this ordered, as per `typing.__all__`: - -is_annotated = _compile_identity_check_function('Annotated', 'is_annotated') -is_annotated.__doc__ = """ -Return whether the argument is the [`Annotated`][typing.Annotated] [special form][]. - -```pycon ->>> is_annotated(Annotated) -True ->>> is_annotated(Annotated[int, ...]) -False -``` -""" - -is_any = _compile_identity_check_function('Any', 'is_any') -is_any.__doc__ = """ -Return whether the argument is the [`Any`][typing.Any] [special form][]. - -```pycon ->>> is_any(Any) -True -``` -""" - -is_classvar = _compile_identity_check_function('ClassVar', 'is_classvar') -is_classvar.__doc__ = """ -Return whether the argument is the [`ClassVar`][typing.ClassVar] [type qualifier][]. - -```pycon ->>> is_classvar(ClassVar) -True ->>> is_classvar(ClassVar[int]) ->>> False -``` -""" - -is_concatenate = _compile_identity_check_function('Concatenate', 'is_concatenate') -is_concatenate.__doc__ = """ -Return whether the argument is the [`Concatenate`][typing.Concatenate] [special form][]. - -```pycon ->>> is_concatenate(Concatenate) -True ->>> is_concatenate(Concatenate[int, P]) -False -``` -""" - -is_final = _compile_identity_check_function('Final', 'is_final') -is_final.__doc__ = """ -Return whether the argument is the [`Final`][typing.Final] [type qualifier][]. - -```pycon ->>> is_final(Final) -True ->>> is_final(Final[int]) -False -``` -""" - - -# Unlikely to have a different version in `typing-extensions`, but keep it consistent. -# Also note that starting in 3.14, this is an alias to `annotationlib.ForwardRef`, but -# accessing it from `typing` doesn't seem to be deprecated. -is_forwardref = _compile_isinstance_check_function('ForwardRef', 'is_forwardref') -is_forwardref.__doc__ = """ -Return whether the argument is an instance of [`ForwardRef`][typing.ForwardRef]. - -```pycon ->>> is_forwardref(ForwardRef('T')) -True -``` -""" - - -is_generic = _compile_identity_check_function('Generic', 'is_generic') -is_generic.__doc__ = """ -Return whether the argument is the [`Generic`][typing.Generic] [special form][]. - -```pycon ->>> is_generic(Generic) -True ->>> is_generic(Generic[T]) -False -``` -""" - -is_literal = _compile_identity_check_function('Literal', 'is_literal') -is_literal.__doc__ = """ -Return whether the argument is the [`Literal`][typing.Literal] [special form][]. - -```pycon ->>> is_literal(Literal) -True ->>> is_literal(Literal["a"]) -False -``` -""" - - -# `get_origin(Optional[int]) is Union`, so `is_optional()` isn't implemented. - -is_paramspec = _compile_isinstance_check_function('ParamSpec', 'is_paramspec') -is_paramspec.__doc__ = """ -Return whether the argument is an instance of [`ParamSpec`][typing.ParamSpec]. - -```pycon ->>> P = ParamSpec('P') ->>> is_paramspec(P) -True -``` -""" - -# Protocol? - -is_typevar = _compile_isinstance_check_function('TypeVar', 'is_typevar') -is_typevar.__doc__ = """ -Return whether the argument is an instance of [`TypeVar`][typing.TypeVar]. - -```pycon ->>> T = TypeVar('T') ->>> is_typevar(T) -True -``` -""" - -is_typevartuple = _compile_isinstance_check_function('TypeVarTuple', 'is_typevartuple') -is_typevartuple.__doc__ = """ -Return whether the argument is an instance of [`TypeVarTuple`][typing.TypeVarTuple]. - -```pycon ->>> Ts = TypeVarTuple('Ts') ->>> is_typevartuple(Ts) -True -``` -""" - -is_union = _compile_identity_check_function('Union', 'is_union') -is_union.__doc__ = """ -Return whether the argument is the [`Union`][typing.Union] [special form][]. - -This function can also be used to check for the [`Optional`][typing.Optional] [special form][], -as at runtime, `Optional[int]` is equivalent to `Union[int, None]`. - -```pycon ->>> is_union(Union) -True ->>> is_union(Union[int, str]) -False -``` - -!!! warning - This does not check for unions using the [new syntax][types-union] (e.g. `int | str`). -""" - - -def is_namedtuple(obj: Any, /) -> bool: - """Return whether the argument is a named tuple type. - - This includes [`NamedTuple`][typing.NamedTuple] subclasses and classes created from the - [`collections.namedtuple`][] factory function. - - ```pycon - >>> class User(NamedTuple): - ... name: str - ... - >>> is_namedtuple(User) - True - >>> City = collections.namedtuple('City', []) - >>> is_namedtuple(City) - True - >>> is_namedtuple(NamedTuple) - False - ``` - """ - return isinstance(obj, type) and issubclass(obj, tuple) and hasattr(obj, '_fields') # pyright: ignore[reportUnknownArgumentType] - - -# TypedDict? - -# BinaryIO? IO? TextIO? - -is_literalstring = _compile_identity_check_function('LiteralString', 'is_literalstring') -is_literalstring.__doc__ = """ -Return whether the argument is the [`LiteralString`][typing.LiteralString] [special form][]. - -```pycon ->>> is_literalstring(LiteralString) -True -``` -""" - -is_never = _compile_identity_check_function('Never', 'is_never') -is_never.__doc__ = """ -Return whether the argument is the [`Never`][typing.Never] [special form][]. - -```pycon ->>> is_never(Never) -True -``` -""" - -if sys.version_info >= (3, 10): - is_newtype = _compile_isinstance_check_function('NewType', 'is_newtype') -else: # On Python 3.10, `NewType` is a function. - - def is_newtype(obj: Any, /) -> bool: - return hasattr(obj, '__supertype__') - - -is_newtype.__doc__ = """ -Return whether the argument is a [`NewType`][typing.NewType]. - -```pycon ->>> UserId = NewType("UserId", int) ->>> is_newtype(UserId) -True -``` -""" - -is_nodefault = _compile_identity_check_function('NoDefault', 'is_nodefault') -is_nodefault.__doc__ = """ -Return whether the argument is the [`NoDefault`][typing.NoDefault] sentinel object. - -```pycon ->>> is_nodefault(NoDefault) -True -``` -""" - -is_noextraitems = _compile_identity_check_function('NoExtraItems', 'is_noextraitems') -is_noextraitems.__doc__ = """ -Return whether the argument is the `NoExtraItems` sentinel object. - -```pycon ->>> is_noextraitems(NoExtraItems) -True -``` -""" - -is_noreturn = _compile_identity_check_function('NoReturn', 'is_noreturn') -is_noreturn.__doc__ = """ -Return whether the argument is the [`NoReturn`][typing.NoReturn] [special form][]. - -```pycon ->>> is_noreturn(NoReturn) -True ->>> is_noreturn(Never) -False -``` -""" - -is_notrequired = _compile_identity_check_function('NotRequired', 'is_notrequired') -is_notrequired.__doc__ = """ -Return whether the argument is the [`NotRequired`][typing.NotRequired] [special form][]. - -```pycon ->>> is_notrequired(NotRequired) -True -``` -""" - -is_paramspecargs = _compile_isinstance_check_function('ParamSpecArgs', 'is_paramspecargs') -is_paramspecargs.__doc__ = """ -Return whether the argument is an instance of [`ParamSpecArgs`][typing.ParamSpecArgs]. - -```pycon ->>> P = ParamSpec('P') ->>> is_paramspecargs(P.args) -True -``` -""" - -is_paramspeckwargs = _compile_isinstance_check_function('ParamSpecKwargs', 'is_paramspeckwargs') -is_paramspeckwargs.__doc__ = """ -Return whether the argument is an instance of [`ParamSpecKwargs`][typing.ParamSpecKwargs]. - -```pycon ->>> P = ParamSpec('P') ->>> is_paramspeckwargs(P.kwargs) -True -``` -""" - -is_readonly = _compile_identity_check_function('ReadOnly', 'is_readonly') -is_readonly.__doc__ = """ -Return whether the argument is the [`ReadOnly`][typing.ReadOnly] [special form][]. - -```pycon ->>> is_readonly(ReadOnly) -True -``` -""" - -is_required = _compile_identity_check_function('Required', 'is_required') -is_required.__doc__ = """ -Return whether the argument is the [`Required`][typing.Required] [special form][]. - -```pycon ->>> is_required(Required) -True -``` -""" - -is_self = _compile_identity_check_function('Self', 'is_self') -is_self.__doc__ = """ -Return whether the argument is the [`Self`][typing.Self] [special form][]. - -```pycon ->>> is_self(Self) -True -``` -""" - -# TYPE_CHECKING? - -is_typealias = _compile_identity_check_function('TypeAlias', 'is_typealias') -is_typealias.__doc__ = """ -Return whether the argument is the [`TypeAlias`][typing.TypeAlias] [special form][]. - -```pycon ->>> is_typealias(TypeAlias) -True -``` -""" - -is_typeguard = _compile_identity_check_function('TypeGuard', 'is_typeguard') -is_typeguard.__doc__ = """ -Return whether the argument is the [`TypeGuard`][typing.TypeGuard] [special form][]. - -```pycon ->>> is_typeguard(TypeGuard) -True -``` -""" - -is_typeis = _compile_identity_check_function('TypeIs', 'is_typeis') -is_typeis.__doc__ = """ -Return whether the argument is the [`TypeIs`][typing.TypeIs] [special form][]. - -```pycon ->>> is_typeis(TypeIs) -True -``` -""" - -_is_typealiastype_inner = _compile_isinstance_check_function('TypeAliasType', '_is_typealiastype_inner') - - -if _IS_PY310: - # Parameterized PEP 695 type aliases are instances of `types.GenericAlias` in typing_extensions>=4.13.0. - # On Python 3.10, with `Alias[int]` being such an instance of `GenericAlias`, - # `isinstance(Alias[int], TypeAliasType)` returns `True`. - # See https://github.com/python/cpython/issues/89828. - def is_typealiastype(obj: Any, /) -> 'TypeIs[TypeAliasType]': - return type(obj) is not GenericAlias and _is_typealiastype_inner(obj) -else: - is_typealiastype = _compile_isinstance_check_function('TypeAliasType', 'is_typealiastype') - -is_typealiastype.__doc__ = """ -Return whether the argument is a [`TypeAliasType`][typing.TypeAliasType] instance. - -```pycon ->>> type MyInt = int ->>> is_typealiastype(MyInt) -True ->>> MyStr = TypeAliasType("MyStr", str) ->>> is_typealiastype(MyStr): -True ->>> type MyList[T] = list[T] ->>> is_typealiastype(MyList[int]) -False -``` -""" - -is_unpack = _compile_identity_check_function('Unpack', 'is_unpack') -is_unpack.__doc__ = """ -Return whether the argument is the [`Unpack`][typing.Unpack] [special form][]. - -```pycon ->>> is_unpack(Unpack) -True ->>> is_unpack(Unpack[Ts]) -False -``` -""" - - -if sys.version_info >= (3, 13): - - def is_deprecated(obj: Any, /) -> 'TypeIs[deprecated]': - return isinstance(obj, (warnings.deprecated, typing_extensions.deprecated)) - -else: - - def is_deprecated(obj: Any, /) -> 'TypeIs[deprecated]': - return isinstance(obj, typing_extensions.deprecated) - - -is_deprecated.__doc__ = """ -Return whether the argument is a [`deprecated`][warnings.deprecated] instance. - -This also includes the [`typing_extensions` backport][typing_extensions.deprecated]. - -```pycon ->>> is_deprecated(warnings.deprecated('message')) -True ->>> is_deprecated(typing_extensions.deprecated('message')) -True -``` -""" - - -# Aliases defined in the `typing` module using `typing._SpecialGenericAlias` (itself aliased as `alias()`): -DEPRECATED_ALIASES: Final[dict[Any, type[Any]]] = { - typing.Hashable: collections.abc.Hashable, - typing.Awaitable: collections.abc.Awaitable, - typing.Coroutine: collections.abc.Coroutine, - typing.AsyncIterable: collections.abc.AsyncIterable, - typing.AsyncIterator: collections.abc.AsyncIterator, - typing.Iterable: collections.abc.Iterable, - typing.Iterator: collections.abc.Iterator, - typing.Reversible: collections.abc.Reversible, - typing.Sized: collections.abc.Sized, - typing.Container: collections.abc.Container, - typing.Collection: collections.abc.Collection, - # type ignore reason: https://github.com/python/typeshed/issues/6257: - typing.Callable: collections.abc.Callable, # pyright: ignore[reportAssignmentType, reportUnknownMemberType] - typing.AbstractSet: collections.abc.Set, - typing.MutableSet: collections.abc.MutableSet, - typing.Mapping: collections.abc.Mapping, - typing.MutableMapping: collections.abc.MutableMapping, - typing.Sequence: collections.abc.Sequence, - typing.MutableSequence: collections.abc.MutableSequence, - typing.Tuple: tuple, - typing.List: list, - typing.Deque: collections.deque, - typing.Set: set, - typing.FrozenSet: frozenset, - typing.MappingView: collections.abc.MappingView, - typing.KeysView: collections.abc.KeysView, - typing.ItemsView: collections.abc.ItemsView, - typing.ValuesView: collections.abc.ValuesView, - typing.Dict: dict, - typing.DefaultDict: collections.defaultdict, - typing.OrderedDict: collections.OrderedDict, - typing.Counter: collections.Counter, - typing.ChainMap: collections.ChainMap, - typing.Generator: collections.abc.Generator, - typing.AsyncGenerator: collections.abc.AsyncGenerator, - typing.Type: type, - # Defined in `typing.__getattr__`: - typing.Pattern: re.Pattern, - typing.Match: re.Match, - typing.ContextManager: contextlib.AbstractContextManager, - typing.AsyncContextManager: contextlib.AbstractAsyncContextManager, - # Skipped: `ByteString` (deprecated, removed in 3.14) -} -"""A mapping between the deprecated typing aliases to their replacement, as per [PEP 585](https://peps.python.org/pep-0585/).""" - - -# Add the `typing_extensions` aliases: -for alias, target in list(DEPRECATED_ALIASES.items()): - # Use `alias.__name__` when we drop support for Python 3.9 - if (te_alias := getattr(typing_extensions, alias._name, None)) is not None: - DEPRECATED_ALIASES[te_alias] = target diff --git a/functions/python/data-processor/typing_inspection/typing_objects.pyi b/functions/python/data-processor/typing_inspection/typing_objects.pyi deleted file mode 100644 index 507159800..000000000 --- a/functions/python/data-processor/typing_inspection/typing_objects.pyi +++ /dev/null @@ -1,417 +0,0 @@ -# Stub file generated using: -# `stubgen --inspect-mode --include-docstrings -m typing_inspection.typing_objects` -# (manual edits need to be applied). -"""Low-level introspection utilities for [`typing`][] members. - -The provided functions in this module check against both the [`typing`][] and [`typing_extensions`][] -variants, if they exists and are different. -""" - -import sys -from typing import Any, Final, ForwardRef, NewType, TypeVar - -from typing_extensions import ParamSpec, ParamSpecArgs, ParamSpecKwargs, TypeAliasType, TypeIs, TypeVarTuple, deprecated - -__all__ = [ - 'DEPRECATED_ALIASES', - 'NoneType', - 'is_annotated', - 'is_any', - 'is_classvar', - 'is_concatenate', - 'is_deprecated', - 'is_final', - 'is_generic', - 'is_literal', - 'is_literalstring', - 'is_namedtuple', - 'is_never', - 'is_newtype', - 'is_nodefault', - 'is_noextraitems', - 'is_noreturn', - 'is_notrequired', - 'is_paramspec', - 'is_paramspecargs', - 'is_paramspeckwargs', - 'is_readonly', - 'is_required', - 'is_self', - 'is_typealias', - 'is_typealiastype', - 'is_typeguard', - 'is_typeis', - 'is_typevar', - 'is_typevartuple', - 'is_union', - 'is_unpack', -] - -if sys.version_info >= (3, 10): - from types import NoneType -else: - NoneType = type(None) - -def is_annotated(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Annotated`][typing.Annotated] [special form][]. - - ```pycon - >>> is_annotated(Annotated) - True - >>> is_annotated(Annotated[int, ...]) - False - ``` - """ - -def is_any(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Any`][typing.Any] [special form][]. - - ```pycon - >>> is_any(Any) - True - ``` - """ - -def is_classvar(obj: Any, /) -> bool: - """ - Return whether the argument is the [`ClassVar`][typing.ClassVar] [type qualifier][]. - - ```pycon - >>> is_classvar(ClassVar) - True - >>> is_classvar(ClassVar[int]) - >>> False - ``` - """ - -def is_concatenate(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Concatenate`][typing.Concatenate] [special form][]. - - ```pycon - >>> is_concatenate(Concatenate) - True - >>> is_concatenate(Concatenate[int, P]) - False - ``` - """ - -def is_final(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Final`][typing.Final] [type qualifier][]. - - ```pycon - >>> is_final(Final) - True - >>> is_final(Final[int]) - False - ``` - """ - -def is_forwardref(obj: Any, /) -> TypeIs[ForwardRef]: - """ - Return whether the argument is an instance of [`ForwardRef`][typing.ForwardRef]. - - ```pycon - >>> is_forwardref(ForwardRef('T')) - True - ``` - """ - -def is_generic(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Generic`][typing.Generic] [special form][]. - - ```pycon - >>> is_generic(Generic) - True - >>> is_generic(Generic[T]) - False - ``` - """ - -def is_literal(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Literal`][typing.Literal] [special form][]. - - ```pycon - >>> is_literal(Literal) - True - >>> is_literal(Literal["a"]) - False - ``` - """ - -def is_paramspec(obj: Any, /) -> TypeIs[ParamSpec]: - """ - Return whether the argument is an instance of [`ParamSpec`][typing.ParamSpec]. - - ```pycon - >>> P = ParamSpec('P') - >>> is_paramspec(P) - True - ``` - """ - -def is_typevar(obj: Any, /) -> TypeIs[TypeVar]: - """ - Return whether the argument is an instance of [`TypeVar`][typing.TypeVar]. - - ```pycon - >>> T = TypeVar('T') - >>> is_typevar(T) - True - ``` - """ - -def is_typevartuple(obj: Any, /) -> TypeIs[TypeVarTuple]: - """ - Return whether the argument is an instance of [`TypeVarTuple`][typing.TypeVarTuple]. - - ```pycon - >>> Ts = TypeVarTuple('Ts') - >>> is_typevartuple(Ts) - True - ``` - """ - -def is_union(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Union`][typing.Union] [special form][]. - - This function can also be used to check for the [`Optional`][typing.Optional] [special form][], - as at runtime, `Optional[int]` is equivalent to `Union[int, None]`. - - ```pycon - >>> is_union(Union) - True - >>> is_union(Union[int, str]) - False - ``` - - !!! warning - This does not check for unions using the [new syntax][types-union] (e.g. `int | str`). - """ - -def is_namedtuple(obj: Any, /) -> bool: - """Return whether the argument is a named tuple type. - - This includes [`NamedTuple`][typing.NamedTuple] subclasses and classes created from the - [`collections.namedtuple`][] factory function. - - ```pycon - >>> class User(NamedTuple): - ... name: str - ... - >>> is_namedtuple(User) - True - >>> City = collections.namedtuple('City', []) - >>> is_namedtuple(City) - True - >>> is_namedtuple(NamedTuple) - False - ``` - """ - -def is_literalstring(obj: Any, /) -> bool: - """ - Return whether the argument is the [`LiteralString`][typing.LiteralString] [special form][]. - - ```pycon - >>> is_literalstring(LiteralString) - True - ``` - """ - -def is_never(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Never`][typing.Never] [special form][]. - - ```pycon - >>> is_never(Never) - True - ``` - """ - -def is_newtype(obj: Any, /) -> TypeIs[NewType]: - """ - Return whether the argument is a [`NewType`][typing.NewType]. - - ```pycon - >>> UserId = NewType("UserId", int) - >>> is_newtype(UserId) - True - ``` - """ - -def is_nodefault(obj: Any, /) -> bool: - """ - Return whether the argument is the [`NoDefault`][typing.NoDefault] sentinel object. - - ```pycon - >>> is_nodefault(NoDefault) - True - ``` - """ - -def is_noextraitems(obj: Any, /) -> bool: - """ - Return whether the argument is the `NoExtraItems` sentinel object. - - ```pycon - >>> is_noextraitems(NoExtraItems) - True - ``` - """ - -def is_noreturn(obj: Any, /) -> bool: - """ - Return whether the argument is the [`NoReturn`][typing.NoReturn] [special form][]. - - ```pycon - >>> is_noreturn(NoReturn) - True - >>> is_noreturn(Never) - False - ``` - """ - -def is_notrequired(obj: Any, /) -> bool: - """ - Return whether the argument is the [`NotRequired`][typing.NotRequired] [special form][]. - - ```pycon - >>> is_notrequired(NotRequired) - True - ``` - """ - -def is_paramspecargs(obj: Any, /) -> TypeIs[ParamSpecArgs]: - """ - Return whether the argument is an instance of [`ParamSpecArgs`][typing.ParamSpecArgs]. - - ```pycon - >>> P = ParamSpec('P') - >>> is_paramspecargs(P.args) - True - ``` - """ - -def is_paramspeckwargs(obj: Any, /) -> TypeIs[ParamSpecKwargs]: - """ - Return whether the argument is an instance of [`ParamSpecKwargs`][typing.ParamSpecKwargs]. - - ```pycon - >>> P = ParamSpec('P') - >>> is_paramspeckwargs(P.kwargs) - True - ``` - """ - -def is_readonly(obj: Any, /) -> bool: - """ - Return whether the argument is the [`ReadOnly`][typing.ReadOnly] [special form][]. - - ```pycon - >>> is_readonly(ReadOnly) - True - ``` - """ - -def is_required(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Required`][typing.Required] [special form][]. - - ```pycon - >>> is_required(Required) - True - ``` - """ - -def is_self(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Self`][typing.Self] [special form][]. - - ```pycon - >>> is_self(Self) - True - ``` - """ - -def is_typealias(obj: Any, /) -> bool: - """ - Return whether the argument is the [`TypeAlias`][typing.TypeAlias] [special form][]. - - ```pycon - >>> is_typealias(TypeAlias) - True - ``` - """ - -def is_typeguard(obj: Any, /) -> bool: - """ - Return whether the argument is the [`TypeGuard`][typing.TypeGuard] [special form][]. - - ```pycon - >>> is_typeguard(TypeGuard) - True - ``` - """ - -def is_typeis(obj: Any, /) -> bool: - """ - Return whether the argument is the [`TypeIs`][typing.TypeIs] [special form][]. - - ```pycon - >>> is_typeis(TypeIs) - True - ``` - """ - -def is_typealiastype(obj: Any, /) -> TypeIs[TypeAliasType]: - """ - Return whether the argument is a [`TypeAliasType`][typing.TypeAliasType] instance. - - ```pycon - >>> type MyInt = int - >>> is_typealiastype(MyInt) - True - >>> MyStr = TypeAliasType("MyStr", str) - >>> is_typealiastype(MyStr): - True - >>> type MyList[T] = list[T] - >>> is_typealiastype(MyList[int]) - False - ``` - """ - -def is_unpack(obj: Any, /) -> bool: - """ - Return whether the argument is the [`Unpack`][typing.Unpack] [special form][]. - - ```pycon - >>> is_unpack(Unpack) - True - >>> is_unpack(Unpack[Ts]) - False - ``` - """ - -def is_deprecated(obj: Any, /) -> TypeIs[deprecated]: - """ - Return whether the argument is a [`deprecated`][warnings.deprecated] instance. - - This also includes the [`typing_extensions` backport][typing_extensions.deprecated]. - - ```pycon - >>> is_deprecated(warnings.deprecated('message')) - True - >>> is_deprecated(typing_extensions.deprecated('deprecated')) - True - ``` - """ - -DEPRECATED_ALIASES: Final[dict[Any, type[Any]]] -"""A mapping between the deprecated typing aliases to their replacement, as per [PEP 585](https://peps.python.org/pep-0585/).""" diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/INSTALLER b/functions/python/data-processor/uvicorn-0.49.0.dist-info/INSTALLER deleted file mode 100644 index a1b589e38..000000000 --- a/functions/python/data-processor/uvicorn-0.49.0.dist-info/INSTALLER +++ /dev/null @@ -1 +0,0 @@ -pip diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/METADATA b/functions/python/data-processor/uvicorn-0.49.0.dist-info/METADATA deleted file mode 100644 index 390a4581f..000000000 --- a/functions/python/data-processor/uvicorn-0.49.0.dist-info/METADATA +++ /dev/null @@ -1,191 +0,0 @@ -Metadata-Version: 2.4 -Name: uvicorn -Version: 0.49.0 -Summary: The lightning-fast ASGI server. -Project-URL: Changelog, https://uvicorn.dev/release-notes -Project-URL: Funding, https://github.com/sponsors/encode -Project-URL: Homepage, https://uvicorn.dev/ -Project-URL: Source, https://github.com/Kludex/uvicorn -Author-email: Tom Christie -Maintainer-email: Marcelo Trylesinski -License-Expression: BSD-3-Clause -License-File: LICENSE.md -Classifier: Development Status :: 4 - Beta -Classifier: Environment :: Web Environment -Classifier: Intended Audience :: Developers -Classifier: Operating System :: OS Independent -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.10 -Classifier: Programming Language :: Python :: 3.11 -Classifier: Programming Language :: Python :: 3.12 -Classifier: Programming Language :: Python :: 3.13 -Classifier: Programming Language :: Python :: 3.14 -Classifier: Programming Language :: Python :: Implementation :: CPython -Classifier: Programming Language :: Python :: Implementation :: PyPy -Classifier: Topic :: Internet :: WWW/HTTP -Requires-Python: >=3.10 -Requires-Dist: click>=7.0 -Requires-Dist: h11>=0.8 -Requires-Dist: typing-extensions>=4.0; python_version < '3.11' -Provides-Extra: standard -Requires-Dist: colorama>=0.4; (sys_platform == 'win32') and extra == 'standard' -Requires-Dist: httptools>=0.8.0; extra == 'standard' -Requires-Dist: python-dotenv>=0.13; extra == 'standard' -Requires-Dist: pyyaml>=5.1; extra == 'standard' -Requires-Dist: uvloop>=0.15.1; (sys_platform != 'win32' and (sys_platform != 'cygwin' and platform_python_implementation != 'PyPy')) and extra == 'standard' -Requires-Dist: watchfiles>=0.20; extra == 'standard' -Requires-Dist: websockets>=10.4; extra == 'standard' -Description-Content-Type: text/markdown - -

- uvicorn -

- -

-An ASGI web server, for Python. -

- ---- - -[![Build Status](https://github.com/Kludex/uvicorn/workflows/Test%20Suite/badge.svg)](https://github.com/Kludex/uvicorn/actions) -[![Package version](https://badge.fury.io/py/uvicorn.svg)](https://pypi.python.org/pypi/uvicorn) -[![Supported Python Version](https://img.shields.io/pypi/pyversions/uvicorn.svg?color=%2334D058)](https://pypi.org/project/uvicorn) -[![Discord](https://img.shields.io/discord/1051468649518616576?logo=discord&logoColor=ffffff&color=7389D8&labelColor=6A7EC2)](https://discord.gg/RxKUF5JuHs) - ---- - -**Documentation**: [https://uvicorn.dev](https://uvicorn.dev) - -**Source Code**: [https://www.github.com/Kludex/uvicorn](https://www.github.com/Kludex/uvicorn) - ---- - -Uvicorn is an ASGI web server implementation for Python. - -Until recently Python has lacked a minimal low-level server/application interface for -async frameworks. The [ASGI specification][asgi] fills this gap, and means we're now able to -start building a common set of tooling usable across all async frameworks. - -Uvicorn supports HTTP/1.1 and WebSockets. - -## Quickstart - -Install using `pip`: - -```shell -$ pip install uvicorn -``` - -This will install uvicorn with minimal (pure Python) dependencies. - -```shell -$ pip install 'uvicorn[standard]' -``` - -This will install uvicorn with "Cython-based" dependencies (where possible) and other "optional extras". - -In this context, "Cython-based" means the following: - -- the event loop `uvloop` will be installed and used if possible. -- the http protocol will be handled by `httptools` if possible. - -Moreover, "optional extras" means that: - -- the websocket protocol will be handled by `websockets` (should you want to use `wsproto` you'd need to install it manually) if possible. -- the `--reload` flag in development mode will use `watchfiles`. -- windows users will have `colorama` installed for the colored logs. -- `python-dotenv` will be installed should you want to use the `--env-file` option. -- `PyYAML` will be installed to allow you to provide a `.yaml` file to `--log-config`, if desired. - -Create an application, in `example.py`: - -```python -async def app(scope, receive, send): - assert scope['type'] == 'http' - - await send({ - 'type': 'http.response.start', - 'status': 200, - 'headers': [ - (b'content-type', b'text/plain'), - ], - }) - await send({ - 'type': 'http.response.body', - 'body': b'Hello, world!', - }) -``` - -Run the server: - -```shell -$ uvicorn example:app -``` - ---- - -## Why ASGI? - -Most well established Python Web frameworks started out as WSGI-based frameworks. - -WSGI applications are a single, synchronous callable that takes a request and returns a response. -This doesn’t allow for long-lived connections, like you get with long-poll HTTP or WebSocket connections, -which WSGI doesn't support well. - -Having an async concurrency model also allows for options such as lightweight background tasks, -and can be less of a limiting factor for endpoints that have long periods being blocked on network -I/O such as dealing with slow HTTP requests. - ---- - -## Alternative ASGI servers - -A strength of the ASGI protocol is that it decouples the server implementation -from the application framework. This allows for an ecosystem of interoperating -webservers and application frameworks. - -### Daphne - -The first ASGI server implementation, originally developed to power Django Channels, is [the Daphne webserver][daphne]. - -It is run widely in production, and supports HTTP/1.1, HTTP/2, and WebSockets. - -Any of the example applications given here can equally well be run using `daphne` instead. - -``` -$ pip install daphne -$ daphne app:App -``` - -### Hypercorn - -[Hypercorn][hypercorn] was initially part of the Quart web framework, before -being separated out into a standalone ASGI server. - -Hypercorn supports HTTP/1.1, HTTP/2, and WebSockets. - -It also supports [the excellent `trio` async framework][trio], as an alternative to `asyncio`. - -``` -$ pip install hypercorn -$ hypercorn app:App -``` - -### Mangum - -[Mangum][mangum] is an adapter for using ASGI applications with AWS Lambda & API Gateway. - -### Granian - -[Granian][granian] is an ASGI compatible Rust HTTP server which supports HTTP/2, TLS and WebSockets. - ---- - -

Uvicorn is BSD licensed code.
Designed & crafted with care.
— 🦄 —

- -[asgi]: https://asgi.readthedocs.io/en/latest/ -[daphne]: https://github.com/django/daphne -[hypercorn]: https://github.com/pgjones/hypercorn -[trio]: https://trio.readthedocs.io -[mangum]: https://github.com/jordaneremieff/mangum -[granian]: https://github.com/emmett-framework/granian diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/RECORD b/functions/python/data-processor/uvicorn-0.49.0.dist-info/RECORD deleted file mode 100644 index c8d5755a1..000000000 --- a/functions/python/data-processor/uvicorn-0.49.0.dist-info/RECORD +++ /dev/null @@ -1,89 +0,0 @@ -../../bin/uvicorn,sha256=uNFEHpmLACm5k5imFWAGgQ_QKmxNKCmf6wReA8ppGTs,187 -uvicorn-0.49.0.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 -uvicorn-0.49.0.dist-info/METADATA,sha256=xOeLMlH-BitEF0pF64qdEHEly7vgYuVBJ2wDxQ6RFDs,6738 -uvicorn-0.49.0.dist-info/RECORD,, -uvicorn-0.49.0.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn-0.49.0.dist-info/WHEEL,sha256=QccIxa26bgl1E6uMy58deGWi-0aeIkkangHcxk2kWfw,87 -uvicorn-0.49.0.dist-info/entry_points.txt,sha256=FW1w-hkc9QgwaGoovMvm0ZY73w_NcycWdGAUfDsNGxw,46 -uvicorn-0.49.0.dist-info/licenses/LICENSE.md,sha256=7-Gs8-YvuZwoiw7HPlp3O3Jo70Mg_nV-qZQhTktjw3E,1526 -uvicorn/__init__.py,sha256=dODKUIVij4EJeg9czHoZ1OATfCe3I_5KJK6LiQIMhZA,147 -uvicorn/__main__.py,sha256=DQizy6nKP0ywhPpnCHgmRDYIMfcqZKVEzNIWQZjqtVQ,62 -uvicorn/__pycache__/__init__.cpython-314.pyc,, -uvicorn/__pycache__/__main__.cpython-314.pyc,, -uvicorn/__pycache__/_compat.cpython-314.pyc,, -uvicorn/__pycache__/_subprocess.cpython-314.pyc,, -uvicorn/__pycache__/_types.cpython-314.pyc,, -uvicorn/__pycache__/config.cpython-314.pyc,, -uvicorn/__pycache__/importer.cpython-314.pyc,, -uvicorn/__pycache__/logging.cpython-314.pyc,, -uvicorn/__pycache__/main.cpython-314.pyc,, -uvicorn/__pycache__/server.cpython-314.pyc,, -uvicorn/__pycache__/workers.cpython-314.pyc,, -uvicorn/_compat.py,sha256=6X49c9ovzmMHir2S3syinOQMzSPBcp8MNuI9ZQrbYiU,2916 -uvicorn/_subprocess.py,sha256=fQ4JTajuXay6mVc8KQNGfXu6JHoab46aS6kyIMWJMPk,2776 -uvicorn/_types.py,sha256=JDM6E-hqZ6o_cSZvjDULjAv77PFH2pLe_Vgqshe8qH4,7677 -uvicorn/config.py,sha256=UhYkFgqeVqj38FZb6vwIn3qyVWYjr7u9eJfNP2LPhfc,23995 -uvicorn/importer.py,sha256=nRt0QQ3qpi264-n_mR0l55C2ddM8nowTNzT1jsWaam8,1128 -uvicorn/lifespan/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn/lifespan/__pycache__/__init__.cpython-314.pyc,, -uvicorn/lifespan/__pycache__/off.cpython-314.pyc,, -uvicorn/lifespan/__pycache__/on.cpython-314.pyc,, -uvicorn/lifespan/off.py,sha256=nfI6qHAUo_8-BEXMBKoHQ9wUbsXrPaXLCbDSS0vKSr8,332 -uvicorn/lifespan/on.py,sha256=iETPFj6SFhPDPcdeGxfa43LKImJU7MI5i7QUUepBcPE,5173 -uvicorn/logging.py,sha256=IBPSo2qRBWJRj5C2nN7I1civFAj9jKZ5Azxk0JpPB98,4235 -uvicorn/loops/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn/loops/__pycache__/__init__.cpython-314.pyc,, -uvicorn/loops/__pycache__/asyncio.cpython-314.pyc,, -uvicorn/loops/__pycache__/auto.cpython-314.pyc,, -uvicorn/loops/__pycache__/uvloop.cpython-314.pyc,, -uvicorn/loops/asyncio.py,sha256=a8wO4fB4pCk-RLICq7CnGr_cxQI7MBLJj-Go4UQldD8,333 -uvicorn/loops/auto.py,sha256=pdTsMcqkbE8fvdNgLs1y4c6hsuN598mZOSOlC0a23S4,566 -uvicorn/loops/uvloop.py,sha256=Ca-TL-W8tbdEW5uvL_GrMih7G8e_t_vhNevi2NKYjEU,236 -uvicorn/main.py,sha256=UdyHBAwxHz9_NYB3uzrn8XVnpL2PrX4JrPulJiFtWx8,18785 -uvicorn/middleware/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn/middleware/__pycache__/__init__.cpython-314.pyc,, -uvicorn/middleware/__pycache__/asgi2.cpython-314.pyc,, -uvicorn/middleware/__pycache__/message_logger.cpython-314.pyc,, -uvicorn/middleware/__pycache__/proxy_headers.cpython-314.pyc,, -uvicorn/middleware/__pycache__/wsgi.cpython-314.pyc,, -uvicorn/middleware/asgi2.py,sha256=YQrQNm3RehFts3mzk3k4yw8aD8Egtj0tRS3N45YkQa0,394 -uvicorn/middleware/message_logger.py,sha256=IHEZUSnFNaMFUFdwtZO3AuFATnYcSor-gVtOjbCzt8M,2859 -uvicorn/middleware/proxy_headers.py,sha256=2CEkCr_CsQYcNzZdnqkha8WB7vH6oX_S2sePHCOJZHc,7078 -uvicorn/middleware/wsgi.py,sha256=VnbnkdCL3_ctLWzk1j39xvxysuoAB3kSYmi8TsOLpNs,7125 -uvicorn/protocols/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn/protocols/__pycache__/__init__.cpython-314.pyc,, -uvicorn/protocols/__pycache__/utils.cpython-314.pyc,, -uvicorn/protocols/http/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn/protocols/http/__pycache__/__init__.cpython-314.pyc,, -uvicorn/protocols/http/__pycache__/auto.cpython-314.pyc,, -uvicorn/protocols/http/__pycache__/flow_control.cpython-314.pyc,, -uvicorn/protocols/http/__pycache__/h11_impl.cpython-314.pyc,, -uvicorn/protocols/http/__pycache__/httptools_impl.cpython-314.pyc,, -uvicorn/protocols/http/auto.py,sha256=YfXGyzWTaaE2p_jkTPWrJCXsxEaQnC3NK0-G7Wgmnls,403 -uvicorn/protocols/http/flow_control.py,sha256=050WVg31EvPOkHwynCoMP1zXFl_vO3U4durlc5vyp4U,1701 -uvicorn/protocols/http/h11_impl.py,sha256=K7SHLVsYORBde6cUGNwDt9rbS2vPfUbLkzbVOnEYeDk,21161 -uvicorn/protocols/http/httptools_impl.py,sha256=zJN4yQl26yzPqdyjYOFYimssb_Z8JfURMbJy8sqxLI0,22314 -uvicorn/protocols/utils.py,sha256=1hKdDCmo_GtGrK1zZfa70jL7Hr3mDE8RnL69O4rHQRg,2058 -uvicorn/protocols/websockets/__init__.py,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 -uvicorn/protocols/websockets/__pycache__/__init__.cpython-314.pyc,, -uvicorn/protocols/websockets/__pycache__/auto.cpython-314.pyc,, -uvicorn/protocols/websockets/__pycache__/websockets_impl.cpython-314.pyc,, -uvicorn/protocols/websockets/__pycache__/websockets_sansio_impl.cpython-314.pyc,, -uvicorn/protocols/websockets/__pycache__/wsproto_impl.cpython-314.pyc,, -uvicorn/protocols/websockets/auto.py,sha256=pAEMIsyXQ12CL6ffaEJQ2cQEC8yAr8sr7bC0H6hzBfs,590 -uvicorn/protocols/websockets/websockets_impl.py,sha256=O2EW1zGQmcB65-kbxkA0XnU-sbSbPSsUKvzh8N-QQoQ,14849 -uvicorn/protocols/websockets/websockets_sansio_impl.py,sha256=KnCKcpF7VEpu5wxudtk0u7SmwY2O3dk52Q8rrQ614Bs,20198 -uvicorn/protocols/websockets/wsproto_impl.py,sha256=JKmDJJ-tcb9PVvNkpP_wz-RsvonfwmSbDKWEX7GSNM4,19538 -uvicorn/py.typed,sha256=AbpHGcgLb-kRsJGnwFEktk7uzpZOCcBY74-YBdrKVGs,1 -uvicorn/server.py,sha256=fXk3__HBf1g-oh_4_OXIpb8tnTyafxuWRiEet6SQEvM,13368 -uvicorn/supervisors/__init__.py,sha256=wT8eOEIqT1yWQgytZtv5taWMul7xoTIY0xm1m4oyPTw,507 -uvicorn/supervisors/__pycache__/__init__.cpython-314.pyc,, -uvicorn/supervisors/__pycache__/basereload.cpython-314.pyc,, -uvicorn/supervisors/__pycache__/multiprocess.cpython-314.pyc,, -uvicorn/supervisors/__pycache__/statreload.cpython-314.pyc,, -uvicorn/supervisors/__pycache__/watchfilesreload.cpython-314.pyc,, -uvicorn/supervisors/basereload.py,sha256=mYOPopNS6fVTrqfQ4HzclH7Xw-VVQy7ZDSINv5_lu04,4018 -uvicorn/supervisors/multiprocess.py,sha256=_lU92Bnf_IqibFmmNoojY3J3fR6LzSBa76BpmcSMfHc,7580 -uvicorn/supervisors/statreload.py,sha256=XPaEtVXaasII1kkG32UUG8oD7-fS0FwyXkQLF9QYPbA,1548 -uvicorn/supervisors/watchfilesreload.py,sha256=u-MuYm0LsZMN670u_TtSWiMBUA2z4j1sIe3O2A67Ep4,2923 -uvicorn/workers.py,sha256=lMCubn1Wi-4h2jxrXNMmhOJZy7ZP8QxbZbP3_2O1iHE,3873 diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/REQUESTED b/functions/python/data-processor/uvicorn-0.49.0.dist-info/REQUESTED deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/WHEEL b/functions/python/data-processor/uvicorn-0.49.0.dist-info/WHEEL deleted file mode 100644 index b1b94fd58..000000000 --- a/functions/python/data-processor/uvicorn-0.49.0.dist-info/WHEEL +++ /dev/null @@ -1,4 +0,0 @@ -Wheel-Version: 1.0 -Generator: hatchling 1.29.0 -Root-Is-Purelib: true -Tag: py3-none-any diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/entry_points.txt b/functions/python/data-processor/uvicorn-0.49.0.dist-info/entry_points.txt deleted file mode 100644 index 4b00fcb60..000000000 --- a/functions/python/data-processor/uvicorn-0.49.0.dist-info/entry_points.txt +++ /dev/null @@ -1,2 +0,0 @@ -[console_scripts] -uvicorn = uvicorn.main:main diff --git a/functions/python/data-processor/uvicorn-0.49.0.dist-info/licenses/LICENSE.md b/functions/python/data-processor/uvicorn-0.49.0.dist-info/licenses/LICENSE.md deleted file mode 100644 index a6bba1455..000000000 --- a/functions/python/data-processor/uvicorn-0.49.0.dist-info/licenses/LICENSE.md +++ /dev/null @@ -1,27 +0,0 @@ -Copyright © 2017-present, [Encode OSS Ltd](https://www.encode.io/). -All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are met: - -* Redistributions of source code must retain the above copyright notice, this - list of conditions and the following disclaimer. - -* Redistributions in binary form must reproduce the above copyright notice, - this list of conditions and the following disclaimer in the documentation - and/or other materials provided with the distribution. - -* Neither the name of the copyright holder nor the names of its - contributors may be used to endorse or promote products derived from - this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" -AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE -IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE -FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL -DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER -CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, -OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/functions/python/data-processor/uvicorn/__init__.py b/functions/python/data-processor/uvicorn/__init__.py deleted file mode 100644 index 2a2760ac7..000000000 --- a/functions/python/data-processor/uvicorn/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from uvicorn.config import Config -from uvicorn.main import Server, main, run - -__version__ = "0.49.0" -__all__ = ["main", "run", "Config", "Server"] diff --git a/functions/python/data-processor/uvicorn/__main__.py b/functions/python/data-processor/uvicorn/__main__.py deleted file mode 100644 index 8a1dc979a..000000000 --- a/functions/python/data-processor/uvicorn/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -import uvicorn - -if __name__ == "__main__": - uvicorn.main() diff --git a/functions/python/data-processor/uvicorn/_compat.py b/functions/python/data-processor/uvicorn/_compat.py deleted file mode 100644 index 118a19ac0..000000000 --- a/functions/python/data-processor/uvicorn/_compat.py +++ /dev/null @@ -1,91 +0,0 @@ -from __future__ import annotations - -import asyncio -import sys -from collections.abc import Callable, Coroutine -from typing import Any, TypeVar - -__all__ = ["asyncio_run", "iscoroutinefunction"] - -if sys.version_info >= (3, 14): - from inspect import iscoroutinefunction -else: - from asyncio import iscoroutinefunction - -_T = TypeVar("_T") - -if sys.version_info >= (3, 12): - asyncio_run = asyncio.run -elif sys.version_info >= (3, 11): - - def asyncio_run( - main: Coroutine[Any, Any, _T], - *, - debug: bool = False, - loop_factory: Callable[[], asyncio.AbstractEventLoop] | None = None, - ) -> _T: - # asyncio.run from Python 3.12 - # https://docs.python.org/3/license.html#psf-license - with asyncio.Runner(debug=debug, loop_factory=loop_factory) as runner: - return runner.run(main) - -else: - # modified version of asyncio.run from Python 3.10 to add loop_factory kwarg - # https://docs.python.org/3/license.html#psf-license - def asyncio_run( - main: Coroutine[Any, Any, _T], - *, - debug: bool = False, - loop_factory: Callable[[], asyncio.AbstractEventLoop] | None = None, - ) -> _T: - try: - asyncio.get_running_loop() - except RuntimeError: - pass - else: - raise RuntimeError("asyncio.run() cannot be called from a running event loop") - - if not asyncio.iscoroutine(main): - raise ValueError(f"a coroutine was expected, got {main!r}") - - if loop_factory is None: - loop = asyncio.new_event_loop() - else: - loop = loop_factory() - try: - if loop_factory is None: - asyncio.set_event_loop(loop) - if debug is not None: - loop.set_debug(debug) - return loop.run_until_complete(main) - finally: - try: - _cancel_all_tasks(loop) - loop.run_until_complete(loop.shutdown_asyncgens()) - loop.run_until_complete(loop.shutdown_default_executor()) - finally: - if loop_factory is None: - asyncio.set_event_loop(None) - loop.close() - - def _cancel_all_tasks(loop: asyncio.AbstractEventLoop) -> None: - to_cancel = asyncio.all_tasks(loop) - if not to_cancel: - return - - for task in to_cancel: - task.cancel() - - loop.run_until_complete(asyncio.gather(*to_cancel, return_exceptions=True)) - - for task in to_cancel: - if task.cancelled(): - continue - if task.exception() is not None: - loop.call_exception_handler( - { - "message": "unhandled exception during asyncio.run() shutdown", - "exception": task.exception(), - "task": task, - } - ) diff --git a/functions/python/data-processor/uvicorn/_subprocess.py b/functions/python/data-processor/uvicorn/_subprocess.py deleted file mode 100644 index c084ec90b..000000000 --- a/functions/python/data-processor/uvicorn/_subprocess.py +++ /dev/null @@ -1,84 +0,0 @@ -""" -Some light wrappers around Python's multiprocessing, to deal with cleanly -starting child processes. -""" - -from __future__ import annotations - -import multiprocessing -import os -import sys -from collections.abc import Callable -from multiprocessing.context import SpawnProcess -from socket import socket - -from uvicorn.config import Config - -multiprocessing.allow_connection_pickling() -spawn = multiprocessing.get_context("spawn") - - -def get_subprocess( - config: Config, - target: Callable[..., None], - sockets: list[socket], -) -> SpawnProcess: - """ - Called in the parent process, to instantiate a new child process instance. - The child is not yet started at this point. - - * config - The Uvicorn configuration instance. - * target - A callable that accepts a list of sockets. In practice this will - be the `Server.run()` method. - * sockets - A list of sockets to pass to the server. Sockets are bound once - by the parent process, and then passed to the child processes. - """ - # We pass across the stdin fileno, and reopen it in the child process. - # This is required for some debugging environments. - try: - stdin_fileno = sys.stdin.fileno() - # The `sys.stdin` can be `None`, see https://docs.python.org/3/library/sys.html#sys.__stdin__. - except (AttributeError, OSError): - stdin_fileno = None - - kwargs = { - "config": config, - "target": target, - "sockets": sockets, - "stdin_fileno": stdin_fileno, - } - - return spawn.Process(target=subprocess_started, kwargs=kwargs) - - -def subprocess_started( - config: Config, - target: Callable[..., None], - sockets: list[socket], - stdin_fileno: int | None, -) -> None: - """ - Called when the child process starts. - - * config - The Uvicorn configuration instance. - * target - A callable that accepts a list of sockets. In practice this will - be the `Server.run()` method. - * sockets - A list of sockets to pass to the server. Sockets are bound once - by the parent process, and then passed to the child processes. - * stdin_fileno - The file number of sys.stdin, so that it can be reattached - to the child process. - """ - # Re-open stdin. - if stdin_fileno is not None: - sys.stdin = os.fdopen(stdin_fileno) # pragma: full coverage - - # Logging needs to be setup again for each child. - config.configure_logging() - - try: - # Now we can call into `Server.run(sockets=sockets)` - target(sockets=sockets) - except KeyboardInterrupt: # pragma: no cover - # suppress the exception to avoid a traceback from subprocess.Popen - # the parent already expects us to end, so no vital information is lost - pass diff --git a/functions/python/data-processor/uvicorn/_types.py b/functions/python/data-processor/uvicorn/_types.py deleted file mode 100644 index 822445a7b..000000000 --- a/functions/python/data-processor/uvicorn/_types.py +++ /dev/null @@ -1,274 +0,0 @@ -""" -Copyright (c) Django Software Foundation and individual contributors. -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, -are permitted provided that the following conditions are met: - - 1. Redistributions of source code must retain the above copyright notice, - this list of conditions and the following disclaimer. - - 2. Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in the - documentation and/or other materials provided with the distribution. - - 3. Neither the name of Django nor the names of its contributors may be used - to endorse or promote products derived from this software without - specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND -ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR -ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES -(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; -LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON -ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS -SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -""" - -from __future__ import annotations - -import sys -import types -from collections.abc import Awaitable, Callable, Iterable, MutableMapping -from typing import Any, Literal, Protocol, TypedDict - -if sys.version_info >= (3, 11): # pragma: py-lt-311 - from typing import NotRequired -else: # pragma: py-gte-311 - from typing_extensions import NotRequired - -# WSGI -Environ = MutableMapping[str, Any] -ExcInfo = tuple[type[BaseException], BaseException, types.TracebackType | None] -StartResponse = Callable[[str, Iterable[tuple[str, str]], ExcInfo | None], None] -WSGIApp = Callable[[Environ, StartResponse], Iterable[bytes] | BaseException] - - -# ASGI -class ASGIVersions(TypedDict): - spec_version: str - version: Literal["2.0"] | Literal["3.0"] - - -class HTTPScope(TypedDict): - type: Literal["http"] - asgi: ASGIVersions - http_version: str - method: str - scheme: str - path: str - raw_path: bytes - query_string: bytes - root_path: str - headers: Iterable[tuple[bytes, bytes]] - client: tuple[str, int] | None - server: tuple[str, int | None] | None - state: NotRequired[dict[str, Any]] - extensions: NotRequired[dict[str, dict[object, object]]] - - -class WebSocketScope(TypedDict): - type: Literal["websocket"] - asgi: ASGIVersions - http_version: str - scheme: str - path: str - raw_path: bytes - query_string: bytes - root_path: str - headers: Iterable[tuple[bytes, bytes]] - client: tuple[str, int] | None - server: tuple[str, int | None] | None - subprotocols: Iterable[str] - state: NotRequired[dict[str, Any]] - extensions: NotRequired[dict[str, dict[object, object]]] - - -class LifespanScope(TypedDict): - type: Literal["lifespan"] - asgi: ASGIVersions - state: NotRequired[dict[str, Any]] - - -WWWScope = HTTPScope | WebSocketScope -Scope = HTTPScope | WebSocketScope | LifespanScope - - -class HTTPRequestEvent(TypedDict): - type: Literal["http.request"] - body: bytes - more_body: bool - - -class HTTPResponseDebugEvent(TypedDict): - type: Literal["http.response.debug"] - info: dict[str, object] - - -class HTTPResponseStartEvent(TypedDict): - type: Literal["http.response.start"] - status: int - headers: NotRequired[Iterable[tuple[bytes, bytes]]] - trailers: NotRequired[bool] - - -class HTTPResponseBodyEvent(TypedDict): - type: Literal["http.response.body"] - body: bytes - more_body: NotRequired[bool] - - -class HTTPResponseTrailersEvent(TypedDict): - type: Literal["http.response.trailers"] - headers: Iterable[tuple[bytes, bytes]] - more_trailers: bool - - -class HTTPServerPushEvent(TypedDict): - type: Literal["http.response.push"] - path: str - headers: Iterable[tuple[bytes, bytes]] - - -class HTTPDisconnectEvent(TypedDict): - type: Literal["http.disconnect"] - - -class WebSocketConnectEvent(TypedDict): - type: Literal["websocket.connect"] - - -class WebSocketAcceptEvent(TypedDict): - type: Literal["websocket.accept"] - subprotocol: NotRequired[str | None] - headers: NotRequired[Iterable[tuple[bytes, bytes]]] - - -class _WebSocketReceiveEventBytes(TypedDict): - type: Literal["websocket.receive"] - bytes: bytes - text: NotRequired[None] - - -class _WebSocketReceiveEventText(TypedDict): - type: Literal["websocket.receive"] - bytes: NotRequired[None] - text: str - - -WebSocketReceiveEvent = _WebSocketReceiveEventBytes | _WebSocketReceiveEventText - - -class _WebSocketSendEventBytes(TypedDict): - type: Literal["websocket.send"] - bytes: bytes - text: NotRequired[None] - - -class _WebSocketSendEventText(TypedDict): - type: Literal["websocket.send"] - bytes: NotRequired[None] - text: str - - -WebSocketSendEvent = _WebSocketSendEventBytes | _WebSocketSendEventText - - -class WebSocketResponseStartEvent(TypedDict): - type: Literal["websocket.http.response.start"] - status: int - headers: Iterable[tuple[bytes, bytes]] - - -class WebSocketResponseBodyEvent(TypedDict): - type: Literal["websocket.http.response.body"] - body: bytes - more_body: NotRequired[bool] - - -class WebSocketDisconnectEvent(TypedDict): - type: Literal["websocket.disconnect"] - code: int - reason: NotRequired[str | None] - - -class WebSocketCloseEvent(TypedDict): - type: Literal["websocket.close"] - code: NotRequired[int] - reason: NotRequired[str | None] - - -class LifespanStartupEvent(TypedDict): - type: Literal["lifespan.startup"] - - -class LifespanShutdownEvent(TypedDict): - type: Literal["lifespan.shutdown"] - - -class LifespanStartupCompleteEvent(TypedDict): - type: Literal["lifespan.startup.complete"] - - -class LifespanStartupFailedEvent(TypedDict): - type: Literal["lifespan.startup.failed"] - message: str - - -class LifespanShutdownCompleteEvent(TypedDict): - type: Literal["lifespan.shutdown.complete"] - - -class LifespanShutdownFailedEvent(TypedDict): - type: Literal["lifespan.shutdown.failed"] - message: str - - -WebSocketEvent = WebSocketReceiveEvent | WebSocketDisconnectEvent | WebSocketConnectEvent - - -ASGIReceiveEvent = ( - HTTPRequestEvent - | HTTPDisconnectEvent - | WebSocketConnectEvent - | WebSocketReceiveEvent - | WebSocketDisconnectEvent - | LifespanStartupEvent - | LifespanShutdownEvent -) - - -ASGISendEvent = ( - HTTPResponseStartEvent - | HTTPResponseBodyEvent - | HTTPResponseTrailersEvent - | HTTPServerPushEvent - | HTTPDisconnectEvent - | WebSocketAcceptEvent - | WebSocketSendEvent - | WebSocketResponseStartEvent - | WebSocketResponseBodyEvent - | WebSocketCloseEvent - | LifespanStartupCompleteEvent - | LifespanStartupFailedEvent - | LifespanShutdownCompleteEvent - | LifespanShutdownFailedEvent -) - - -ASGIReceiveCallable = Callable[[], Awaitable[ASGIReceiveEvent]] -ASGISendCallable = Callable[[ASGISendEvent], Awaitable[None]] - - -class ASGI2Protocol(Protocol): - def __init__(self, scope: Scope) -> None: ... # pragma: no cover - - async def __call__(self, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: ... # pragma: no cover - - -ASGI2Application = type[ASGI2Protocol] -ASGI3Application = Callable[[Scope, ASGIReceiveCallable, ASGISendCallable], Awaitable[None]] -ASGIApplication = ASGI2Application | ASGI3Application diff --git a/functions/python/data-processor/uvicorn/config.py b/functions/python/data-processor/uvicorn/config.py deleted file mode 100644 index c17ba60a4..000000000 --- a/functions/python/data-processor/uvicorn/config.py +++ /dev/null @@ -1,588 +0,0 @@ -from __future__ import annotations - -import asyncio -import inspect -import json -import logging -import logging.config -import os -import socket -import ssl -import sys -from collections.abc import Awaitable, Callable -from configparser import RawConfigParser -from pathlib import Path -from typing import IO, Any, Literal - -import click - -from uvicorn._compat import iscoroutinefunction -from uvicorn._types import ASGIApplication -from uvicorn.importer import ImportFromStringError, import_from_string -from uvicorn.logging import TRACE_LOG_LEVEL -from uvicorn.middleware.asgi2 import ASGI2Middleware -from uvicorn.middleware.message_logger import MessageLoggerMiddleware -from uvicorn.middleware.proxy_headers import ProxyHeadersMiddleware -from uvicorn.middleware.wsgi import WSGIMiddleware - -HTTPProtocolType = Literal["auto", "h11", "httptools"] -WSProtocolType = Literal["auto", "none", "websockets", "websockets-sansio", "wsproto"] -LifespanType = Literal["auto", "on", "off"] -LoopFactoryType = Literal["none", "auto", "asyncio", "uvloop"] -InterfaceType = Literal["auto", "asgi3", "asgi2", "wsgi"] - -LOG_LEVELS: dict[str, int] = { - "critical": logging.CRITICAL, - "error": logging.ERROR, - "warning": logging.WARNING, - "info": logging.INFO, - "debug": logging.DEBUG, - "trace": TRACE_LOG_LEVEL, -} -HTTP_PROTOCOLS: dict[str, str] = { - "auto": "uvicorn.protocols.http.auto:AutoHTTPProtocol", - "h11": "uvicorn.protocols.http.h11_impl:H11Protocol", - "httptools": "uvicorn.protocols.http.httptools_impl:HttpToolsProtocol", -} -WS_PROTOCOLS: dict[str, str | None] = { - "auto": "uvicorn.protocols.websockets.auto:AutoWebSocketsProtocol", - "none": None, - "websockets": "uvicorn.protocols.websockets.websockets_impl:WebSocketProtocol", - "websockets-sansio": "uvicorn.protocols.websockets.websockets_sansio_impl:WebSocketsSansIOProtocol", - "wsproto": "uvicorn.protocols.websockets.wsproto_impl:WSProtocol", -} -LIFESPAN: dict[str, str] = { - "auto": "uvicorn.lifespan.on:LifespanOn", - "on": "uvicorn.lifespan.on:LifespanOn", - "off": "uvicorn.lifespan.off:LifespanOff", -} -LOOP_FACTORIES: dict[str, str | None] = { - "none": None, - "auto": "uvicorn.loops.auto:auto_loop_factory", - "asyncio": "uvicorn.loops.asyncio:asyncio_loop_factory", - "uvloop": "uvicorn.loops.uvloop:uvloop_loop_factory", -} -INTERFACES: list[InterfaceType] = ["auto", "asgi3", "asgi2", "wsgi"] - -SSL_PROTOCOL_VERSION: int = ssl.PROTOCOL_TLS_SERVER - -LOGGING_CONFIG: dict[str, Any] = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "default": { - "()": "uvicorn.logging.DefaultFormatter", - "fmt": "%(levelprefix)s %(message)s", - "use_colors": None, - }, - "access": { - "()": "uvicorn.logging.AccessFormatter", - "fmt": '%(levelprefix)s %(client_addr)s - "%(request_line)s" %(status_code)s', # noqa: E501 - }, - }, - "handlers": { - "default": { - "formatter": "default", - "class": "logging.StreamHandler", - "stream": "ext://sys.stderr", - }, - "access": { - "formatter": "access", - "class": "logging.StreamHandler", - "stream": "ext://sys.stdout", - }, - }, - "loggers": { - "uvicorn": {"handlers": ["default"], "level": "INFO", "propagate": False}, - "uvicorn.error": {"level": "INFO"}, - "uvicorn.access": {"handlers": ["access"], "level": "INFO", "propagate": False}, - }, -} - -logger = logging.getLogger("uvicorn.error") - - -def create_ssl_context( - certfile: str | os.PathLike[str], - keyfile: str | os.PathLike[str] | None, - password: str | None, - ssl_version: int, - cert_reqs: int, - ca_certs: str | os.PathLike[str] | None, - ciphers: str | None, -) -> ssl.SSLContext: - ctx = ssl.SSLContext(ssl_version) - get_password = (lambda: password) if password else None - ctx.load_cert_chain(certfile, keyfile, get_password) - ctx.verify_mode = ssl.VerifyMode(cert_reqs) - if ca_certs: - ctx.load_verify_locations(ca_certs) - if ciphers: - ctx.set_ciphers(ciphers) - return ctx - - -def is_dir(path: Path) -> bool: - try: - if not path.is_absolute(): - path = path.resolve() - return path.is_dir() - except OSError: # pragma: full coverage - return False - - -def resolve_reload_patterns(patterns_list: list[str], directories_list: list[str]) -> tuple[list[str], list[Path]]: - directories: list[Path] = list(set(map(Path, directories_list.copy()))) - patterns: list[str] = patterns_list.copy() - - current_working_directory = Path.cwd() - for pattern in patterns_list: - # Special case for the .* pattern, otherwise this would only match - # hidden directories which is probably undesired - if pattern == ".*": - continue # pragma: py-not-linux - patterns.append(pattern) - if is_dir(Path(pattern)): - directories.append(Path(pattern)) - else: - for match in current_working_directory.glob(pattern): - if is_dir(match): - directories.append(match) - - directories = list(set(directories)) - directories = list(map(Path, directories)) - directories = list(map(lambda x: x.resolve(), directories)) - directories = list({reload_path for reload_path in directories if is_dir(reload_path)}) - - children = [] - for j in range(len(directories)): - for k in range(j + 1, len(directories)): # pragma: full coverage - if directories[j] in directories[k].parents: - children.append(directories[k]) - elif directories[k] in directories[j].parents: - children.append(directories[j]) - - directories = list(set(directories).difference(set(children))) - - return list(set(patterns)), directories - - -def _normalize_dirs(dirs: list[str] | str | None) -> list[str]: - if dirs is None: - return [] - if isinstance(dirs, str): - return [dirs] - return list(set(dirs)) - - -class Config: - def __init__( - self, - app: ASGIApplication | Callable[..., Any] | str, - host: str = "127.0.0.1", - port: int = 8000, - uds: str | None = None, - fd: int | None = None, - loop: LoopFactoryType | str = "auto", - http: type[asyncio.Protocol] | HTTPProtocolType | str = "auto", - ws: type[asyncio.Protocol] | WSProtocolType | str = "auto", - ws_max_size: int = 16 * 1024 * 1024, - ws_max_queue: int = 32, - ws_ping_interval: float | None = 20.0, - ws_ping_timeout: float | None = 20.0, - ws_per_message_deflate: bool = True, - lifespan: LifespanType = "auto", - env_file: str | os.PathLike[str] | None = None, - log_config: dict[str, Any] | str | os.PathLike[str] | RawConfigParser | IO[Any] | None = LOGGING_CONFIG, - log_level: str | int | None = None, - access_log: bool = True, - use_colors: bool | None = None, - interface: InterfaceType = "auto", - reload: bool = False, - reload_dirs: list[str] | str | None = None, - reload_delay: float = 0.25, - reload_includes: list[str] | str | None = None, - reload_excludes: list[str] | str | None = None, - workers: int | None = None, - proxy_headers: bool = True, - server_header: bool = True, - date_header: bool = True, - forwarded_allow_ips: list[str] | str | None = None, - root_path: str = "", - limit_concurrency: int | None = None, - limit_max_requests: int | None = None, - limit_max_requests_jitter: int = 0, - backlog: int = 2048, - timeout_keep_alive: int = 5, - timeout_notify: int = 30, - timeout_graceful_shutdown: int | None = None, - timeout_worker_healthcheck: int = 5, - callback_notify: Callable[..., Awaitable[None]] | None = None, - ssl_keyfile: str | os.PathLike[str] | None = None, - ssl_certfile: str | os.PathLike[str] | None = None, - ssl_keyfile_password: str | None = None, - ssl_version: int = SSL_PROTOCOL_VERSION, - ssl_cert_reqs: int = ssl.CERT_NONE, - ssl_ca_certs: str | os.PathLike[str] | None = None, - ssl_ciphers: str | None = None, - ssl_context_factory: Callable[[Config, Callable[[], ssl.SSLContext]], ssl.SSLContext] | None = None, - headers: list[tuple[str, str]] | None = None, - factory: bool = False, - h11_max_incomplete_event_size: int | None = None, - reset_contextvars: bool = False, - ): - self.app = app - self.host = host - self.port = port - self.uds = uds - self.fd = fd - self.loop = loop - self.http = http - self.ws = ws - self.ws_max_size = ws_max_size - self.ws_max_queue = ws_max_queue - self.ws_ping_interval = ws_ping_interval - self.ws_ping_timeout = ws_ping_timeout - self.ws_per_message_deflate = ws_per_message_deflate - self.lifespan = lifespan - self.log_config = log_config - self.log_level = log_level - self.access_log = access_log - self.use_colors = use_colors - self.interface = interface - self.reload = reload - self.reload_delay = reload_delay - self.workers = workers or 1 - self.proxy_headers = proxy_headers - self.server_header = server_header - self.date_header = date_header - self.root_path = root_path - self.limit_concurrency = limit_concurrency - self.limit_max_requests = limit_max_requests - self.limit_max_requests_jitter = limit_max_requests_jitter - self.backlog = backlog - self.timeout_keep_alive = timeout_keep_alive - self.timeout_notify = timeout_notify - self.timeout_graceful_shutdown = timeout_graceful_shutdown - self.timeout_worker_healthcheck = timeout_worker_healthcheck - self.callback_notify = callback_notify - self.ssl_keyfile = ssl_keyfile - self.ssl_certfile = ssl_certfile - self.ssl_keyfile_password = ssl_keyfile_password - self.ssl_version = ssl_version - self.ssl_cert_reqs = ssl_cert_reqs - self.ssl_ca_certs = ssl_ca_certs - self.ssl_ciphers = ssl_ciphers - self.ssl_context_factory = ssl_context_factory - self.headers: list[tuple[str, str]] = headers or [] - self.encoded_headers: list[tuple[bytes, bytes]] = [] - self.factory = factory - self.h11_max_incomplete_event_size = h11_max_incomplete_event_size - self.reset_contextvars = reset_contextvars - - self.loaded = False - self.configure_logging() - - self.reload_dirs: list[Path] = [] - self.reload_dirs_excludes: list[Path] = [] - self.reload_includes: list[str] = [] - self.reload_excludes: list[str] = [] - - if (reload_dirs or reload_includes or reload_excludes) and not self.should_reload: - logger.warning( - "Current configuration will not reload as not all conditions are met, please refer to documentation." - ) - - if self.should_reload: - reload_dirs = _normalize_dirs(reload_dirs) - reload_includes = _normalize_dirs(reload_includes) - reload_excludes = _normalize_dirs(reload_excludes) - - self.reload_includes, self.reload_dirs = resolve_reload_patterns(reload_includes, reload_dirs) - - self.reload_excludes, self.reload_dirs_excludes = resolve_reload_patterns(reload_excludes, []) - - reload_dirs_tmp = self.reload_dirs.copy() - - for directory in self.reload_dirs_excludes: - for reload_directory in reload_dirs_tmp: - if directory == reload_directory or directory in reload_directory.parents: - try: - self.reload_dirs.remove(reload_directory) - except ValueError: # pragma: full coverage - pass - - for pattern in self.reload_excludes: - if pattern in self.reload_includes: - self.reload_includes.remove(pattern) # pragma: full coverage - - if not self.reload_dirs: - if reload_dirs: - logger.warning( - "Provided reload directories %s did not contain valid " - + "directories, watching current working directory.", - reload_dirs, - ) - self.reload_dirs = [Path.cwd()] - - logger.info( - "Will watch for changes in these directories: %s", - sorted(list(map(str, self.reload_dirs))), - ) - - if env_file is not None: - from dotenv import load_dotenv - - logger.info("Loading environment from '%s'", env_file) - load_dotenv(dotenv_path=env_file) - - if workers is None and "WEB_CONCURRENCY" in os.environ: - self.workers = int(os.environ["WEB_CONCURRENCY"]) - - self.forwarded_allow_ips: list[str] | str - if forwarded_allow_ips is None: - self.forwarded_allow_ips = os.environ.get("FORWARDED_ALLOW_IPS", "127.0.0.1") - else: - self.forwarded_allow_ips = forwarded_allow_ips # pragma: full coverage - - if self.reload and self.workers > 1: - logger.warning('"workers" flag is ignored when reloading is enabled.') - - @property - def asgi_version(self) -> Literal["2.0", "3.0"]: - mapping: dict[str, Literal["2.0", "3.0"]] = { - "asgi2": "2.0", - "asgi3": "3.0", - "wsgi": "3.0", - } - return mapping[self.interface] - - @property - def is_ssl(self) -> bool: - return bool(self.ssl_keyfile or self.ssl_certfile or self.ssl_context_factory) - - @property - def use_subprocess(self) -> bool: - return bool(self.reload or self.workers > 1) - - def configure_logging(self) -> None: - logging.addLevelName(TRACE_LOG_LEVEL, "TRACE") - - if self.log_config is not None: - if isinstance(self.log_config, os.PathLike): - self.log_config = os.fspath(self.log_config) - - if isinstance(self.log_config, dict): - if self.use_colors in (True, False): - self.log_config["formatters"]["default"]["use_colors"] = self.use_colors - self.log_config["formatters"]["access"]["use_colors"] = self.use_colors - logging.config.dictConfig(self.log_config) - elif isinstance(self.log_config, str) and self.log_config.endswith(".json"): - with open(self.log_config) as file: - loaded_config = json.load(file) - logging.config.dictConfig(loaded_config) - elif isinstance(self.log_config, str) and self.log_config.endswith((".yaml", ".yml")): - try: - import yaml - except ImportError as e: - raise ImportError( - "Install the PyYAML package or uvicorn[standard] to use `--log-config` with YAML files." - ) from e - - with open(self.log_config) as file: - loaded_config = yaml.safe_load(file) - logging.config.dictConfig(loaded_config) - else: - # See the note about fileConfig() here: - # https://docs.python.org/3/library/logging.config.html#configuration-file-format - logging.config.fileConfig(self.log_config, disable_existing_loggers=False) - - if self.log_level is not None: - if isinstance(self.log_level, str): - log_level = LOG_LEVELS[self.log_level.lower()] - else: - log_level = self.log_level - logging.getLogger("uvicorn.error").setLevel(log_level) - logging.getLogger("uvicorn.access").setLevel(log_level) - logging.getLogger("uvicorn.asgi").setLevel(log_level) - if self.access_log is False: - logging.getLogger("uvicorn.access").handlers = [] - logging.getLogger("uvicorn.access").propagate = False - - def load_app(self) -> Any: - """Import the app and return it. Exits on failure.""" - try: - return import_from_string(self.app) - except ImportFromStringError as exc: - logger.error("Error loading ASGI app. %s" % exc) - sys.exit(1) - - def load(self) -> None: - assert not self.loaded - - if self.ssl_context_factory is not None: - - def default_factory() -> ssl.SSLContext: - if not self.ssl_certfile: - raise RuntimeError( - "`default_ssl_context_factory()` requires `ssl_certfile` to be set on `Config`. " - "Either pass `ssl_certfile` (and optionally `ssl_keyfile`) or build the `SSLContext` " - "directly inside `ssl_context_factory` without calling the default factory." - ) - return create_ssl_context( - keyfile=self.ssl_keyfile, - certfile=self.ssl_certfile, - password=self.ssl_keyfile_password, - ssl_version=self.ssl_version, - cert_reqs=self.ssl_cert_reqs, - ca_certs=self.ssl_ca_certs, - ciphers=self.ssl_ciphers, - ) - - context = self.ssl_context_factory(self, default_factory) - if not isinstance(context, ssl.SSLContext): - raise TypeError(f"`ssl_context_factory` must return an `ssl.SSLContext`, got {type(context).__name__}") - self.ssl: ssl.SSLContext | None = context - elif self.is_ssl: - assert self.ssl_certfile - self.ssl = create_ssl_context( - keyfile=self.ssl_keyfile, - certfile=self.ssl_certfile, - password=self.ssl_keyfile_password, - ssl_version=self.ssl_version, - cert_reqs=self.ssl_cert_reqs, - ca_certs=self.ssl_ca_certs, - ciphers=self.ssl_ciphers, - ) - else: - self.ssl = None - - encoded_headers = [(key.lower().encode("latin1"), value.encode("latin1")) for key, value in self.headers] - self.encoded_headers = ( - [(b"server", b"uvicorn")] + encoded_headers - if b"server" not in dict(encoded_headers) and self.server_header - else encoded_headers - ) - - if isinstance(self.http, str): - http_protocol_class = import_from_string(HTTP_PROTOCOLS.get(self.http, self.http)) - self.http_protocol_class: type[asyncio.Protocol] = http_protocol_class - else: - self.http_protocol_class = self.http - - if isinstance(self.ws, str): - ws_protocol_class = import_from_string(WS_PROTOCOLS.get(self.ws, self.ws)) - self.ws_protocol_class: type[asyncio.Protocol] | None = ws_protocol_class - else: - self.ws_protocol_class = self.ws - - self.lifespan_class = import_from_string(LIFESPAN[self.lifespan]) - - self.loaded_app = self.load_app() - - try: - self.loaded_app = self.loaded_app() - except TypeError as exc: - if self.factory: - logger.error("Error loading ASGI app factory: %s", exc) - sys.exit(1) - else: - if not self.factory: - logger.warning( - "ASGI app factory detected. Using it, but please consider setting the --factory flag explicitly." - ) - - if self.interface == "auto": - if inspect.isclass(self.loaded_app): - use_asgi_3 = hasattr(self.loaded_app, "__await__") - elif inspect.isfunction(self.loaded_app): - use_asgi_3 = iscoroutinefunction(self.loaded_app) - else: - call = getattr(self.loaded_app, "__call__", None) - use_asgi_3 = iscoroutinefunction(call) - self.interface = "asgi3" if use_asgi_3 else "asgi2" - - if self.interface == "wsgi": - self.loaded_app = WSGIMiddleware(self.loaded_app) - self.ws_protocol_class = None - elif self.interface == "asgi2": - self.loaded_app = ASGI2Middleware(self.loaded_app) - - if logger.getEffectiveLevel() <= TRACE_LOG_LEVEL: - self.loaded_app = MessageLoggerMiddleware(self.loaded_app) - if self.proxy_headers: - self.loaded_app = ProxyHeadersMiddleware(self.loaded_app, trusted_hosts=self.forwarded_allow_ips) - - self.loaded = True - - def setup_event_loop(self) -> None: - raise AttributeError( - "The `setup_event_loop` method was replaced by `get_loop_factory` in uvicorn 0.36.0.\n" - "None of those methods are supposed to be used directly. If you are doing it, please let me know here: " - "https://github.com/Kludex/uvicorn/discussions/2706. Thank you, and sorry for the inconvenience." - ) - - def get_loop_factory(self) -> Callable[[], asyncio.AbstractEventLoop] | None: - if self.loop in LOOP_FACTORIES: - loop_factory: Callable[..., Any] | None = import_from_string(LOOP_FACTORIES[self.loop]) - else: - try: - return import_from_string(self.loop) - except ImportFromStringError as exc: - logger.error("Error loading custom loop setup function. %s" % exc) - sys.exit(1) - if loop_factory is None: - return None - return loop_factory(use_subprocess=self.use_subprocess) - - def bind_socket(self) -> socket.socket: - logger_args: list[str | int] - if self.uds is not None: # pragma: py-win32 - path = self.uds - sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - try: - sock.bind(path) - uds_perms = 0o666 - os.chmod(self.uds, uds_perms) - except OSError as exc: # pragma: full coverage - logger.error(exc) - sys.exit(1) - - message = "Uvicorn running on unix socket %s (Press CTRL+C to quit)" - sock_name_format = "%s" - color_message = "Uvicorn running on " + click.style(sock_name_format, bold=True) + " (Press CTRL+C to quit)" - logger_args = [self.uds] - elif self.fd is not None: # pragma: py-win32 - sock = socket.fromfd(self.fd, socket.AF_UNIX, socket.SOCK_STREAM) - message = "Uvicorn running on socket %s (Press CTRL+C to quit)" - fd_name_format = "%s" - color_message = "Uvicorn running on " + click.style(fd_name_format, bold=True) + " (Press CTRL+C to quit)" - logger_args = [sock.getsockname()] - else: - family = socket.AF_INET - addr_format = "%s://%s:%d" - - if self.host and ":" in self.host: # pragma: full coverage - # It's an IPv6 address. - family = socket.AF_INET6 - addr_format = "%s://[%s]:%d" - - sock = socket.socket(family=family) - sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - try: - sock.bind((self.host, self.port)) - except OSError as exc: # pragma: full coverage - logger.error(exc) - sys.exit(1) - - message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" - color_message = "Uvicorn running on " + click.style(addr_format, bold=True) + " (Press CTRL+C to quit)" - protocol_name = "https" if self.is_ssl else "http" - logger_args = [protocol_name, self.host, sock.getsockname()[1]] - logger.info(message, *logger_args, extra={"color_message": color_message}) - sock.set_inheritable(True) - return sock - - @property - def should_reload(self) -> bool: - return isinstance(self.app, str) and self.reload diff --git a/functions/python/data-processor/uvicorn/importer.py b/functions/python/data-processor/uvicorn/importer.py deleted file mode 100644 index f77520ee1..000000000 --- a/functions/python/data-processor/uvicorn/importer.py +++ /dev/null @@ -1,34 +0,0 @@ -import importlib -from typing import Any - - -class ImportFromStringError(Exception): - pass - - -def import_from_string(import_str: Any) -> Any: - if not isinstance(import_str, str): - return import_str - - module_str, _, attrs_str = import_str.partition(":") - if not module_str or not attrs_str: - message = 'Import string "{import_str}" must be in format ":".' - raise ImportFromStringError(message.format(import_str=import_str)) - - try: - module = importlib.import_module(module_str) - except ModuleNotFoundError as exc: - if exc.name != module_str: - raise exc from None - message = 'Could not import module "{module_str}".' - raise ImportFromStringError(message.format(module_str=module_str)) - - instance = module - try: - for attr_str in attrs_str.split("."): - instance = getattr(instance, attr_str) - except AttributeError: - message = 'Attribute "{attrs_str}" not found in module "{module_str}".' - raise ImportFromStringError(message.format(attrs_str=attrs_str, module_str=module_str)) - - return instance diff --git a/functions/python/data-processor/uvicorn/lifespan/__init__.py b/functions/python/data-processor/uvicorn/lifespan/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn/lifespan/off.py b/functions/python/data-processor/uvicorn/lifespan/off.py deleted file mode 100644 index 74554b1e2..000000000 --- a/functions/python/data-processor/uvicorn/lifespan/off.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -from typing import Any - -from uvicorn import Config - - -class LifespanOff: - def __init__(self, config: Config) -> None: - self.should_exit = False - self.state: dict[str, Any] = {} - - async def startup(self) -> None: - pass - - async def shutdown(self) -> None: - pass diff --git a/functions/python/data-processor/uvicorn/lifespan/on.py b/functions/python/data-processor/uvicorn/lifespan/on.py deleted file mode 100644 index 71ac39472..000000000 --- a/functions/python/data-processor/uvicorn/lifespan/on.py +++ /dev/null @@ -1,137 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -from asyncio import Queue -from typing import Any - -from uvicorn import Config -from uvicorn._types import ( - LifespanScope, - LifespanShutdownCompleteEvent, - LifespanShutdownEvent, - LifespanShutdownFailedEvent, - LifespanStartupCompleteEvent, - LifespanStartupEvent, - LifespanStartupFailedEvent, -) - -LifespanReceiveMessage = LifespanStartupEvent | LifespanShutdownEvent -LifespanSendMessage = ( - LifespanStartupFailedEvent - | LifespanShutdownFailedEvent - | LifespanStartupCompleteEvent - | LifespanShutdownCompleteEvent -) - - -STATE_TRANSITION_ERROR = "Got invalid state transition on lifespan protocol." - - -class LifespanOn: - def __init__(self, config: Config) -> None: - if not config.loaded: - config.load() - - self.config = config - self.logger = logging.getLogger("uvicorn.error") - self.startup_event = asyncio.Event() - self.shutdown_event = asyncio.Event() - self.receive_queue: Queue[LifespanReceiveMessage] = asyncio.Queue() - self.error_occurred = False - self.startup_failed = False - self.shutdown_failed = False - self.should_exit = False - self.state: dict[str, Any] = {} - - async def startup(self) -> None: - self.logger.info("Waiting for application startup.") - - loop = asyncio.get_event_loop() - main_lifespan_task = loop.create_task(self.main()) # noqa: F841 - # Keep a hard reference to prevent garbage collection - # See https://github.com/Kludex/uvicorn/pull/972 - startup_event: LifespanStartupEvent = {"type": "lifespan.startup"} - await self.receive_queue.put(startup_event) - await self.startup_event.wait() - - if self.startup_failed or (self.error_occurred and self.config.lifespan == "on"): - self.logger.error("Application startup failed. Exiting.") - self.should_exit = True - else: - self.logger.info("Application startup complete.") - - async def shutdown(self) -> None: - if self.error_occurred: - return - self.logger.info("Waiting for application shutdown.") - shutdown_event: LifespanShutdownEvent = {"type": "lifespan.shutdown"} - await self.receive_queue.put(shutdown_event) - await self.shutdown_event.wait() - - if self.shutdown_failed or (self.error_occurred and self.config.lifespan == "on"): - self.logger.error("Application shutdown failed. Exiting.") - self.should_exit = True - else: - self.logger.info("Application shutdown complete.") - - async def main(self) -> None: - try: - app = self.config.loaded_app - scope: LifespanScope = { - "type": "lifespan", - "asgi": {"version": self.config.asgi_version, "spec_version": "2.0"}, - "state": self.state, - } - await app(scope, self.receive, self.send) - except BaseException as exc: - self.asgi = None - self.error_occurred = True - if self.startup_failed or self.shutdown_failed: - return - if self.config.lifespan == "auto": - msg = "ASGI 'lifespan' protocol appears unsupported." - self.logger.info(msg) - else: - msg = "Exception in 'lifespan' protocol\n" - self.logger.error(msg, exc_info=exc) - finally: - self.startup_event.set() - self.shutdown_event.set() - - async def send(self, message: LifespanSendMessage) -> None: - assert message["type"] in ( - "lifespan.startup.complete", - "lifespan.startup.failed", - "lifespan.shutdown.complete", - "lifespan.shutdown.failed", - ) - - if message["type"] == "lifespan.startup.complete": - assert not self.startup_event.is_set(), STATE_TRANSITION_ERROR - assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR - self.startup_event.set() - - elif message["type"] == "lifespan.startup.failed": - assert not self.startup_event.is_set(), STATE_TRANSITION_ERROR - assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR - self.startup_event.set() - self.startup_failed = True - if message.get("message"): - self.logger.error(message["message"]) - - elif message["type"] == "lifespan.shutdown.complete": - assert self.startup_event.is_set(), STATE_TRANSITION_ERROR - assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR - self.shutdown_event.set() - - elif message["type"] == "lifespan.shutdown.failed": - assert self.startup_event.is_set(), STATE_TRANSITION_ERROR - assert not self.shutdown_event.is_set(), STATE_TRANSITION_ERROR - self.shutdown_event.set() - self.shutdown_failed = True - if message.get("message"): - self.logger.error(message["message"]) - - async def receive(self) -> LifespanReceiveMessage: - return await self.receive_queue.get() diff --git a/functions/python/data-processor/uvicorn/logging.py b/functions/python/data-processor/uvicorn/logging.py deleted file mode 100644 index bd1611531..000000000 --- a/functions/python/data-processor/uvicorn/logging.py +++ /dev/null @@ -1,117 +0,0 @@ -from __future__ import annotations - -import http -import logging -import sys -from copy import copy -from typing import Literal - -import click - -TRACE_LOG_LEVEL = 5 - - -class ColourizedFormatter(logging.Formatter): - """ - A custom log formatter class that: - - * Outputs the LOG_LEVEL with an appropriate color. - * If a log call includes an `extra={"color_message": ...}` it will be used - for formatting the output, instead of the plain text message. - """ - - level_name_colors = { - TRACE_LOG_LEVEL: lambda level_name: click.style(str(level_name), fg="blue"), - logging.DEBUG: lambda level_name: click.style(str(level_name), fg="cyan"), - logging.INFO: lambda level_name: click.style(str(level_name), fg="green"), - logging.WARNING: lambda level_name: click.style(str(level_name), fg="yellow"), - logging.ERROR: lambda level_name: click.style(str(level_name), fg="red"), - logging.CRITICAL: lambda level_name: click.style(str(level_name), fg="bright_red"), - } - - def __init__( - self, - fmt: str | None = None, - datefmt: str | None = None, - style: Literal["%", "{", "$"] = "%", - use_colors: bool | None = None, - ): - if use_colors in (True, False): - self.use_colors = use_colors - else: - self.use_colors = sys.stdout.isatty() - super().__init__(fmt=fmt, datefmt=datefmt, style=style) - - def color_level_name(self, level_name: str, level_no: int) -> str: - def default(level_name: str) -> str: - return str(level_name) # pragma: no cover - - func = self.level_name_colors.get(level_no, default) - return func(level_name) - - def should_use_colors(self) -> bool: - return True # pragma: no cover - - def formatMessage(self, record: logging.LogRecord) -> str: - recordcopy = copy(record) - levelname = recordcopy.levelname - separator = " " * (8 - len(recordcopy.levelname)) - if self.use_colors: - levelname = self.color_level_name(levelname, recordcopy.levelno) - if "color_message" in recordcopy.__dict__: - recordcopy.msg = recordcopy.__dict__["color_message"] - recordcopy.__dict__["message"] = recordcopy.getMessage() - recordcopy.__dict__["levelprefix"] = levelname + ":" + separator - return super().formatMessage(recordcopy) - - -class DefaultFormatter(ColourizedFormatter): - def should_use_colors(self) -> bool: - return sys.stderr.isatty() # pragma: no cover - - -class AccessFormatter(ColourizedFormatter): - status_code_colours = { - 1: lambda code: click.style(str(code), fg="bright_white"), - 2: lambda code: click.style(str(code), fg="green"), - 3: lambda code: click.style(str(code), fg="yellow"), - 4: lambda code: click.style(str(code), fg="red"), - 5: lambda code: click.style(str(code), fg="bright_red"), - } - - def get_status_code(self, status_code: int) -> str: - try: - status_phrase = http.HTTPStatus(status_code).phrase - except ValueError: - status_phrase = "" - status_and_phrase = f"{status_code} {status_phrase}" - if self.use_colors: - - def default(code: int) -> str: - return status_and_phrase # pragma: no cover - - func = self.status_code_colours.get(status_code // 100, default) - return func(status_and_phrase) - return status_and_phrase - - def formatMessage(self, record: logging.LogRecord) -> str: - recordcopy = copy(record) - ( - client_addr, - method, - full_path, - http_version, - status_code, - ) = recordcopy.args # type: ignore[misc] - status_code = self.get_status_code(int(status_code)) # type: ignore[arg-type] - request_line = f"{method} {full_path} HTTP/{http_version}" - if self.use_colors: - request_line = click.style(request_line, bold=True) - recordcopy.__dict__.update( - { - "client_addr": client_addr, - "request_line": request_line, - "status_code": status_code, - } - ) - return super().formatMessage(recordcopy) diff --git a/functions/python/data-processor/uvicorn/loops/__init__.py b/functions/python/data-processor/uvicorn/loops/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn/loops/asyncio.py b/functions/python/data-processor/uvicorn/loops/asyncio.py deleted file mode 100644 index ad6121ee0..000000000 --- a/functions/python/data-processor/uvicorn/loops/asyncio.py +++ /dev/null @@ -1,11 +0,0 @@ -from __future__ import annotations - -import asyncio -import sys -from collections.abc import Callable - - -def asyncio_loop_factory(use_subprocess: bool = False) -> Callable[[], asyncio.AbstractEventLoop]: - if sys.platform == "win32" and not use_subprocess: - return asyncio.ProactorEventLoop - return asyncio.SelectorEventLoop diff --git a/functions/python/data-processor/uvicorn/loops/auto.py b/functions/python/data-processor/uvicorn/loops/auto.py deleted file mode 100644 index 190839905..000000000 --- a/functions/python/data-processor/uvicorn/loops/auto.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import annotations - -import asyncio -from collections.abc import Callable - - -def auto_loop_factory(use_subprocess: bool = False) -> Callable[[], asyncio.AbstractEventLoop]: - try: - import uvloop # noqa - except ImportError: # pragma: no cover - from uvicorn.loops.asyncio import asyncio_loop_factory as loop_factory - - return loop_factory(use_subprocess=use_subprocess) - else: # pragma: no cover - from uvicorn.loops.uvloop import uvloop_loop_factory - - return uvloop_loop_factory(use_subprocess=use_subprocess) diff --git a/functions/python/data-processor/uvicorn/loops/uvloop.py b/functions/python/data-processor/uvicorn/loops/uvloop.py deleted file mode 100644 index c6692c58f..000000000 --- a/functions/python/data-processor/uvicorn/loops/uvloop.py +++ /dev/null @@ -1,10 +0,0 @@ -from __future__ import annotations - -import asyncio -from collections.abc import Callable - -import uvloop - - -def uvloop_loop_factory(use_subprocess: bool = False) -> Callable[[], asyncio.AbstractEventLoop]: - return uvloop.new_event_loop diff --git a/functions/python/data-processor/uvicorn/main.py b/functions/python/data-processor/uvicorn/main.py deleted file mode 100644 index 0b8eb0ead..000000000 --- a/functions/python/data-processor/uvicorn/main.py +++ /dev/null @@ -1,644 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import os -import platform -import ssl -import sys -import warnings -from collections.abc import Callable -from configparser import RawConfigParser -from typing import IO, Any, get_args - -import click - -import uvicorn -from uvicorn._types import ASGIApplication -from uvicorn.config import ( - INTERFACES, - LIFESPAN, - LOG_LEVELS, - LOGGING_CONFIG, - SSL_PROTOCOL_VERSION, - Config, - HTTPProtocolType, - InterfaceType, - LifespanType, - LoopFactoryType, - WSProtocolType, -) -from uvicorn.server import Server -from uvicorn.supervisors import ChangeReload, Multiprocess - -LEVEL_CHOICES = click.Choice(list(LOG_LEVELS.keys())) -LIFESPAN_CHOICES = click.Choice(list(LIFESPAN.keys())) -INTERFACE_CHOICES = click.Choice(INTERFACES) - - -def _metavar_from_type(_type: Any) -> str: - return f"[{'|'.join(key for key in get_args(_type) if key != 'none')}]" - - -STARTUP_FAILURE = 3 - -logger = logging.getLogger("uvicorn.error") - - -def print_version(ctx: click.Context, param: click.Parameter, value: bool) -> None: - if not value or ctx.resilient_parsing: - return - click.echo( - "Running uvicorn {version} with {py_implementation} {py_version} on {system}".format( # noqa: UP032 - version=uvicorn.__version__, - py_implementation=platform.python_implementation(), - py_version=platform.python_version(), - system=platform.system(), - ) - ) - ctx.exit() - - -@click.command(context_settings={"auto_envvar_prefix": "UVICORN"}) -@click.argument("app", envvar="UVICORN_APP") -@click.option( - "--host", - type=str, - default="127.0.0.1", - help="Bind socket to this host.", - show_default=True, -) -@click.option( - "--port", - type=int, - default=8000, - help="Bind socket to this port. If 0, an available port will be picked.", - show_default=True, -) -@click.option("--uds", type=str, default=None, help="Bind to a UNIX domain socket.") -@click.option("--fd", type=int, default=None, help="Bind to socket from this file descriptor.") -@click.option("--reload", is_flag=True, default=False, help="Enable auto-reload.") -@click.option( - "--reload-dir", - "reload_dirs", - multiple=True, - help="Set reload directories explicitly, instead of using the current working directory.", - type=click.Path(exists=True), -) -@click.option( - "--reload-include", - "reload_includes", - multiple=True, - help="Set glob patterns to include while watching for files. Includes '*.py' " - "by default; these defaults can be overridden with `--reload-exclude`. " - "This option has no effect unless watchfiles is installed.", -) -@click.option( - "--reload-exclude", - "reload_excludes", - multiple=True, - help="Set glob patterns to exclude while watching for files. Includes " - "'.*, .py[cod], .sw.*, ~*' by default; these defaults can be overridden " - "with `--reload-include`. This option has no effect unless watchfiles is " - "installed.", -) -@click.option( - "--reload-delay", - type=float, - default=0.25, - show_default=True, - help="Delay between previous and next check if application needs to be. Defaults to 0.25s.", -) -@click.option( - "--workers", - default=None, - type=int, - help="Number of worker processes. Defaults to the $WEB_CONCURRENCY environment" - " variable if available, or 1. Not valid with --reload.", -) -@click.option( - "--loop", - type=str, - metavar=_metavar_from_type(LoopFactoryType), - default="auto", - help="Event loop factory implementation.", - show_default=True, -) -@click.option( - "--http", - type=str, - metavar=_metavar_from_type(HTTPProtocolType), - default="auto", - help="HTTP protocol implementation.", - show_default=True, -) -@click.option( - "--ws", - type=str, - metavar=_metavar_from_type(WSProtocolType), - default="auto", - help="WebSocket protocol implementation.", - show_default=True, -) -@click.option( - "--ws-max-size", - type=int, - default=16777216, - help="WebSocket max size message in bytes", - show_default=True, -) -@click.option( - "--ws-max-queue", - type=int, - default=32, - help="The maximum length of the WebSocket message queue.", - show_default=True, -) -@click.option( - "--ws-ping-interval", - type=float, - default=20.0, - help="WebSocket ping interval in seconds.", - show_default=True, -) -@click.option( - "--ws-ping-timeout", - type=float, - default=20.0, - help="WebSocket ping timeout in seconds.", - show_default=True, -) -@click.option( - "--ws-per-message-deflate", - type=bool, - default=True, - help="WebSocket per-message-deflate compression", - show_default=True, -) -@click.option( - "--lifespan", - type=LIFESPAN_CHOICES, - default="auto", - help="Lifespan implementation.", - show_default=True, -) -@click.option( - "--interface", - type=INTERFACE_CHOICES, - default="auto", - help="Select ASGI3, ASGI2, or WSGI as the application interface.", - show_default=True, -) -@click.option( - "--env-file", - type=click.Path(exists=True), - default=None, - help="Environment configuration file.", - show_default=True, -) -@click.option( - "--log-config", - type=click.Path(exists=True), - default=None, - help="Logging configuration file. Supported formats: .ini, .json, .yaml.", - show_default=True, -) -@click.option( - "--log-level", - type=LEVEL_CHOICES, - default=None, - help="Log level. [default: info]", - show_default=True, -) -@click.option( - "--access-log/--no-access-log", - is_flag=True, - default=True, - help="Enable/Disable access log.", -) -@click.option( - "--use-colors/--no-use-colors", - is_flag=True, - default=None, - help="Enable/Disable colorized logging.", -) -@click.option( - "--proxy-headers/--no-proxy-headers", - is_flag=True, - default=True, - help="Enable/Disable X-Forwarded-Proto, X-Forwarded-For to populate url scheme and remote address info.", -) -@click.option( - "--server-header/--no-server-header", - is_flag=True, - default=True, - help="Enable/Disable default Server header.", -) -@click.option( - "--date-header/--no-date-header", - is_flag=True, - default=True, - help="Enable/Disable default Date header.", -) -@click.option( - "--forwarded-allow-ips", - type=str, - default=None, - help="Comma separated list of IP Addresses, IP Networks, or literals " - "(e.g. UNIX Socket path) to trust with proxy headers. Defaults to the " - "$FORWARDED_ALLOW_IPS environment variable if available, or '127.0.0.1'. " - "The literal '*' means trust everything.", -) -@click.option( - "--root-path", - type=str, - default="", - help="Set the ASGI 'root_path' for applications submounted below a given URL path.", -) -@click.option( - "--limit-concurrency", - type=int, - default=None, - help="Maximum number of concurrent connections or tasks to allow, before issuing HTTP 503 responses.", -) -@click.option( - "--backlog", - type=int, - default=2048, - help="Maximum number of connections to hold in backlog", -) -@click.option( - "--limit-max-requests", - type=int, - default=None, - help="Maximum number of requests to service before terminating the process.", -) -@click.option( - "--limit-max-requests-jitter", - type=int, - default=0, - help="Maximum jitter to add to limit_max_requests." - " Staggers worker restarts to avoid all workers restarting simultaneously.", - show_default=True, -) -@click.option( - "--timeout-keep-alive", - type=int, - default=5, - help="Close Keep-Alive connections if no new data is received within this timeout (in seconds).", - show_default=True, -) -@click.option( - "--timeout-graceful-shutdown", - type=int, - default=None, - help="Maximum number of seconds to wait for graceful shutdown.", -) -@click.option( - "--timeout-worker-healthcheck", - type=int, - default=5, - help="Maximum number of seconds to wait for a worker to respond to a healthcheck.", - show_default=True, -) -@click.option("--ssl-keyfile", type=str, default=None, help="SSL key file", show_default=True) -@click.option( - "--ssl-certfile", - type=str, - default=None, - help="SSL certificate file", - show_default=True, -) -@click.option( - "--ssl-keyfile-password", - type=str, - default=None, - help="SSL keyfile password", - show_default=True, -) -@click.option( - "--ssl-version", - type=int, - default=int(SSL_PROTOCOL_VERSION), - help="SSL version to use (see stdlib ssl module's)", - show_default=True, -) -@click.option( - "--ssl-cert-reqs", - type=int, - default=int(ssl.CERT_NONE), - help="Whether client certificate is required (see stdlib ssl module's)", - show_default=True, -) -@click.option( - "--ssl-ca-certs", - type=str, - default=None, - help="CA certificates file", - show_default=True, -) -@click.option( - "--ssl-ciphers", - type=str, - default=None, - help="Ciphers to use (see stdlib ssl module's). Defaults to OpenSSL's safe defaults.", - show_default=True, -) -@click.option( - "--header", - "headers", - multiple=True, - help="Specify custom default HTTP response headers as a Name:Value pair", -) -@click.option( - "--version", - is_flag=True, - callback=print_version, - expose_value=False, - is_eager=True, - help="Display the uvicorn version and exit.", -) -@click.option( - "--app-dir", - default="", - show_default=True, - help="Look for APP in the specified directory, by adding this to the PYTHONPATH." - " Defaults to the current working directory.", -) -@click.option( - "--h11-max-incomplete-event-size", - "h11_max_incomplete_event_size", - type=int, - default=None, - help="For h11, the maximum number of bytes to buffer of an incomplete event.", -) -@click.option( - "--reset-contextvars", - is_flag=True, - default=False, - help="Run each ASGI request in a fresh contextvars.Context. Hides context set in the lifespan.", - show_default=True, -) -@click.option( - "--factory", - is_flag=True, - default=False, - help="Treat APP as an application factory, i.e. a () -> callable.", - show_default=True, -) -def main( - app: str, - host: str, - port: int, - uds: str, - fd: int, - loop: LoopFactoryType | str, - http: HTTPProtocolType | str, - ws: WSProtocolType | str, - ws_max_size: int, - ws_max_queue: int, - ws_ping_interval: float, - ws_ping_timeout: float, - ws_per_message_deflate: bool, - lifespan: LifespanType, - interface: InterfaceType, - reload: bool, - reload_dirs: list[str], - reload_includes: list[str], - reload_excludes: list[str], - reload_delay: float, - workers: int, - env_file: str, - log_config: str, - log_level: str, - access_log: bool, - proxy_headers: bool, - server_header: bool, - date_header: bool, - forwarded_allow_ips: str, - root_path: str, - limit_concurrency: int, - backlog: int, - limit_max_requests: int, - limit_max_requests_jitter: int, - timeout_keep_alive: int, - timeout_graceful_shutdown: int | None, - timeout_worker_healthcheck: int, - ssl_keyfile: str, - ssl_certfile: str, - ssl_keyfile_password: str, - ssl_version: int, - ssl_cert_reqs: int, - ssl_ca_certs: str, - ssl_ciphers: str | None, - headers: list[str], - use_colors: bool, - app_dir: str, - h11_max_incomplete_event_size: int | None, - reset_contextvars: bool, - factory: bool, -) -> None: - run( - app, - host=host, - port=port, - uds=uds, - fd=fd, - loop=loop, - http=http, - ws=ws, - ws_max_size=ws_max_size, - ws_max_queue=ws_max_queue, - ws_ping_interval=ws_ping_interval, - ws_ping_timeout=ws_ping_timeout, - ws_per_message_deflate=ws_per_message_deflate, - lifespan=lifespan, - env_file=env_file, - log_config=LOGGING_CONFIG if log_config is None else log_config, - log_level=log_level, - access_log=access_log, - interface=interface, - reload=reload, - reload_dirs=reload_dirs or None, - reload_includes=reload_includes or None, - reload_excludes=reload_excludes or None, - reload_delay=reload_delay, - workers=workers, - proxy_headers=proxy_headers, - server_header=server_header, - date_header=date_header, - forwarded_allow_ips=forwarded_allow_ips, - root_path=root_path, - limit_concurrency=limit_concurrency, - backlog=backlog, - limit_max_requests=limit_max_requests, - limit_max_requests_jitter=limit_max_requests_jitter, - timeout_keep_alive=timeout_keep_alive, - timeout_graceful_shutdown=timeout_graceful_shutdown, - timeout_worker_healthcheck=timeout_worker_healthcheck, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_keyfile_password=ssl_keyfile_password, - ssl_version=ssl_version, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, - ssl_ciphers=ssl_ciphers, - headers=[header.split(":", 1) for header in headers], # type: ignore[misc] - use_colors=use_colors, - factory=factory, - app_dir=app_dir, - h11_max_incomplete_event_size=h11_max_incomplete_event_size, - reset_contextvars=reset_contextvars, - ) - - -def run( - app: ASGIApplication | Callable[..., Any] | str, - *, - host: str = "127.0.0.1", - port: int = 8000, - uds: str | None = None, - fd: int | None = None, - loop: LoopFactoryType | str = "auto", - http: type[asyncio.Protocol] | HTTPProtocolType | str = "auto", - ws: type[asyncio.Protocol] | WSProtocolType | str = "auto", - ws_max_size: int = 16777216, - ws_max_queue: int = 32, - ws_ping_interval: float | None = 20.0, - ws_ping_timeout: float | None = 20.0, - ws_per_message_deflate: bool = True, - lifespan: LifespanType = "auto", - interface: InterfaceType = "auto", - reload: bool = False, - reload_dirs: list[str] | str | None = None, - reload_includes: list[str] | str | None = None, - reload_excludes: list[str] | str | None = None, - reload_delay: float = 0.25, - workers: int | None = None, - env_file: str | os.PathLike[str] | None = None, - log_config: dict[str, Any] | str | os.PathLike[str] | RawConfigParser | IO[Any] | None = LOGGING_CONFIG, - log_level: str | int | None = None, - access_log: bool = True, - proxy_headers: bool = True, - server_header: bool = True, - date_header: bool = True, - forwarded_allow_ips: list[str] | str | None = None, - root_path: str = "", - limit_concurrency: int | None = None, - backlog: int = 2048, - limit_max_requests: int | None = None, - limit_max_requests_jitter: int = 0, - timeout_keep_alive: int = 5, - timeout_graceful_shutdown: int | None = None, - timeout_worker_healthcheck: int = 5, - ssl_keyfile: str | os.PathLike[str] | None = None, - ssl_certfile: str | os.PathLike[str] | None = None, - ssl_keyfile_password: str | None = None, - ssl_version: int = SSL_PROTOCOL_VERSION, - ssl_cert_reqs: int = ssl.CERT_NONE, - ssl_ca_certs: str | os.PathLike[str] | None = None, - ssl_ciphers: str | None = None, - ssl_context_factory: Callable[[Config, Callable[[], ssl.SSLContext]], ssl.SSLContext] | None = None, - headers: list[tuple[str, str]] | None = None, - use_colors: bool | None = None, - app_dir: str | None = None, - factory: bool = False, - h11_max_incomplete_event_size: int | None = None, - reset_contextvars: bool = False, -) -> None: - if app_dir is not None: - sys.path.insert(0, app_dir) - - config = Config( - app, - host=host, - port=port, - uds=uds, - fd=fd, - loop=loop, - http=http, - ws=ws, - ws_max_size=ws_max_size, - ws_max_queue=ws_max_queue, - ws_ping_interval=ws_ping_interval, - ws_ping_timeout=ws_ping_timeout, - ws_per_message_deflate=ws_per_message_deflate, - lifespan=lifespan, - interface=interface, - reload=reload, - reload_dirs=reload_dirs, - reload_includes=reload_includes, - reload_excludes=reload_excludes, - reload_delay=reload_delay, - workers=workers, - env_file=env_file, - log_config=log_config, - log_level=log_level, - access_log=access_log, - proxy_headers=proxy_headers, - server_header=server_header, - date_header=date_header, - forwarded_allow_ips=forwarded_allow_ips, - root_path=root_path, - limit_concurrency=limit_concurrency, - backlog=backlog, - limit_max_requests=limit_max_requests, - limit_max_requests_jitter=limit_max_requests_jitter, - timeout_keep_alive=timeout_keep_alive, - timeout_graceful_shutdown=timeout_graceful_shutdown, - timeout_worker_healthcheck=timeout_worker_healthcheck, - ssl_keyfile=ssl_keyfile, - ssl_certfile=ssl_certfile, - ssl_keyfile_password=ssl_keyfile_password, - ssl_version=ssl_version, - ssl_cert_reqs=ssl_cert_reqs, - ssl_ca_certs=ssl_ca_certs, - ssl_ciphers=ssl_ciphers, - ssl_context_factory=ssl_context_factory, - headers=headers, - use_colors=use_colors, - factory=factory, - h11_max_incomplete_event_size=h11_max_incomplete_event_size, - reset_contextvars=reset_contextvars, - ) - if (config.reload or config.workers > 1) and not isinstance(app, str): - logger = logging.getLogger("uvicorn.error") - logger.warning("You must pass the application as an import string to enable 'reload' or 'workers'.") - sys.exit(1) - - config.load_app() - server = Server(config=config) - - try: - if config.should_reload: - sock = config.bind_socket() - ChangeReload(config, target=server.run, sockets=[sock]).run() - elif config.workers > 1: - sock = config.bind_socket() - Multiprocess(config, target=server.run, sockets=[sock]).run() - else: - server.run() - except KeyboardInterrupt: # pragma: full coverage - pass - finally: - if config.uds and os.path.exists(config.uds): - os.remove(config.uds) # pragma: py-win32 - - if not server.started and not config.should_reload and config.workers == 1: - sys.exit(STARTUP_FAILURE) - - -def __getattr__(name: str) -> Any: - if name == "ServerState": - warnings.warn( - "uvicorn.main.ServerState is deprecated, use uvicorn.server.ServerState instead.", - DeprecationWarning, - ) - from uvicorn.server import ServerState - - return ServerState - raise AttributeError(f"module {__name__} has no attribute {name}") - - -if __name__ == "__main__": - main() # pragma: no cover diff --git a/functions/python/data-processor/uvicorn/middleware/__init__.py b/functions/python/data-processor/uvicorn/middleware/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn/middleware/asgi2.py b/functions/python/data-processor/uvicorn/middleware/asgi2.py deleted file mode 100644 index 4e15d1599..000000000 --- a/functions/python/data-processor/uvicorn/middleware/asgi2.py +++ /dev/null @@ -1,15 +0,0 @@ -from uvicorn._types import ( - ASGI2Application, - ASGIReceiveCallable, - ASGISendCallable, - Scope, -) - - -class ASGI2Middleware: - def __init__(self, app: "ASGI2Application"): - self.app = app - - async def __call__(self, scope: "Scope", receive: "ASGIReceiveCallable", send: "ASGISendCallable") -> None: - instance = self.app(scope) - await instance(receive, send) diff --git a/functions/python/data-processor/uvicorn/middleware/message_logger.py b/functions/python/data-processor/uvicorn/middleware/message_logger.py deleted file mode 100644 index 0174bcce6..000000000 --- a/functions/python/data-processor/uvicorn/middleware/message_logger.py +++ /dev/null @@ -1,87 +0,0 @@ -import logging -from typing import Any - -from uvicorn._types import ( - ASGI3Application, - ASGIReceiveCallable, - ASGIReceiveEvent, - ASGISendCallable, - ASGISendEvent, - WWWScope, -) -from uvicorn.logging import TRACE_LOG_LEVEL - -PLACEHOLDER_FORMAT = { - "body": "<{length} bytes>", - "bytes": "<{length} bytes>", - "text": "<{length} chars>", - "headers": "<...>", -} - - -def message_with_placeholders(message: Any) -> Any: - """ - Return an ASGI message, with any body-type content omitted and replaced - with a placeholder. - """ - new_message = message.copy() - for attr in PLACEHOLDER_FORMAT.keys(): - if message.get(attr) is not None: - content = message[attr] - placeholder = PLACEHOLDER_FORMAT[attr].format(length=len(content)) - new_message[attr] = placeholder - return new_message - - -class MessageLoggerMiddleware: - def __init__(self, app: "ASGI3Application"): - self.task_counter = 0 - self.app = app - self.logger = logging.getLogger("uvicorn.asgi") - - def trace(message: Any, *args: Any, **kwargs: Any) -> None: - self.logger.log(TRACE_LOG_LEVEL, message, *args, **kwargs) - - self.logger.trace = trace # type: ignore - - async def __call__( - self, - scope: "WWWScope", - receive: "ASGIReceiveCallable", - send: "ASGISendCallable", - ) -> None: - self.task_counter += 1 - - task_counter = self.task_counter - client = scope.get("client") - prefix = "%s:%d - ASGI" % (client[0], client[1]) if client else "ASGI" - - async def inner_receive() -> "ASGIReceiveEvent": - message = await receive() - logged_message = message_with_placeholders(message) - log_text = "%s [%d] Receive %s" - self.logger.trace( # type: ignore - log_text, prefix, task_counter, logged_message - ) - return message - - async def inner_send(message: "ASGISendEvent") -> None: - logged_message = message_with_placeholders(message) - log_text = "%s [%d] Send %s" - self.logger.trace( # type: ignore - log_text, prefix, task_counter, logged_message - ) - await send(message) - - logged_scope = message_with_placeholders(scope) - log_text = "%s [%d] Started scope=%s" - self.logger.trace(log_text, prefix, task_counter, logged_scope) # type: ignore - try: - await self.app(scope, inner_receive, inner_send) - except BaseException as exc: - log_text = "%s [%d] Raised exception" - self.logger.trace(log_text, prefix, task_counter) # type: ignore - raise exc from None - else: - log_text = "%s [%d] Completed" - self.logger.trace(log_text, prefix, task_counter) # type: ignore diff --git a/functions/python/data-processor/uvicorn/middleware/proxy_headers.py b/functions/python/data-processor/uvicorn/middleware/proxy_headers.py deleted file mode 100644 index f1944694c..000000000 --- a/functions/python/data-processor/uvicorn/middleware/proxy_headers.py +++ /dev/null @@ -1,180 +0,0 @@ -from __future__ import annotations - -import ipaddress - -from uvicorn._types import ASGI3Application, ASGIReceiveCallable, ASGISendCallable, Scope - - -class ProxyHeadersMiddleware: - """Middleware for handling known proxy headers - - This middleware can be used when a known proxy is fronting the application, - and is trusted to be properly setting the `X-Forwarded-Proto` and - `X-Forwarded-For` headers with the connecting client information. - - Modifies the `client` and `scheme` information so that they reference - the connecting client, rather that the connecting proxy. - - References: - - - - - """ - - def __init__(self, app: ASGI3Application, trusted_hosts: list[str] | str = "127.0.0.1") -> None: - self.app = app - self.trusted_hosts = _TrustedHosts(trusted_hosts) - - async def __call__(self, scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: - if scope["type"] == "lifespan": - return await self.app(scope, receive, send) - - client_addr = scope.get("client") - client_host = client_addr[0] if client_addr else None - - if client_host in self.trusted_hosts: - x_forwarded_proto_value: bytes | None = None - x_forwarded_for_values: list[bytes] = [] - for name, value in scope["headers"]: - if name == b"x-forwarded-proto": - x_forwarded_proto_value = value - elif name == b"x-forwarded-for": - x_forwarded_for_values.append(value) - - if x_forwarded_proto_value is not None: - x_forwarded_proto = x_forwarded_proto_value.decode("latin1").strip() - - if x_forwarded_proto in {"http", "https", "ws", "wss"}: - if scope["type"] == "websocket": - scope["scheme"] = x_forwarded_proto.replace("http", "ws") - else: - scope["scheme"] = x_forwarded_proto - - if x_forwarded_for_values: - x_forwarded_for = b", ".join(x_forwarded_for_values).decode("latin1") - host, port = self.trusted_hosts.get_trusted_client_address(x_forwarded_for) - - if host: - # If the x-forwarded-for header is empty then host is an empty string. - # Only set the client if we actually got something usable. - # See: https://github.com/Kludex/uvicorn/issues/1068 - scope["client"] = (host, port) - - return await self.app(scope, receive, send) - - -def _parse_raw_hosts(value: str) -> list[str]: - return [item.strip() for item in value.split(",")] - - -def _parse_host_port(value: str) -> tuple[str, int]: - """Parse a forwarded host value into host and optional port. - - Accepts bare IPs, IPv4 `host:port`, and bracketed IPv6 `[host]:port`. - Any unrecognized or malformed value is treated conservatively and returned - without a port so trust checks do not silently normalize arbitrary input. - """ - - if value.startswith("["): - bracket_end = value.find("]") - if bracket_end == -1: - return value, 0 - - host = value[1:bracket_end] - remainder = value[bracket_end + 1 :] - if not remainder: - return host, 0 - if not remainder.startswith(":"): - return value, 0 - - try: - return host, int(remainder[1:]) - except ValueError: - return host, 0 - - if value.count(":") == 1: - host, port = value.rsplit(":", 1) - try: - return host, int(port) - except ValueError: - return value, 0 - - return value, 0 - - -class _TrustedHosts: - """Container for trusted hosts and networks""" - - def __init__(self, trusted_hosts: list[str] | str) -> None: - self.always_trust: bool = trusted_hosts in ("*", ["*"]) - - self.trusted_literals: set[str] = set() - self.trusted_hosts: set[ipaddress.IPv4Address | ipaddress.IPv6Address] = set() - self.trusted_networks: set[ipaddress.IPv4Network | ipaddress.IPv6Network] = set() - - # Notes: - # - We separate hosts from literals as there are many ways to write - # an IPv6 Address so we need to compare by object. - # - We don't convert IP Address to single host networks (e.g. /32 / 128) as - # it more efficient to do an address lookup in a set than check for - # membership in each network. - # - We still allow literals as it might be possible that we receive a - # something that isn't an IP Address e.g. a unix socket. - - if not self.always_trust: - if isinstance(trusted_hosts, str): - trusted_hosts = _parse_raw_hosts(trusted_hosts) - - for host in trusted_hosts: - # Note: because we always convert invalid IP types to literals it - # is not possible for the user to know they provided a malformed IP - # type - this may lead to unexpected / difficult to debug behaviour. - - if "/" in host: - # Looks like a network - try: - self.trusted_networks.add(ipaddress.ip_network(host)) - except ValueError: - # Was not a valid IP Network - self.trusted_literals.add(host) - else: - try: - self.trusted_hosts.add(ipaddress.ip_address(host)) - except ValueError: - # Was not a valid IP Address - self.trusted_literals.add(host) - - def __contains__(self, host: str | None) -> bool: - if self.always_trust: - return True - - if not host: - return False - - try: - ip = ipaddress.ip_address(host) - if ip in self.trusted_hosts: - return True - return any(ip in net for net in self.trusted_networks) - - except ValueError: - return host in self.trusted_literals - - def get_trusted_client_address(self, x_forwarded_for: str) -> tuple[str, int]: - """Extract the client address from x_forwarded_for header. - - In general this is the first "untrusted" host in the forwarded for list. - """ - x_forwarded_for_hosts = _parse_raw_hosts(x_forwarded_for) - - if self.always_trust: - return _parse_host_port(x_forwarded_for_hosts[0]) - - # Note: each proxy appends to the header list so check it in reverse order - for host_port in reversed(x_forwarded_for_hosts): - host, port = _parse_host_port(host_port) - if host not in self: - return host, port - - # All hosts are trusted meaning that the client was also a trusted proxy - # See https://github.com/Kludex/uvicorn/issues/1068#issuecomment-855371576 - return _parse_host_port(x_forwarded_for_hosts[0]) diff --git a/functions/python/data-processor/uvicorn/middleware/wsgi.py b/functions/python/data-processor/uvicorn/middleware/wsgi.py deleted file mode 100644 index b2e3cb77d..000000000 --- a/functions/python/data-processor/uvicorn/middleware/wsgi.py +++ /dev/null @@ -1,199 +0,0 @@ -from __future__ import annotations - -import asyncio -import concurrent.futures -import io -import sys -import warnings -from collections import deque -from collections.abc import Iterable - -from uvicorn._types import ( - ASGIReceiveCallable, - ASGIReceiveEvent, - ASGISendCallable, - ASGISendEvent, - Environ, - ExcInfo, - HTTPRequestEvent, - HTTPResponseBodyEvent, - HTTPResponseStartEvent, - HTTPScope, - StartResponse, - WSGIApp, -) - - -def build_environ(scope: HTTPScope, message: ASGIReceiveEvent, body: io.BytesIO) -> Environ: - """ - Builds a scope and request message into a WSGI environ object. - """ - script_name = scope.get("root_path", "").encode("utf8").decode("latin1") - path_info = scope["path"].encode("utf8").decode("latin1") - if path_info.startswith(script_name): - path_info = path_info[len(script_name) :] - environ = { - "REQUEST_METHOD": scope["method"], - "SCRIPT_NAME": script_name, - "PATH_INFO": path_info, - "QUERY_STRING": scope["query_string"].decode("ascii"), - "SERVER_PROTOCOL": "HTTP/%s" % scope["http_version"], - "wsgi.version": (1, 0), - "wsgi.url_scheme": scope.get("scheme", "http"), - "wsgi.input": body, - "wsgi.errors": sys.stdout, - "wsgi.multithread": True, - "wsgi.multiprocess": True, - "wsgi.run_once": False, - } - - # Get server name and port - required in WSGI, not in ASGI - server = scope.get("server") - if server is None: - server = ("localhost", 80) - environ["SERVER_NAME"] = server[0] - environ["SERVER_PORT"] = server[1] - - # Get client IP address - client = scope.get("client") - if client is not None: - environ["REMOTE_ADDR"] = client[0] - - # Go through headers and make them into environ entries - for name, value in scope.get("headers", []): - name_str: str = name.decode("latin1") - if name_str == "content-length": - corrected_name = "CONTENT_LENGTH" - elif name_str == "content-type": - corrected_name = "CONTENT_TYPE" - else: - corrected_name = "HTTP_%s" % name_str.upper().replace("-", "_") - # HTTPbis say only ASCII chars are allowed in headers, but we latin1 - # just in case - value_str: str = value.decode("latin1") - if corrected_name in environ: - corrected_name_environ = environ[corrected_name] - assert isinstance(corrected_name_environ, str) - value_str = corrected_name_environ + "," + value_str - environ[corrected_name] = value_str - return environ - - -class _WSGIMiddleware: - def __init__(self, app: WSGIApp, workers: int = 10): - warnings.warn( - "Uvicorn's native WSGI implementation is deprecated, you should switch to a2wsgi (`pip install a2wsgi`).", - DeprecationWarning, - ) - self.app = app - self.executor = concurrent.futures.ThreadPoolExecutor(max_workers=workers) - - async def __call__( - self, - scope: HTTPScope, - receive: ASGIReceiveCallable, - send: ASGISendCallable, - ) -> None: - assert scope["type"] == "http" - instance = WSGIResponder(self.app, self.executor, scope) - await instance(receive, send) - - -class WSGIResponder: - def __init__( - self, - app: WSGIApp, - executor: concurrent.futures.ThreadPoolExecutor, - scope: HTTPScope, - ): - self.app = app - self.executor = executor - self.scope = scope - self.status = None - self.response_headers = None - self.send_event = asyncio.Event() - self.send_queue: deque[ASGISendEvent | None] = deque() - self.loop: asyncio.AbstractEventLoop = asyncio.get_event_loop() - self.response_started = False - self.exc_info: ExcInfo | None = None - - async def __call__(self, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: - message: HTTPRequestEvent = await receive() # type: ignore[assignment] - body = io.BytesIO(message.get("body", b"")) - more_body = message.get("more_body", False) - if more_body: - body.seek(0, io.SEEK_END) - while more_body: - body_message: HTTPRequestEvent = ( - await receive() # type: ignore[assignment] - ) - body.write(body_message.get("body", b"")) - more_body = body_message.get("more_body", False) - body.seek(0) - environ = build_environ(self.scope, message, body) - self.loop = asyncio.get_event_loop() - wsgi = self.loop.run_in_executor(self.executor, self.wsgi, environ, self.start_response) - sender = self.loop.create_task(self.sender(send)) - try: - await asyncio.wait_for(wsgi, None) - finally: - self.send_queue.append(None) - self.send_event.set() - await asyncio.wait_for(sender, None) - if self.exc_info is not None: - raise self.exc_info[0].with_traceback(self.exc_info[1], self.exc_info[2]) - - async def sender(self, send: ASGISendCallable) -> None: - while True: - if self.send_queue: - message = self.send_queue.popleft() - if message is None: - return - await send(message) - else: # pragma: no cover - await self.send_event.wait() - self.send_event.clear() - - def start_response( - self, - status: str, - response_headers: Iterable[tuple[str, str]], - exc_info: ExcInfo | None = None, - ) -> None: - self.exc_info = exc_info - if not self.response_started: - self.response_started = True - status_code_str, _ = status.split(" ", 1) - status_code = int(status_code_str) - headers = [(name.encode("ascii"), value.encode("ascii")) for name, value in response_headers] - http_response_start_event: HTTPResponseStartEvent = { - "type": "http.response.start", - "status": status_code, - "headers": headers, - } - self.send_queue.append(http_response_start_event) - self.loop.call_soon_threadsafe(self.send_event.set) - - def wsgi(self, environ: Environ, start_response: StartResponse) -> None: - for chunk in self.app(environ, start_response): # type: ignore - response_body: HTTPResponseBodyEvent = { - "type": "http.response.body", - "body": chunk, - "more_body": True, - } - self.send_queue.append(response_body) - self.loop.call_soon_threadsafe(self.send_event.set) - - empty_body: HTTPResponseBodyEvent = { - "type": "http.response.body", - "body": b"", - "more_body": False, - } - self.send_queue.append(empty_body) - self.loop.call_soon_threadsafe(self.send_event.set) - - -try: - from a2wsgi import WSGIMiddleware -except ModuleNotFoundError: # pragma: no cover - WSGIMiddleware = _WSGIMiddleware # type: ignore[misc, assignment] diff --git a/functions/python/data-processor/uvicorn/protocols/__init__.py b/functions/python/data-processor/uvicorn/protocols/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn/protocols/http/__init__.py b/functions/python/data-processor/uvicorn/protocols/http/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn/protocols/http/auto.py b/functions/python/data-processor/uvicorn/protocols/http/auto.py deleted file mode 100644 index a14bec144..000000000 --- a/functions/python/data-processor/uvicorn/protocols/http/auto.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import annotations - -import asyncio - -AutoHTTPProtocol: type[asyncio.Protocol] -try: - import httptools # noqa -except ImportError: # pragma: no cover - from uvicorn.protocols.http.h11_impl import H11Protocol - - AutoHTTPProtocol = H11Protocol -else: # pragma: no cover - from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol - - AutoHTTPProtocol = HttpToolsProtocol diff --git a/functions/python/data-processor/uvicorn/protocols/http/flow_control.py b/functions/python/data-processor/uvicorn/protocols/http/flow_control.py deleted file mode 100644 index 2d1b5fa2d..000000000 --- a/functions/python/data-processor/uvicorn/protocols/http/flow_control.py +++ /dev/null @@ -1,54 +0,0 @@ -import asyncio - -from uvicorn._types import ASGIReceiveCallable, ASGISendCallable, Scope - -CLOSE_HEADER = (b"connection", b"close") - -HIGH_WATER_LIMIT = 65536 - - -class FlowControl: - def __init__(self, transport: asyncio.Transport) -> None: - self._transport = transport - self.read_paused = False - self.write_paused = False - self._is_writable_event = asyncio.Event() - self._is_writable_event.set() - - async def drain(self) -> None: - await self._is_writable_event.wait() # pragma: full coverage - - def pause_reading(self) -> None: - if not self.read_paused: - self.read_paused = True - self._transport.pause_reading() - - def resume_reading(self) -> None: - if self.read_paused: - self.read_paused = False - self._transport.resume_reading() - - def pause_writing(self) -> None: - if not self.write_paused: # pragma: full coverage - self.write_paused = True - self._is_writable_event.clear() - - def resume_writing(self) -> None: - if self.write_paused: # pragma: full coverage - self.write_paused = False - self._is_writable_event.set() - - -async def service_unavailable(scope: Scope, receive: ASGIReceiveCallable, send: ASGISendCallable) -> None: - await send( - { - "type": "http.response.start", - "status": 503, - "headers": [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"content-length", b"19"), - (b"connection", b"close"), - ], - } - ) - await send({"type": "http.response.body", "body": b"Service Unavailable", "more_body": False}) diff --git a/functions/python/data-processor/uvicorn/protocols/http/h11_impl.py b/functions/python/data-processor/uvicorn/protocols/http/h11_impl.py deleted file mode 100644 index cb68ea0ec..000000000 --- a/functions/python/data-processor/uvicorn/protocols/http/h11_impl.py +++ /dev/null @@ -1,544 +0,0 @@ -from __future__ import annotations - -import asyncio -import contextvars -import http -import logging -import sys -from collections.abc import Callable -from typing import Any, Literal -from urllib.parse import unquote - -import h11 -from h11._connection import DEFAULT_MAX_INCOMPLETE_EVENT_SIZE - -from uvicorn._types import ( - ASGI3Application, - ASGIReceiveEvent, - ASGISendEvent, - HTTPRequestEvent, - HTTPResponseBodyEvent, - HTTPResponseStartEvent, - HTTPScope, -) -from uvicorn.config import Config -from uvicorn.logging import TRACE_LOG_LEVEL -from uvicorn.protocols.http.flow_control import CLOSE_HEADER, HIGH_WATER_LIMIT, FlowControl, service_unavailable -from uvicorn.protocols.utils import get_client_addr, get_local_addr, get_path_with_query_string, get_remote_addr, is_ssl -from uvicorn.server import ServerState - - -def _get_status_phrase(status_code: int) -> bytes: - try: - return http.HTTPStatus(status_code).phrase.encode() - except ValueError: - return b"" - - -STATUS_PHRASES = {status_code: _get_status_phrase(status_code) for status_code in range(100, 600)} - - -class H11Protocol(asyncio.Protocol): - def __init__( - self, - config: Config, - server_state: ServerState, - app_state: dict[str, Any], - _loop: asyncio.AbstractEventLoop | None = None, - ) -> None: - if not config.loaded: - config.load() - - self.config = config - self.app = config.loaded_app - self.loop = _loop or asyncio.get_event_loop() - self.logger = logging.getLogger("uvicorn.error") - self.access_logger = logging.getLogger("uvicorn.access") - self.access_log = self.access_logger.hasHandlers() - self.conn = h11.Connection( - h11.SERVER, - config.h11_max_incomplete_event_size - if config.h11_max_incomplete_event_size is not None - else DEFAULT_MAX_INCOMPLETE_EVENT_SIZE, - ) - self.ws_protocol_class = config.ws_protocol_class - self.root_path = config.root_path - self.limit_concurrency = config.limit_concurrency - self.app_state = app_state - - # Timeouts - self.timeout_keep_alive_task: asyncio.TimerHandle | None = None - self.timeout_keep_alive = config.timeout_keep_alive - - # Shared server state - self.server_state = server_state - self.connections = server_state.connections - self.tasks = server_state.tasks - - # Per-connection state - self.transport: asyncio.Transport = None # type: ignore[assignment] - self.flow: FlowControl = None # type: ignore[assignment] - self.server: tuple[str, int | None] | None = None - self.client: tuple[str, int] | None = None - self.scheme: Literal["http", "https"] | None = None - - # Per-request state - self.scope: HTTPScope = None # type: ignore[assignment] - self.headers: list[tuple[bytes, bytes]] = None # type: ignore[assignment] - self.cycle: RequestResponseCycle = None # type: ignore[assignment] - - # Protocol interface - def connection_made( # type: ignore[override] - self, transport: asyncio.Transport - ) -> None: - self.connections.add(self) - - self.transport = transport - self.flow = FlowControl(transport) - self.server = get_local_addr(transport) - self.client = get_remote_addr(transport) - self.scheme = "https" if is_ssl(transport) else "http" - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sHTTP connection made", prefix) - - def connection_lost(self, exc: Exception | None) -> None: - self.connections.discard(self) - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sHTTP connection lost", prefix) - - if self.cycle and not self.cycle.response_complete: - self.cycle.disconnected = True - if self.conn.our_state != h11.ERROR: - event = h11.ConnectionClosed() - try: - self.conn.send(event) - except h11.LocalProtocolError: - # Premature client disconnect - pass - - if self.cycle is not None: - self.cycle.message_event.set() - if self.flow is not None: - self.flow.resume_writing() - if exc is None: - self.transport.close() - self._unset_keepalive_if_required() - - def eof_received(self) -> None: - pass - - def _unset_keepalive_if_required(self) -> None: - if self.timeout_keep_alive_task is not None: - self.timeout_keep_alive_task.cancel() - self.timeout_keep_alive_task = None - - def _get_upgrade(self) -> bytes | None: - connection = [] - upgrade = None - for name, value in self.headers: - if name == b"connection": - connection = [token.lower().strip() for token in value.split(b",")] - if name == b"upgrade": - upgrade = value.lower() - if b"upgrade" in connection: - return upgrade - return None - - def _should_upgrade_to_ws(self) -> bool: - if self.ws_protocol_class is None: - return False - return True - - def _unsupported_upgrade_warning(self) -> None: - msg = "Unsupported upgrade request." - self.logger.warning(msg) - if not self._should_upgrade_to_ws(): - msg = "No supported WebSocket library detected. Please use \"pip install 'uvicorn[standard]'\", or install 'websockets' or 'wsproto' manually." # noqa: E501 - self.logger.warning(msg) - - def _should_upgrade(self) -> bool: - upgrade = self._get_upgrade() - if upgrade == b"websocket" and self._should_upgrade_to_ws(): - return True - if upgrade is not None: - self._unsupported_upgrade_warning() - return False - - def data_received(self, data: bytes) -> None: - self._unset_keepalive_if_required() - - self.conn.receive_data(data) - self.handle_events() - - def handle_events(self) -> None: - while True: - try: - event = self.conn.next_event() - except h11.RemoteProtocolError: - msg = "Invalid HTTP request received." - self.logger.warning(msg) - self.send_400_response(msg) - return - - if event is h11.NEED_DATA: - break - - elif event is h11.PAUSED: - # This case can occur in HTTP pipelining, so we need to - # stop reading any more data, and ensure that at the end - # of the active request/response cycle we handle any - # events that have been buffered up. - self.flow.pause_reading() - break - - elif isinstance(event, h11.Request): - self.headers = [(key.lower(), value) for key, value in event.headers] - raw_path, _, query_string = event.target.partition(b"?") - path = unquote(raw_path.decode("ascii")) - full_path = self.root_path + path - full_raw_path = self.root_path.encode("ascii") + raw_path - self.scope = { - "type": "http", - "asgi": {"version": self.config.asgi_version, "spec_version": "2.3"}, - "http_version": event.http_version.decode("ascii"), - "server": self.server, - "client": self.client, - "scheme": self.scheme, # type: ignore[typeddict-item] - "method": event.method.decode("ascii"), - "root_path": self.root_path, - "path": full_path, - "raw_path": full_raw_path, - "query_string": query_string, - "headers": self.headers, - "state": self.app_state.copy(), - } - if self._should_upgrade(): - self.handle_websocket_upgrade(event) - return - - # Handle 503 responses when 'limit_concurrency' is exceeded. - if self.limit_concurrency is not None and ( - len(self.connections) >= self.limit_concurrency or len(self.tasks) >= self.limit_concurrency - ): - app = service_unavailable - message = "Exceeded concurrency limit." - self.logger.warning(message) - else: - app = self.app - - # When starting to process a request, disable the keep-alive - # timeout. Normally we disable this when receiving data from - # client and set back when finishing processing its request. - # However, for pipelined requests processing finishes after - # already receiving the next request and thus the timer may - # be set here, which we don't want. - self._unset_keepalive_if_required() - - self.cycle = RequestResponseCycle( - scope=self.scope, - conn=self.conn, - transport=self.transport, - flow=self.flow, - logger=self.logger, - access_logger=self.access_logger, - access_log=self.access_log, - default_headers=self.server_state.default_headers, - message_event=asyncio.Event(), - on_response=self.on_response_complete, - ) - if self.config.reset_contextvars: - # Opt-in workaround for https://github.com/python/cpython/issues/140947: - # asyncio can leak context vars between tasks. Hides context set in the - # lifespan or by external instrumentation. - if sys.version_info >= (3, 11): # pragma: py-lt-311 - task = self.loop.create_task(self.cycle.run_asgi(app), context=contextvars.Context()) - else: # pragma: py-gte-311 - task = contextvars.Context().run(self.loop.create_task, self.cycle.run_asgi(app)) - else: - task = self.loop.create_task(self.cycle.run_asgi(app)) - task.add_done_callback(self.tasks.discard) - self.tasks.add(task) - - elif isinstance(event, h11.Data): - if self.conn.our_state is h11.DONE: - continue - self.cycle.body += event.data - if len(self.cycle.body) > HIGH_WATER_LIMIT: - self.flow.pause_reading() - self.cycle.message_event.set() - - elif isinstance(event, h11.EndOfMessage): - if self.conn.our_state is h11.DONE: - self.transport.resume_reading() - self.conn.start_next_cycle() - continue - self.cycle.more_body = False - self.cycle.message_event.set() - if self.conn.their_state == h11.MUST_CLOSE: - break - - def handle_websocket_upgrade(self, event: h11.Request) -> None: - if self.logger.level <= TRACE_LOG_LEVEL: # pragma: full coverage - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sUpgrading to WebSocket", prefix) - - self.connections.discard(self) - output = [event.method, b" ", event.target, b" HTTP/1.1\r\n"] - for name, value in self.headers: - output += [name, b": ", value, b"\r\n"] - output.append(b"\r\n") - protocol = self.ws_protocol_class( # type: ignore[call-arg, misc] - config=self.config, - server_state=self.server_state, - app_state=self.app_state, - ) - protocol.connection_made(self.transport) - protocol.data_received(b"".join(output)) - self.transport.set_protocol(protocol) - - def send_400_response(self, msg: str) -> None: - reason = STATUS_PHRASES[400] - headers: list[tuple[bytes, bytes]] = [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"connection", b"close"), - ] - event = h11.Response(status_code=400, headers=headers, reason=reason) - output = self.conn.send(event) - self.transport.write(output) - - output = self.conn.send(event=h11.Data(data=msg.encode("ascii"))) - self.transport.write(output) - - output = self.conn.send(event=h11.EndOfMessage()) - self.transport.write(output) - - self.transport.close() - - def on_response_complete(self) -> None: - self.server_state.total_requests += 1 - - if self.transport.is_closing(): - return - - # Set a short Keep-Alive timeout. - self._unset_keepalive_if_required() - - self.timeout_keep_alive_task = self.loop.call_later(self.timeout_keep_alive, self.timeout_keep_alive_handler) - - # Unpause data reads if needed. - self.flow.resume_reading() - - # Unblock any pipelined events. - if self.conn.our_state is h11.DONE and self.conn.their_state is h11.DONE: - self.conn.start_next_cycle() - self.handle_events() - - def shutdown(self) -> None: - """ - Called by the server to commence a graceful shutdown. - """ - if self.cycle is None or self.cycle.response_complete: - event = h11.ConnectionClosed() - self.conn.send(event) - self.transport.close() - else: - self.cycle.keep_alive = False - - def pause_writing(self) -> None: - """ - Called by the transport when the write buffer exceeds the high water mark. - """ - self.flow.pause_writing() # pragma: full coverage - - def resume_writing(self) -> None: - """ - Called by the transport when the write buffer drops below the low water mark. - """ - self.flow.resume_writing() # pragma: full coverage - - def timeout_keep_alive_handler(self) -> None: - """ - Called on a keep-alive connection if no new data is received after a short - delay. - """ - if not self.transport.is_closing(): - event = h11.ConnectionClosed() - self.conn.send(event) - self.transport.close() - - -class RequestResponseCycle: - def __init__( - self, - scope: HTTPScope, - conn: h11.Connection, - transport: asyncio.Transport, - flow: FlowControl, - logger: logging.Logger, - access_logger: logging.Logger, - access_log: bool, - default_headers: list[tuple[bytes, bytes]], - message_event: asyncio.Event, - on_response: Callable[..., None], - ) -> None: - self.scope = scope - self.conn = conn - self.transport = transport - self.flow = flow - self.logger = logger - self.access_logger = access_logger - self.access_log = access_log - self.default_headers = default_headers - self.message_event = message_event - self.on_response = on_response - - # Connection state - self.disconnected = False - self.keep_alive = True - self.waiting_for_100_continue = conn.they_are_waiting_for_100_continue - - # Request state - self.body = bytearray() - self.more_body = True - - # Response state - self.response_started = False - self.response_complete = False - - # ASGI exception wrapper - async def run_asgi(self, app: ASGI3Application) -> None: - try: - result = await app( # type: ignore[func-returns-value] - self.scope, self.receive, self.send - ) - except BaseException as exc: - msg = "Exception in ASGI application\n" - self.logger.error(msg, exc_info=exc) - if not self.response_started: - await self.send_500_response() - else: - self.transport.close() - else: - if result is not None: - msg = "ASGI callable should return None, but returned '%s'." - self.logger.error(msg, result) - self.transport.close() - elif not self.response_started and not self.disconnected: - msg = "ASGI callable returned without starting response." - self.logger.error(msg) - await self.send_500_response() - elif not self.response_complete and not self.disconnected: - msg = "ASGI callable returned without completing response." - self.logger.error(msg) - self.transport.close() - finally: - self.on_response = lambda: None - - async def send_500_response(self) -> None: - response_start_event: HTTPResponseStartEvent = { - "type": "http.response.start", - "status": 500, - "headers": [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"connection", b"close"), - ], - } - await self.send(response_start_event) - response_body_event: HTTPResponseBodyEvent = { - "type": "http.response.body", - "body": b"Internal Server Error", - "more_body": False, - } - await self.send(response_body_event) - - # ASGI interface - async def send(self, message: ASGISendEvent) -> None: - if self.flow.write_paused and not self.disconnected: - await self.flow.drain() # pragma: full coverage - - if self.disconnected: - return # pragma: full coverage - - if not self.response_started: - # Sending response status line and headers - if message["type"] != "http.response.start": - raise RuntimeError(f"Expected ASGI message 'http.response.start', but got '{message['type']}'.") - - self.response_started = True - self.waiting_for_100_continue = False - - status = message["status"] - headers = self.default_headers + list(message.get("headers", [])) - - if CLOSE_HEADER in self.scope["headers"] and CLOSE_HEADER not in headers: - headers = headers + [CLOSE_HEADER] - - if self.access_log: - self.access_logger.info( - '%s - "%s %s HTTP/%s" %d', - get_client_addr(self.scope), - self.scope["method"], - get_path_with_query_string(self.scope), - self.scope["http_version"], - status, - ) - - # Write response status line and headers - reason = STATUS_PHRASES[status] - response = h11.Response(status_code=status, headers=headers, reason=reason) - output = self.conn.send(event=response) - self.transport.write(output) - - elif not self.response_complete: - # Sending response body - if message["type"] != "http.response.body": - raise RuntimeError(f"Expected ASGI message 'http.response.body', but got '{message['type']}'.") - - body = message.get("body", b"") - more_body = message.get("more_body", False) - - # Write response body - data = b"" if self.scope["method"] == "HEAD" else body - output = self.conn.send(event=h11.Data(data=data)) - self.transport.write(output) - - # Handle response completion - if not more_body: - self.response_complete = True - self.message_event.set() - output = self.conn.send(event=h11.EndOfMessage()) - self.transport.write(output) - - else: - # Response already sent - raise RuntimeError(f"Unexpected ASGI message '{message['type']}' sent, after response already completed.") - - if self.response_complete: - if self.conn.our_state is h11.MUST_CLOSE or not self.keep_alive: - self.conn.send(event=h11.ConnectionClosed()) - self.transport.close() - self.on_response() - - async def receive(self) -> ASGIReceiveEvent: - if self.waiting_for_100_continue and not self.transport.is_closing(): - headers: list[tuple[str, str]] = [] - event = h11.InformationalResponse(status_code=100, headers=headers, reason="Continue") - output = self.conn.send(event=event) - self.transport.write(output) - self.waiting_for_100_continue = False - - if not self.disconnected and not self.response_complete: - self.flow.resume_reading() - await self.message_event.wait() - self.message_event.clear() - - if self.disconnected or self.response_complete: - return {"type": "http.disconnect"} - - message: HTTPRequestEvent = {"type": "http.request", "body": bytes(self.body), "more_body": self.more_body} - self.body = bytearray() - return message diff --git a/functions/python/data-processor/uvicorn/protocols/http/httptools_impl.py b/functions/python/data-processor/uvicorn/protocols/http/httptools_impl.py deleted file mode 100644 index de9c6fc66..000000000 --- a/functions/python/data-processor/uvicorn/protocols/http/httptools_impl.py +++ /dev/null @@ -1,576 +0,0 @@ -from __future__ import annotations - -import asyncio -import contextvars -import http -import logging -import re -import sys -import urllib -from asyncio.events import TimerHandle -from collections import deque -from collections.abc import Callable -from typing import Any, Literal - -import httptools - -from uvicorn._types import ( - ASGI3Application, - ASGIReceiveEvent, - ASGISendEvent, - HTTPRequestEvent, - HTTPScope, -) -from uvicorn.config import Config -from uvicorn.logging import TRACE_LOG_LEVEL -from uvicorn.protocols.http.flow_control import CLOSE_HEADER, HIGH_WATER_LIMIT, FlowControl, service_unavailable -from uvicorn.protocols.utils import get_client_addr, get_local_addr, get_path_with_query_string, get_remote_addr, is_ssl -from uvicorn.server import ServerState - -HEADER_RE = re.compile(b'[\x00-\x1f\x7f()<>@,;:\\[\\]={} \t\\\\"]') -HEADER_VALUE_RE = re.compile(b"[\x00-\x08\x0a-\x1f\x7f]") - - -def _get_status_line(status_code: int) -> bytes: - try: - phrase = http.HTTPStatus(status_code).phrase.encode() - except ValueError: - phrase = b"" - return b"".join([b"HTTP/1.1 ", str(status_code).encode(), b" ", phrase, b"\r\n"]) - - -STATUS_LINE = {status_code: _get_status_line(status_code) for status_code in range(100, 600)} - - -class HttpToolsProtocol(asyncio.Protocol): - def __init__( - self, - config: Config, - server_state: ServerState, - app_state: dict[str, Any], - _loop: asyncio.AbstractEventLoop | None = None, - ) -> None: - if not config.loaded: - config.load() - - self.config = config - self.app = config.loaded_app - self.loop = _loop or asyncio.get_event_loop() - self.logger = logging.getLogger("uvicorn.error") - self.access_logger = logging.getLogger("uvicorn.access") - self.access_log = self.access_logger.hasHandlers() - self.parser = httptools.HttpRequestParser(self) - - try: - # Enable dangerous leniencies to allow server to a response on the first request from a pipelined request. - self.parser.set_dangerous_leniencies(lenient_data_after_close=True) - except AttributeError: # pragma: no cover - # httptools < 0.6.3 - pass - - self.ws_protocol_class = config.ws_protocol_class - self.root_path = config.root_path - self.limit_concurrency = config.limit_concurrency - self.app_state = app_state - - # Timeouts - self.timeout_keep_alive_task: TimerHandle | None = None - self.timeout_keep_alive = config.timeout_keep_alive - - # Global state - self.server_state = server_state - self.connections = server_state.connections - self.tasks = server_state.tasks - - # Per-connection state - self.transport: asyncio.Transport = None # type: ignore[assignment] - self.flow: FlowControl = None # type: ignore[assignment] - self.server: tuple[str, int | None] | None = None - self.client: tuple[str, int] | None = None - self.scheme: Literal["http", "https"] | None = None - self.pipeline: deque[tuple[RequestResponseCycle, ASGI3Application]] = deque() - - # Per-request state - self.scope: HTTPScope = None # type: ignore[assignment] - self.headers: list[tuple[bytes, bytes]] = None # type: ignore[assignment] - self.expect_100_continue = False - self.cycle: RequestResponseCycle = None # type: ignore[assignment] - - # Protocol interface - def connection_made( # type: ignore[override] - self, transport: asyncio.Transport - ) -> None: - self.connections.add(self) - - self.transport = transport - self.flow = FlowControl(transport) - self.server = get_local_addr(transport) - self.client = get_remote_addr(transport) - self.scheme = "https" if is_ssl(transport) else "http" - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sHTTP connection made", prefix) - - def connection_lost(self, exc: Exception | None) -> None: - self.connections.discard(self) - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sHTTP connection lost", prefix) - - if self.cycle and not self.cycle.response_complete: - self.cycle.disconnected = True - if self.cycle is not None: - self.cycle.message_event.set() - if self.flow is not None: - self.flow.resume_writing() - if exc is None: - self.transport.close() - self._unset_keepalive_if_required() - - self.parser = None # type: ignore[assignment] - - def eof_received(self) -> None: - pass - - def _unset_keepalive_if_required(self) -> None: - if self.timeout_keep_alive_task is not None: - self.timeout_keep_alive_task.cancel() - self.timeout_keep_alive_task = None - - def _get_upgrade(self) -> bytes | None: - connection = [] - upgrade = None - for name, value in self.headers: - if name == b"connection": - connection = [token.lower().strip() for token in value.split(b",")] - if name == b"upgrade": - upgrade = value.lower() - if b"upgrade" in connection: - return upgrade - return None # pragma: full coverage - - def _should_upgrade_to_ws(self) -> bool: - if self.ws_protocol_class is None: - return False - return True - - def _unsupported_upgrade_warning(self) -> None: - self.logger.warning("Unsupported upgrade request.") - if not self._should_upgrade_to_ws(): - msg = "No supported WebSocket library detected. Please use \"pip install 'uvicorn[standard]'\", or install 'websockets' or 'wsproto' manually." # noqa: E501 - self.logger.warning(msg) - - def _should_upgrade(self) -> bool: - upgrade = self._get_upgrade() - return upgrade == b"websocket" and self._should_upgrade_to_ws() - - def data_received(self, data: bytes) -> None: - self._unset_keepalive_if_required() - - try: - self.parser.feed_data(data) - except httptools.HttpParserError: - msg = "Invalid HTTP request received." - self.logger.warning(msg) - self.send_400_response(msg) - return - except httptools.HttpParserUpgrade: - if self._should_upgrade(): - self.handle_websocket_upgrade() - else: - self._unsupported_upgrade_warning() - - def handle_websocket_upgrade(self) -> None: - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sUpgrading to WebSocket", prefix) - - self.connections.discard(self) - method = self.scope["method"].encode() - output = [method, b" ", self.url, b" HTTP/1.1\r\n"] - for name, value in self.scope["headers"]: - output += [name, b": ", value, b"\r\n"] - output.append(b"\r\n") - protocol = self.ws_protocol_class( # type: ignore[call-arg, misc] - config=self.config, - server_state=self.server_state, - app_state=self.app_state, - ) - protocol.connection_made(self.transport) - protocol.data_received(b"".join(output)) - self.transport.set_protocol(protocol) - - def send_400_response(self, msg: str) -> None: - content = [STATUS_LINE[400]] - for name, value in self.server_state.default_headers: - content.extend([name, b": ", value, b"\r\n"]) # pragma: full coverage - content.extend( - [ - b"content-type: text/plain; charset=utf-8\r\n", - b"content-length: " + str(len(msg)).encode("ascii") + b"\r\n", - b"connection: close\r\n", - b"\r\n", - msg.encode("ascii"), - ] - ) - self.transport.write(b"".join(content)) - self.transport.close() - - def on_message_begin(self) -> None: - self.url = b"" - self.expect_100_continue = False - self.headers = [] - self.scope = { # type: ignore[typeddict-item] - "type": "http", - "asgi": {"version": self.config.asgi_version, "spec_version": "2.3"}, - "http_version": "1.1", - "server": self.server, - "client": self.client, - "scheme": self.scheme, # type: ignore[typeddict-item] - "root_path": self.root_path, - "headers": self.headers, - "state": self.app_state.copy(), - } - - # Parser callbacks - def on_url(self, url: bytes) -> None: - self.url += url - - def on_header(self, name: bytes, value: bytes) -> None: - name = name.lower() - if name == b"expect" and value.lower() == b"100-continue": - self.expect_100_continue = True - self.headers.append((name, value)) - - def on_headers_complete(self) -> None: - http_version = self.parser.get_http_version() - method = self.parser.get_method() - self.scope["method"] = method.decode("ascii") - if http_version != "1.1": - self.scope["http_version"] = http_version - if self.parser.should_upgrade() and self._should_upgrade(): - return - parsed_url = httptools.parse_url(self.url) - raw_path = parsed_url.path - path = raw_path.decode("ascii") - if "%" in path: - path = urllib.parse.unquote(path) - full_path = self.root_path + path - full_raw_path = self.root_path.encode("ascii") + raw_path - self.scope["path"] = full_path - self.scope["raw_path"] = full_raw_path - self.scope["query_string"] = parsed_url.query or b"" - - # Handle 503 responses when 'limit_concurrency' is exceeded. - if self.limit_concurrency is not None and ( - len(self.connections) >= self.limit_concurrency or len(self.tasks) >= self.limit_concurrency - ): - app = service_unavailable - message = "Exceeded concurrency limit." - self.logger.warning(message) - else: - app = self.app - - existing_cycle = self.cycle - self.cycle = RequestResponseCycle( - scope=self.scope, - transport=self.transport, - flow=self.flow, - logger=self.logger, - access_logger=self.access_logger, - access_log=self.access_log, - default_headers=self.server_state.default_headers, - message_event=asyncio.Event(), - expect_100_continue=self.expect_100_continue, - keep_alive=http_version != "1.0", - on_response=self.on_response_complete, - ) - if existing_cycle is None or existing_cycle.response_complete: - # Standard case - start processing the request. - self._start_asgi_task(self.cycle, app) - else: - # Pipelined HTTP requests need to be queued up. - self.flow.pause_reading() - self.pipeline.appendleft((self.cycle, app)) - - def _start_asgi_task(self, cycle: RequestResponseCycle, app: ASGI3Application) -> None: - if self.config.reset_contextvars: - # Opt-in workaround for https://github.com/python/cpython/issues/140947: - # asyncio can leak context vars between tasks. Hides context set in the - # lifespan or by external instrumentation. - if sys.version_info >= (3, 11): # pragma: py-lt-311 - task = self.loop.create_task(cycle.run_asgi(app), context=contextvars.Context()) - else: # pragma: py-gte-311 - task = contextvars.Context().run(self.loop.create_task, cycle.run_asgi(app)) - else: - task = self.loop.create_task(cycle.run_asgi(app)) - task.add_done_callback(self.tasks.discard) - self.tasks.add(task) - - def on_body(self, body: bytes) -> None: - if (self.parser.should_upgrade() and self._should_upgrade()) or self.cycle.response_complete: - return - self.cycle.body += body - if len(self.cycle.body) > HIGH_WATER_LIMIT: - self.flow.pause_reading() - self.cycle.message_event.set() - - def on_message_complete(self) -> None: - if (self.parser.should_upgrade() and self._should_upgrade()) or self.cycle.response_complete: - return - self.cycle.more_body = False - self.cycle.message_event.set() - - def on_response_complete(self) -> None: - # Callback for pipelined HTTP requests to be started. - self.server_state.total_requests += 1 - - if self.transport.is_closing(): - return - - self._unset_keepalive_if_required() - - # Unpause data reads if needed. - self.flow.resume_reading() - - # Unblock any pipelined events. If there are none, arm the - # Keep-Alive timeout instead. - if self.pipeline: - cycle, app = self.pipeline.pop() - self._start_asgi_task(cycle, app) - else: - self.timeout_keep_alive_task = self.loop.call_later( - self.timeout_keep_alive, self.timeout_keep_alive_handler - ) - - def shutdown(self) -> None: - """ - Called by the server to commence a graceful shutdown. - """ - if self.cycle is None or self.cycle.response_complete: - self.transport.close() - else: - self.cycle.keep_alive = False - - def pause_writing(self) -> None: - """ - Called by the transport when the write buffer exceeds the high water mark. - """ - self.flow.pause_writing() # pragma: full coverage - - def resume_writing(self) -> None: - """ - Called by the transport when the write buffer drops below the low water mark. - """ - self.flow.resume_writing() # pragma: full coverage - - def timeout_keep_alive_handler(self) -> None: - """ - Called on a keep-alive connection if no new data is received after a short - delay. - """ - if not self.transport.is_closing(): - self.transport.close() - - -class RequestResponseCycle: - def __init__( - self, - scope: HTTPScope, - transport: asyncio.Transport, - flow: FlowControl, - logger: logging.Logger, - access_logger: logging.Logger, - access_log: bool, - default_headers: list[tuple[bytes, bytes]], - message_event: asyncio.Event, - expect_100_continue: bool, - keep_alive: bool, - on_response: Callable[..., None], - ): - self.scope = scope - self.transport = transport - self.flow = flow - self.logger = logger - self.access_logger = access_logger - self.access_log = access_log - self.default_headers = default_headers - self.message_event = message_event - self.on_response = on_response - - # Connection state - self.disconnected = False - self.keep_alive = keep_alive - self.waiting_for_100_continue = expect_100_continue - - # Request state - self.body = bytearray() - self.more_body = True - - # Response state - self.response_started = False - self.response_complete = False - self.chunked_encoding: bool | None = None - self.expected_content_length = 0 - - # ASGI exception wrapper - async def run_asgi(self, app: ASGI3Application) -> None: - try: - result = await app( # type: ignore[func-returns-value] - self.scope, self.receive, self.send - ) - except BaseException as exc: - msg = "Exception in ASGI application\n" - self.logger.error(msg, exc_info=exc) - if not self.response_started: - await self.send_500_response() - else: - self.transport.close() - else: - if result is not None: - msg = "ASGI callable should return None, but returned '%s'." - self.logger.error(msg, result) - self.transport.close() - elif not self.response_started and not self.disconnected: - msg = "ASGI callable returned without starting response." - self.logger.error(msg) - await self.send_500_response() - elif not self.response_complete and not self.disconnected: - msg = "ASGI callable returned without completing response." - self.logger.error(msg) - self.transport.close() - finally: - self.on_response = lambda: None - - async def send_500_response(self) -> None: - await self.send( - { - "type": "http.response.start", - "status": 500, - "headers": [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"content-length", b"21"), - (b"connection", b"close"), - ], - } - ) - await self.send({"type": "http.response.body", "body": b"Internal Server Error", "more_body": False}) - - # ASGI interface - async def send(self, message: ASGISendEvent) -> None: - if self.flow.write_paused and not self.disconnected: - await self.flow.drain() # pragma: full coverage - - if self.disconnected: - return # pragma: full coverage - - if not self.response_started: - # Sending response status line and headers - if message["type"] != "http.response.start": - raise RuntimeError(f"Expected ASGI message 'http.response.start', but got '{message['type']}'.") - - self.response_started = True - self.waiting_for_100_continue = False - - status_code = message["status"] - headers = self.default_headers + list(message.get("headers", [])) - - if CLOSE_HEADER in self.scope["headers"] and CLOSE_HEADER not in headers: - headers = headers + [CLOSE_HEADER] - - if self.access_log: - self.access_logger.info( - '%s - "%s %s HTTP/%s" %d', - get_client_addr(self.scope), - self.scope["method"], - get_path_with_query_string(self.scope), - self.scope["http_version"], - status_code, - ) - - # Write response status line and headers - content = [STATUS_LINE[status_code]] - - for name, value in headers: - if HEADER_RE.search(name): - raise RuntimeError("Invalid HTTP header name.") # pragma: full coverage - if HEADER_VALUE_RE.search(value): - raise RuntimeError("Invalid HTTP header value.") - - name = name.lower() - if name == b"content-length" and self.chunked_encoding is None: - self.expected_content_length = int(value.decode()) - self.chunked_encoding = False - elif name == b"transfer-encoding" and value.lower() == b"chunked": - self.expected_content_length = 0 - self.chunked_encoding = True - elif name == b"connection" and value.lower() == b"close": - self.keep_alive = False - content.extend([name, b": ", value, b"\r\n"]) - - if self.chunked_encoding is None and self.scope["method"] != "HEAD" and status_code not in (204, 304): - # Neither content-length nor transfer-encoding specified - self.chunked_encoding = True - content.append(b"transfer-encoding: chunked\r\n") - - content.append(b"\r\n") - self.transport.write(b"".join(content)) - - elif not self.response_complete: - # Sending response body - if message["type"] != "http.response.body": - raise RuntimeError(f"Expected ASGI message 'http.response.body', but got '{message['type']}'.") - - body = message.get("body", b"") - more_body = message.get("more_body", False) - - # Write response body - if self.scope["method"] == "HEAD": - self.expected_content_length = 0 - elif self.chunked_encoding: - if body: - content = [b"%x\r\n" % len(body), body, b"\r\n"] - else: - content = [] - if not more_body: - content.append(b"0\r\n\r\n") - self.transport.write(b"".join(content)) - else: - num_bytes = len(body) - if num_bytes > self.expected_content_length: - raise RuntimeError("Response content longer than Content-Length") - else: - self.expected_content_length -= num_bytes - self.transport.write(body) - - # Handle response completion - if not more_body: - if self.expected_content_length != 0: - raise RuntimeError("Response content shorter than Content-Length") - self.response_complete = True - self.message_event.set() - if not self.keep_alive: - self.transport.close() - self.on_response() - - else: - # Response already sent - raise RuntimeError(f"Unexpected ASGI message '{message['type']}' sent, after response already completed.") - - async def receive(self) -> ASGIReceiveEvent: - if self.waiting_for_100_continue and not self.transport.is_closing(): - self.transport.write(b"HTTP/1.1 100 Continue\r\n\r\n") - self.waiting_for_100_continue = False - - if not self.disconnected and not self.response_complete: - self.flow.resume_reading() - await self.message_event.wait() - self.message_event.clear() - - if self.disconnected or self.response_complete: - return {"type": "http.disconnect"} - message: HTTPRequestEvent = {"type": "http.request", "body": bytes(self.body), "more_body": self.more_body} - self.body = bytearray() - return message diff --git a/functions/python/data-processor/uvicorn/protocols/utils.py b/functions/python/data-processor/uvicorn/protocols/utils.py deleted file mode 100644 index a9a89065c..000000000 --- a/functions/python/data-processor/uvicorn/protocols/utils.py +++ /dev/null @@ -1,62 +0,0 @@ -from __future__ import annotations - -import asyncio -import socket -import urllib.parse - -from uvicorn._types import WWWScope - - -class ClientDisconnected(OSError): ... - - -def get_remote_addr(transport: asyncio.Transport) -> tuple[str, int] | None: - socket_info: socket.socket | None = transport.get_extra_info("socket") - if socket_info is not None: - try: - info = socket_info.getpeername() - return (str(info[0]), int(info[1])) if isinstance(info, tuple) else None - except OSError: # pragma: no cover - # This case appears to inconsistently occur with uvloop - # bound to a unix domain socket. - return None - - info = transport.get_extra_info("peername") - if info is not None and isinstance(info, list | tuple) and len(info) == 2: - return (str(info[0]), int(info[1])) - return None - - -def get_local_addr(transport: asyncio.Transport) -> tuple[str, int | None] | None: - socket_info: socket.socket | None = transport.get_extra_info("socket") - if socket_info is not None: - info = socket_info.getsockname() - if isinstance(info, tuple): - return (str(info[0]), int(info[1])) - if isinstance(info, str): - return (info, None) - return None - info = transport.get_extra_info("sockname") - if info is not None and isinstance(info, list | tuple) and len(info) == 2: - return (str(info[0]), int(info[1])) - if isinstance(info, str): - return (info, None) - return None - - -def is_ssl(transport: asyncio.Transport) -> bool: - return bool(transport.get_extra_info("sslcontext")) - - -def get_client_addr(scope: WWWScope) -> str: - client = scope.get("client") - if not client: - return "" - return "%s:%d" % client - - -def get_path_with_query_string(scope: WWWScope) -> str: - path_with_query_string = urllib.parse.quote(scope["path"]) - if scope["query_string"]: - path_with_query_string = "{}?{}".format(path_with_query_string, scope["query_string"].decode("ascii")) - return path_with_query_string diff --git a/functions/python/data-processor/uvicorn/protocols/websockets/__init__.py b/functions/python/data-processor/uvicorn/protocols/websockets/__init__.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/python/data-processor/uvicorn/protocols/websockets/auto.py b/functions/python/data-processor/uvicorn/protocols/websockets/auto.py deleted file mode 100644 index 7180ad397..000000000 --- a/functions/python/data-processor/uvicorn/protocols/websockets/auto.py +++ /dev/null @@ -1,21 +0,0 @@ -from __future__ import annotations - -import asyncio -from collections.abc import Callable - -AutoWebSocketsProtocol: Callable[..., asyncio.Protocol] | None -try: - import websockets # noqa -except ImportError: # pragma: no cover - try: - import wsproto # noqa - except ImportError: - AutoWebSocketsProtocol = None - else: - from uvicorn.protocols.websockets.wsproto_impl import WSProtocol - - AutoWebSocketsProtocol = WSProtocol -else: - from uvicorn.protocols.websockets.websockets_impl import WebSocketProtocol - - AutoWebSocketsProtocol = WebSocketProtocol diff --git a/functions/python/data-processor/uvicorn/protocols/websockets/websockets_impl.py b/functions/python/data-processor/uvicorn/protocols/websockets/websockets_impl.py deleted file mode 100644 index e4139391a..000000000 --- a/functions/python/data-processor/uvicorn/protocols/websockets/websockets_impl.py +++ /dev/null @@ -1,372 +0,0 @@ -from __future__ import annotations - -import asyncio -import http -import logging -from collections.abc import Sequence -from typing import Any, Literal, cast -from urllib.parse import unquote - -import websockets -import websockets.legacy.handshake -from websockets.datastructures import Headers -from websockets.exceptions import ConnectionClosed -from websockets.extensions.base import ServerExtensionFactory -from websockets.extensions.permessage_deflate import ServerPerMessageDeflateFactory -from websockets.legacy.server import HTTPResponse -from websockets.server import WebSocketServerProtocol -from websockets.typing import Subprotocol - -from uvicorn._types import ( - ASGI3Application, - ASGISendEvent, - WebSocketConnectEvent, - WebSocketDisconnectEvent, - WebSocketReceiveEvent, - WebSocketScope, -) -from uvicorn.config import Config -from uvicorn.logging import TRACE_LOG_LEVEL -from uvicorn.protocols.utils import ( - ClientDisconnected, - get_client_addr, - get_local_addr, - get_path_with_query_string, - get_remote_addr, - is_ssl, -) -from uvicorn.server import ServerState - - -class Server: - closing = False - - def register(self, ws: WebSocketServerProtocol) -> None: - pass - - def unregister(self, ws: WebSocketServerProtocol) -> None: - pass - - def is_serving(self) -> bool: - return not self.closing - - -class WebSocketProtocol(WebSocketServerProtocol): - extra_headers: list[tuple[str, str]] - logger: logging.Logger | logging.LoggerAdapter[Any] - - def __init__( - self, - config: Config, - server_state: ServerState, - app_state: dict[str, Any], - _loop: asyncio.AbstractEventLoop | None = None, - ): - if not config.loaded: - config.load() - - self.config = config - self.app = cast(ASGI3Application, config.loaded_app) - self.loop = _loop or asyncio.get_event_loop() - self.root_path = config.root_path - self.app_state = app_state - - # Shared server state - self.connections = server_state.connections - self.tasks = server_state.tasks - - # Connection state - self.transport: asyncio.Transport = None # type: ignore[assignment] - self.server: tuple[str, int | None] | None = None - self.client: tuple[str, int] | None = None - self.scheme: Literal["wss", "ws"] = None # type: ignore[assignment] - - # Connection events - self.scope: WebSocketScope - self.handshake_started_event = asyncio.Event() - self.handshake_completed_event = asyncio.Event() - self.closed_event = asyncio.Event() - self.initial_response: HTTPResponse | None = None - self.connect_sent = False - self.lost_connection_before_handshake = False - self.accepted_subprotocol: Subprotocol | None = None - - self.ws_server: Server = Server() # type: ignore[assignment] - - extensions: list[ServerExtensionFactory] = [] - if self.config.ws_per_message_deflate: - extensions.append(ServerPerMessageDeflateFactory()) - - super().__init__( - ws_handler=self.ws_handler, - ws_server=self.ws_server, # type: ignore[arg-type] - max_size=self.config.ws_max_size, - max_queue=self.config.ws_max_queue, - ping_interval=self.config.ws_ping_interval, - ping_timeout=self.config.ws_ping_timeout, - extensions=extensions, - logger=logging.getLogger("uvicorn.error"), - ) - self.server_header = None - self.extra_headers = [ - (name.decode("latin-1"), value.decode("latin-1")) for name, value in server_state.default_headers - ] - - def connection_made( # type: ignore[override] - self, transport: asyncio.Transport - ) -> None: - self.connections.add(self) - self.transport = transport - self.server = get_local_addr(transport) - self.client = get_remote_addr(transport) - self.scheme = "wss" if is_ssl(transport) else "ws" - - if self.logger.isEnabledFor(TRACE_LOG_LEVEL): - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket connection made", prefix) - - super().connection_made(transport) - - def connection_lost(self, exc: Exception | None) -> None: - self.connections.remove(self) - - if self.logger.isEnabledFor(TRACE_LOG_LEVEL): - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket connection lost", prefix) - - self.lost_connection_before_handshake = not self.handshake_completed_event.is_set() - self.handshake_completed_event.set() - super().connection_lost(exc) - if exc is None: - self.transport.close() - - def shutdown(self) -> None: - self.ws_server.closing = True - if self.handshake_completed_event.is_set(): - self.fail_connection(1012) - else: - self.send_500_response() - self.transport.close() - - def on_task_complete(self, task: asyncio.Task[None]) -> None: - self.tasks.discard(task) - - async def process_request(self, path: str, request_headers: Headers) -> HTTPResponse | None: - """ - This hook is called to determine if the websocket should return - an HTTP response and close. - - Our behavior here is to start the ASGI application, and then wait - for either `accept` or `close` in order to determine if we should - close the connection. - """ - path_portion, _, query_string = path.partition("?") - - websockets.legacy.handshake.check_request(request_headers) - - subprotocols: list[str] = [] - for header in request_headers.get_all("Sec-WebSocket-Protocol"): - subprotocols.extend([token.strip() for token in header.split(",")]) - - asgi_headers = [ - (name.encode("ascii"), value.encode("ascii", errors="surrogateescape")) - for name, value in request_headers.raw_items() - ] - path = unquote(path_portion) - full_path = self.root_path + path - full_raw_path = self.root_path.encode("ascii") + path_portion.encode("ascii") - - self.scope = { - "type": "websocket", - "asgi": {"version": self.config.asgi_version, "spec_version": "2.4"}, - "http_version": "1.1", - "scheme": self.scheme, - "server": self.server, - "client": self.client, - "root_path": self.root_path, - "path": full_path, - "raw_path": full_raw_path, - "query_string": query_string.encode("ascii"), - "headers": asgi_headers, - "subprotocols": subprotocols, - "state": self.app_state.copy(), - "extensions": {"websocket.http.response": {}}, - } - task = self.loop.create_task(self.run_asgi()) - task.add_done_callback(self.on_task_complete) - self.tasks.add(task) - await self.handshake_started_event.wait() - return self.initial_response - - def process_subprotocol( - self, headers: Headers, available_subprotocols: Sequence[Subprotocol] | None - ) -> Subprotocol | None: - """ - We override the standard 'process_subprotocol' behavior here so that - we return whatever subprotocol is sent in the 'accept' message. - """ - return self.accepted_subprotocol - - def send_500_response(self) -> None: - msg = b"Internal Server Error" - content = [ - b"HTTP/1.1 500 Internal Server Error\r\ncontent-type: text/plain; charset=utf-8\r\n", - b"content-length: " + str(len(msg)).encode("ascii") + b"\r\n", - b"connection: close\r\n", - b"\r\n", - msg, - ] - self.transport.write(b"".join(content)) - # Allow handler task to terminate cleanly, as websockets doesn't cancel it by - # itself (see https://github.com/Kludex/uvicorn/issues/920) - self.handshake_started_event.set() - - async def ws_handler(self, protocol: WebSocketServerProtocol, path: str) -> Any: # type: ignore[override] - """ - This is the main handler function for the 'websockets' implementation - to call into. We just wait for close then return, and instead allow - 'send' and 'receive' events to drive the flow. - """ - self.handshake_completed_event.set() - await self.wait_closed() - - async def run_asgi(self) -> None: - """ - Wrapper around the ASGI callable, handling exceptions and unexpected - termination states. - """ - try: - result = await self.app(self.scope, self.asgi_receive, self.asgi_send) # type: ignore[func-returns-value] - except ClientDisconnected: # pragma: full coverage - self.closed_event.set() - except BaseException: - self.closed_event.set() - self.logger.exception("Exception in ASGI application\n") - if not self.handshake_started_event.is_set(): - self.send_500_response() - else: - await self.handshake_completed_event.wait() - else: - self.closed_event.set() - if not self.handshake_started_event.is_set(): - self.logger.error("ASGI callable returned without sending handshake.") - self.send_500_response() - elif result is not None: - self.logger.error("ASGI callable should return None, but returned '%s'.", result) - await self.handshake_completed_event.wait() - self.transport.close() - - async def asgi_send(self, message: ASGISendEvent) -> None: - if not self.handshake_started_event.is_set(): - if message["type"] == "websocket.accept": - self.logger.info( - '%s - "WebSocket %s" [accepted]', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - ) - self.initial_response = None - self.accepted_subprotocol = cast(Subprotocol | None, message.get("subprotocol")) - if "headers" in message: - self.extra_headers.extend( - # ASGI spec requires bytes - # But for compatibility we need to convert it to strings - (name.decode("latin-1"), value.decode("latin-1")) - for name, value in message["headers"] - ) - self.handshake_started_event.set() - - elif message["type"] == "websocket.close": - self.logger.info( - '%s - "WebSocket %s" 403', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - ) - self.initial_response = (http.HTTPStatus.FORBIDDEN, [], b"") - self.handshake_started_event.set() - self.closed_event.set() - - elif message["type"] == "websocket.http.response.start": - self.logger.info( - '%s - "WebSocket %s" %d', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - message["status"], - ) - # websockets requires the status to be an enum. look it up. - status = http.HTTPStatus(message["status"]) - headers = [ - (name.decode("latin-1"), value.decode("latin-1")) for name, value in message.get("headers", []) - ] - self.initial_response = (status, headers, b"") - self.handshake_started_event.set() - - else: - raise RuntimeError( - "Expected ASGI message 'websocket.accept', 'websocket.close', " - f"or 'websocket.http.response.start' but got '{message['type']}'." - ) - - elif not self.closed_event.is_set() and self.initial_response is None: - await self.handshake_completed_event.wait() - - try: - if message["type"] == "websocket.send": - bytes_data = message.get("bytes") - text_data = message.get("text") - data = text_data if bytes_data is None else bytes_data - await self.send(data) # type: ignore[arg-type] - - elif message["type"] == "websocket.close": - code = message.get("code", 1000) - reason = message.get("reason", "") or "" - await self.close(code, reason) - self.closed_event.set() - - else: - raise RuntimeError( - f"Expected ASGI message 'websocket.send' or 'websocket.close', but got '{message['type']}'." - ) - except ConnectionClosed as exc: - raise ClientDisconnected from exc - - elif self.initial_response is not None: - if message["type"] == "websocket.http.response.body": - body = self.initial_response[2] + message["body"] - self.initial_response = self.initial_response[:2] + (body,) - if not message.get("more_body", False): - self.closed_event.set() - else: - raise RuntimeError(f"Expected ASGI message 'websocket.http.response.body' but got '{message['type']}'.") - - else: - raise RuntimeError( - f"Unexpected ASGI message '{message['type']}', after sending 'websocket.close' " - "or response already completed." - ) - - async def asgi_receive(self) -> WebSocketDisconnectEvent | WebSocketConnectEvent | WebSocketReceiveEvent: - if not self.connect_sent: - self.connect_sent = True - return {"type": "websocket.connect"} - - await self.handshake_completed_event.wait() - - if self.lost_connection_before_handshake: - # If the handshake failed or the app closed before handshake completion, - # use 1006 Abnormal Closure. - return {"type": "websocket.disconnect", "code": 1006} - - if self.closed_event.is_set(): - return {"type": "websocket.disconnect", "code": 1005} - - try: - data = await self.recv() - except ConnectionClosed: - self.closed_event.set() - if self.ws_server.closing: - return {"type": "websocket.disconnect", "code": 1012} - return {"type": "websocket.disconnect", "code": self.close_code or 1005, "reason": self.close_reason} - - if isinstance(data, str): - return {"type": "websocket.receive", "text": data} - return {"type": "websocket.receive", "bytes": data} diff --git a/functions/python/data-processor/uvicorn/protocols/websockets/websockets_sansio_impl.py b/functions/python/data-processor/uvicorn/protocols/websockets/websockets_sansio_impl.py deleted file mode 100644 index 036b6ee01..000000000 --- a/functions/python/data-processor/uvicorn/protocols/websockets/websockets_sansio_impl.py +++ /dev/null @@ -1,477 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import random -import struct -import sys -from asyncio import TimerHandle -from asyncio.transports import BaseTransport, Transport -from http import HTTPStatus -from typing import Any, Literal, cast -from urllib.parse import unquote - -from websockets.exceptions import InvalidState -from websockets.extensions.permessage_deflate import ServerPerMessageDeflateFactory -from websockets.frames import Frame, Opcode -from websockets.http11 import Request -from websockets.server import ServerProtocol - -from uvicorn._types import ( - ASGIReceiveEvent, - ASGISendEvent, - WebSocketScope, -) -from uvicorn.config import Config -from uvicorn.logging import TRACE_LOG_LEVEL -from uvicorn.protocols.utils import ( - ClientDisconnected, - get_client_addr, - get_local_addr, - get_path_with_query_string, - get_remote_addr, - is_ssl, -) -from uvicorn.server import ServerState - -if sys.version_info >= (3, 11): # pragma: no cover - from typing import assert_never -else: # pragma: no cover - from typing_extensions import assert_never - - -class WebSocketsSansIOProtocol(asyncio.Protocol): - def __init__( - self, - config: Config, - server_state: ServerState, - app_state: dict[str, Any], - _loop: asyncio.AbstractEventLoop | None = None, - ) -> None: - if not config.loaded: - config.load() # pragma: no cover - - self.config = config - self.app = config.loaded_app - self.loop = _loop or asyncio.get_event_loop() - self.logger = logging.getLogger("uvicorn.error") - self.root_path = config.root_path - self.app_state = app_state - - # Shared server state - self.connections = server_state.connections - self.tasks = server_state.tasks - self.default_headers = server_state.default_headers - - # Connection state - self.transport: asyncio.Transport = None # type: ignore[assignment] - self.server: tuple[str, int | None] | None = None - self.client: tuple[str, int] | None = None - self.scheme: Literal["wss", "ws"] = None # type: ignore[assignment] - - # WebSocket state - self.queue: asyncio.Queue[ASGIReceiveEvent] = asyncio.Queue() - self.handshake_initiated = False - self.handshake_complete = False - self.close_sent = False - self.initial_response: tuple[int, list[tuple[str, str]], bytes] | None = None - - extensions = [] - if self.config.ws_per_message_deflate: - extensions = [ - ServerPerMessageDeflateFactory( - server_max_window_bits=12, - client_max_window_bits=12, - compress_settings={"memLevel": 5}, - ) - ] - self.conn = ServerProtocol( - extensions=extensions, - max_size=self.config.ws_max_size, - logger=logging.getLogger("uvicorn.error"), - ) - - self.read_paused = False - self.writable = asyncio.Event() - self.writable.set() - - # Keepalive state - self.ping_interval = config.ws_ping_interval - self.ping_timeout = config.ws_ping_timeout - self.ping_timer: TimerHandle | None = None - self.pong_timer: TimerHandle | None = None - self.pending_ping_payload: bytes | None = None - self.ping_sent_at: float = 0.0 - self.last_ping_rtt: float = 0.0 - - # Buffers - self.bytes = bytearray() - - def connection_made(self, transport: BaseTransport) -> None: - """Called when a connection is made.""" - transport = cast(Transport, transport) - self.connections.add(self) - self.transport = transport - self.server = get_local_addr(transport) - self.client = get_remote_addr(transport) - self.scheme = "wss" if is_ssl(transport) else "ws" - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket connection made", prefix) - - def connection_lost(self, exc: Exception | None) -> None: - self.stop_keepalive() - code = 1005 if self.handshake_complete else 1006 - self.queue.put_nowait({"type": "websocket.disconnect", "code": code}) - self.connections.remove(self) - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket connection lost", prefix) - - self.handshake_complete = True - if exc is None: - self.transport.close() - - def eof_received(self) -> None: - pass - - def shutdown(self) -> None: - self.stop_keepalive() - if self.handshake_complete: - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1012}) - self.conn.send_close(1012) - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - else: - self.send_500_response() - self.transport.close() - - def data_received(self, data: bytes) -> None: - self.conn.receive_data(data) - if self.conn.parser_exc is not None: # pragma: no cover - self.handle_parser_exception() - return - self.handle_events() - - def handle_events(self) -> None: - for event in self.conn.events_received(): - if isinstance(event, Request): - self.handle_connect(event) - if isinstance(event, Frame): - if event.opcode == Opcode.CONT: - self.handle_cont(event) # pragma: no cover - elif event.opcode == Opcode.TEXT: - self.handle_text(event) - elif event.opcode == Opcode.BINARY: - self.handle_bytes(event) - elif event.opcode == Opcode.PING: - self.handle_ping() - elif event.opcode == Opcode.PONG: - self.handle_pong(event) - elif event.opcode == Opcode.CLOSE: - self.handle_close(event) - else: - assert_never(event.opcode) # pragma: no cover - - # Event handlers - - def handle_connect(self, event: Request) -> None: - self.request = event - self.response = self.conn.accept(event) - self.handshake_initiated = True - if self.response.status_code != 101: - self.handshake_complete = True - self.close_sent = True - self.conn.send_response(self.response) - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - self.transport.close() - return - - headers = [ - (key.encode("ascii"), value.encode("ascii", errors="surrogateescape")) - for key, value in event.headers.raw_items() - ] - raw_path, _, query_string = event.path.partition("?") - self.scope: WebSocketScope = { - "type": "websocket", - "asgi": {"version": self.config.asgi_version, "spec_version": "2.4"}, - "http_version": "1.1", - "scheme": self.scheme, - "server": self.server, - "client": self.client, - "root_path": self.root_path, - "path": self.root_path + unquote(raw_path), - "raw_path": self.root_path.encode("ascii") + raw_path.encode("ascii"), - "query_string": query_string.encode("ascii"), - "headers": headers, - "subprotocols": event.headers.get_all("Sec-WebSocket-Protocol"), - "state": self.app_state.copy(), - "extensions": {"websocket.http.response": {}}, - } - self.queue.put_nowait({"type": "websocket.connect"}) - task = self.loop.create_task(self.run_asgi()) - task.add_done_callback(self.on_task_complete) - self.tasks.add(task) - - def handle_cont(self, event: Frame) -> None: - self.bytes.extend(event.data) - if event.fin: - self.send_receive_event_to_app() - - def handle_text(self, event: Frame) -> None: - self.bytes = bytearray(event.data) - self.curr_msg_data_type: Literal["text", "bytes"] = "text" - if event.fin: - self.send_receive_event_to_app() - - def handle_bytes(self, event: Frame) -> None: - self.bytes = bytearray(event.data) - self.curr_msg_data_type = "bytes" - if event.fin: - self.send_receive_event_to_app() - - def send_receive_event_to_app(self) -> None: - if self.curr_msg_data_type == "text": - try: - self.queue.put_nowait({"type": "websocket.receive", "text": self.bytes.decode()}) - except UnicodeDecodeError: # pragma: no cover - self.logger.exception("Invalid UTF-8 sequence received from client.") - self.conn.send_close(1007) - self.handle_parser_exception() - return - else: - self.queue.put_nowait({"type": "websocket.receive", "bytes": bytes(self.bytes)}) - if not self.read_paused: - self.read_paused = True - self.transport.pause_reading() - - def handle_ping(self) -> None: - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - - def handle_pong(self, event: Frame) -> None: - # Ignore unsolicited pongs and stale pongs whose payload doesn't match the ping currently in flight - if self.pending_ping_payload is None or bytes(event.data) != self.pending_ping_payload: - return # pragma: no cover - - self.last_ping_rtt = self.loop.time() - self.ping_sent_at - self.pending_ping_payload = None - # The peer answered in time; cancel the pong deadline and chain the next ping. This `schedule_ping()` call is - # what keeps the keepalive loop running when ping_timeout is set. When ping_timeout is None the next ping is - # already scheduled by `send_keepalive_ping`, so we must not schedule a duplicate here. - if self.pong_timer is not None: - self.pong_timer.cancel() - self.pong_timer = None - self.schedule_ping() - - def start_keepalive(self) -> None: - if self.ping_interval is not None and self.ping_interval > 0: - self.schedule_ping() - - def stop_keepalive(self) -> None: - if self.ping_timer is not None: - self.ping_timer.cancel() - self.ping_timer = None - if self.pong_timer is not None: # pragma: no cover - self.pong_timer.cancel() - self.pong_timer = None - self.pending_ping_payload = None - - def schedule_ping(self) -> None: - assert self.ping_interval is not None - delay = max(0.0, self.ping_interval - self.last_ping_rtt) - self.ping_timer = self.loop.call_later(delay, self.send_keepalive_ping) - - def send_keepalive_ping(self) -> None: - self.ping_timer = None - if self.close_sent or self.transport.is_closing(): # pragma: no cover - return - # Random 4-byte payload identifies this ping; `handle_pong` uses it to ignore stale or unsolicited pongs. - # See https://github.com/python-websockets/websockets/blob/4d229bf9f583d593aa103287aee0a77c9fbc3a79/src/websockets/asyncio/connection.py#L624 - self.pending_ping_payload = struct.pack("!I", random.getrandbits(32)) - self.ping_sent_at = self.loop.time() - self.conn.send_ping(self.pending_ping_payload) - self.transport.write(b"".join(self.conn.data_to_send())) - if self.ping_timeout is not None: - self.pong_timer = self.loop.call_later(self.ping_timeout, self.keepalive_timeout) - else: # pragma: no cover - self.schedule_ping() - - def keepalive_timeout(self) -> None: - self.pong_timer = None - self.pending_ping_payload = None - if self.close_sent or self.transport.is_closing(): # pragma: no cover - return - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket keepalive ping timeout", prefix) - self.conn.fail(1011, "keepalive ping timeout") - self.transport.write(b"".join(self.conn.data_to_send())) - self.close_sent = True - self.transport.close() - - def handle_close(self, event: Frame) -> None: - if not self.close_sent and not self.transport.is_closing(): - assert self.conn.close_rcvd is not None - code = self.conn.close_rcvd.code - reason = self.conn.close_rcvd.reason - self.queue.put_nowait({"type": "websocket.disconnect", "code": code, "reason": reason}) - - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - self.transport.close() - - def handle_parser_exception(self) -> None: # pragma: no cover - assert self.conn.close_sent is not None - code = self.conn.close_sent.code - reason = self.conn.close_sent.reason - self.queue.put_nowait({"type": "websocket.disconnect", "code": code, "reason": reason}) - - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - self.close_sent = True - self.transport.close() - - def on_task_complete(self, task: asyncio.Task[None]) -> None: - self.tasks.discard(task) - - async def run_asgi(self) -> None: - try: - result = await self.app(self.scope, self.receive, self.send) - except ClientDisconnected: - pass # pragma: full coverage - except BaseException: - self.logger.exception("Exception in ASGI application\n") - self.send_500_response() - else: - if not self.handshake_complete: - self.logger.error("ASGI callable returned without completing handshake.") - self.send_500_response() - elif result is not None: - self.logger.error("ASGI callable should return None, but returned '%s'.", result) - self.transport.close() - - def send_500_response(self) -> None: - if self.initial_response or self.handshake_complete: - return - response = self.conn.reject(500, "Internal Server Error") - self.conn.send_response(response) - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - - async def send(self, message: ASGISendEvent) -> None: - await self.writable.wait() - - if not self.handshake_complete and self.initial_response is None: - if message["type"] == "websocket.accept": - self.logger.info( - '%s - "WebSocket %s" [accepted]', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - ) - headers = [ - (name.decode("latin-1").lower(), value.decode("latin-1")) - for name, value in (self.default_headers + list(message.get("headers", []))) - ] - accepted_subprotocol = message.get("subprotocol") - if accepted_subprotocol: - headers.append(("Sec-WebSocket-Protocol", accepted_subprotocol)) - self.response.headers.update(headers) - - if not self.transport.is_closing(): - self.handshake_complete = True - self.conn.send_response(self.response) - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - self.start_keepalive() - - elif message["type"] == "websocket.close": - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1006}) - self.logger.info( - '%s - "WebSocket %s" 403', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - ) - response = self.conn.reject(HTTPStatus.FORBIDDEN, "") - self.conn.send_response(response) - output = self.conn.data_to_send() - self.close_sent = True - self.handshake_complete = True - self.transport.write(b"".join(output)) - self.transport.close() - elif message["type"] == "websocket.http.response.start" and self.initial_response is None: - if not (100 <= message["status"] < 600): - raise RuntimeError("Invalid HTTP status code '%d' in response." % message["status"]) - self.logger.info( - '%s - "WebSocket %s" %d', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - message["status"], - ) - headers = [ - (name.decode("latin-1"), value.decode("latin-1")) - for name, value in list(message.get("headers", [])) - ] - self.initial_response = (message["status"], headers, b"") - else: - raise RuntimeError( - "Expected ASGI message 'websocket.accept', 'websocket.close' " - f"or 'websocket.http.response.start' but got '{message['type']}'." - ) - - elif not self.close_sent and self.initial_response is None: - try: - if message["type"] == "websocket.send": - bytes_data = message.get("bytes") - text_data = message.get("text") - if bytes_data is not None: - self.conn.send_binary(bytes_data) - elif text_data is not None: - self.conn.send_text(text_data.encode()) - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - - elif message["type"] == "websocket.close": - if not self.transport.is_closing(): - code = message.get("code", 1000) - reason = message.get("reason", "") or "" - self.queue.put_nowait({"type": "websocket.disconnect", "code": code, "reason": reason}) - self.conn.send_close(code, reason) - output = self.conn.data_to_send() - self.transport.write(b"".join(output)) - self.close_sent = True - self.transport.close() - else: - raise RuntimeError( - f"Expected ASGI message 'websocket.send' or 'websocket.close', but got '{message['type']}'." - ) - except InvalidState: - raise ClientDisconnected() - elif self.initial_response is not None: - if message["type"] == "websocket.http.response.body": - body = self.initial_response[2] + message["body"] - self.initial_response = self.initial_response[:2] + (body,) - if not message.get("more_body", False): - response = self.conn.reject(self.initial_response[0], body.decode()) - response.headers.update(self.initial_response[1]) - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1006}) - self.conn.send_response(response) - output = self.conn.data_to_send() - self.close_sent = True - self.transport.write(b"".join(output)) - self.transport.close() - else: # pragma: no cover - raise RuntimeError(f"Expected ASGI message 'websocket.http.response.body' but got '{message['type']}'.") - - else: - raise RuntimeError(f"Unexpected ASGI message '{message['type']}', after sending 'websocket.close'.") - - async def receive(self) -> ASGIReceiveEvent: - message = await self.queue.get() - if self.read_paused and self.queue.empty(): - self.read_paused = False - self.transport.resume_reading() - return message diff --git a/functions/python/data-processor/uvicorn/protocols/websockets/wsproto_impl.py b/functions/python/data-processor/uvicorn/protocols/websockets/wsproto_impl.py deleted file mode 100644 index adca63f9d..000000000 --- a/functions/python/data-processor/uvicorn/protocols/websockets/wsproto_impl.py +++ /dev/null @@ -1,458 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import random -import struct -from asyncio import TimerHandle -from io import BytesIO, StringIO -from typing import Any, Literal, cast -from urllib.parse import unquote - -import wsproto -from wsproto import ConnectionType, events -from wsproto.connection import ConnectionState -from wsproto.extensions import Extension, PerMessageDeflate -from wsproto.utilities import LocalProtocolError, RemoteProtocolError - -from uvicorn._types import ASGI3Application, ASGISendEvent, WebSocketEvent, WebSocketReceiveEvent, WebSocketScope -from uvicorn.config import Config -from uvicorn.logging import TRACE_LOG_LEVEL -from uvicorn.protocols.utils import ( - ClientDisconnected, - get_client_addr, - get_local_addr, - get_path_with_query_string, - get_remote_addr, - is_ssl, -) -from uvicorn.server import ServerState - - -class FrameTooLargeError(Exception): - """Raised when accumulated websocket message bytes exceed `ws_max_size`.""" - - -class WebsocketBuffer: - def __init__(self, max_length: int) -> None: - self.value: BytesIO | StringIO | None = None - self.length = 0 - self.max_length = max_length - - def extend(self, event: events.TextMessage | events.BytesMessage) -> None: - if self.value is None: - self.value = StringIO() if isinstance(event, events.TextMessage) else BytesIO() - self.value.write(event.data) # type: ignore[arg-type] - # `ws_max_size` is a byte budget, so count UTF-8 bytes for text. - self.length += len(event.data.encode()) if isinstance(event, events.TextMessage) else len(event.data) - if self.length > self.max_length: - raise FrameTooLargeError - - def clear(self) -> None: - self.value = None - self.length = 0 - - def to_message(self) -> WebSocketReceiveEvent: - if isinstance(self.value, StringIO): - return {"type": "websocket.receive", "text": self.value.getvalue()} - assert isinstance(self.value, BytesIO) - return {"type": "websocket.receive", "bytes": self.value.getvalue()} - - -class WSProtocol(asyncio.Protocol): - def __init__( - self, - config: Config, - server_state: ServerState, - app_state: dict[str, Any], - _loop: asyncio.AbstractEventLoop | None = None, - ) -> None: - if not config.loaded: - config.load() # pragma: full coverage - - self.config = config - self.app = cast(ASGI3Application, config.loaded_app) - self.loop = _loop or asyncio.get_event_loop() - self.logger = logging.getLogger("uvicorn.error") - self.root_path = config.root_path - self.app_state = app_state - - # Shared server state - self.connections = server_state.connections - self.tasks = server_state.tasks - self.default_headers = server_state.default_headers - - # Connection state - self.transport: asyncio.Transport = None # type: ignore[assignment] - self.server: tuple[str, int | None] | None = None - self.client: tuple[str, int] | None = None - self.scheme: Literal["wss", "ws"] = None # type: ignore[assignment] - - # WebSocket state - self.queue: asyncio.Queue[WebSocketEvent] = asyncio.Queue() - self.handshake_complete = False - self.close_sent = False - - # Rejection state - self.response_started = False - - self.conn = wsproto.WSConnection(connection_type=ConnectionType.SERVER) - - self.read_paused = False - self.writable = asyncio.Event() - self.writable.set() - - # Keepalive state - self.ping_interval = config.ws_ping_interval - self.ping_timeout = config.ws_ping_timeout - self.ping_timer: TimerHandle | None = None - self.pong_timer: TimerHandle | None = None - self.pending_ping_payload: bytes | None = None - self.ping_sent_at: float = 0.0 - self.last_ping_rtt: float = 0.0 - - # Buffer - self.buffer = WebsocketBuffer(self.config.ws_max_size) - - # Protocol interface - - def connection_made(self, transport: asyncio.Transport) -> None: # type: ignore[override] - self.connections.add(self) - self.transport = transport - self.server = get_local_addr(transport) - self.client = get_remote_addr(transport) - self.scheme = "wss" if is_ssl(transport) else "ws" - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket connection made", prefix) - - def connection_lost(self, exc: Exception | None) -> None: - self.stop_keepalive() - code = 1005 if self.handshake_complete else 1006 - self.queue.put_nowait({"type": "websocket.disconnect", "code": code}) - self.connections.remove(self) - - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket connection lost", prefix) - - self.handshake_complete = True - if exc is None: - self.transport.close() - - def eof_received(self) -> None: - pass - - def data_received(self, data: bytes) -> None: - try: - self.conn.receive_data(data) - except RemoteProtocolError as err: - # TODO: Remove `type: ignore` when wsproto fixes the type annotation. - self.transport.write(self.conn.send(err.event_hint)) # type: ignore[arg-type] # noqa: E501 - self.transport.close() - else: - self.handle_events() - - def handle_events(self) -> None: - for event in self.conn.events(): - if self.close_sent: - return - if isinstance(event, events.Request): - self.handle_connect(event) - elif isinstance(event, (events.TextMessage, events.BytesMessage)): - self.handle_message(event) - elif isinstance(event, events.CloseConnection): - self.handle_close(event) - elif isinstance(event, events.Ping): - self.handle_ping(event) - elif isinstance(event, events.Pong): - self.handle_pong(event) - - def pause_writing(self) -> None: - """ - Called by the transport when the write buffer exceeds the high water mark. - """ - self.writable.clear() # pragma: full coverage - - def resume_writing(self) -> None: - """ - Called by the transport when the write buffer drops below the low water mark. - """ - self.writable.set() # pragma: full coverage - - def shutdown(self) -> None: - self.stop_keepalive() - if self.handshake_complete: - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1012}) - output = self.conn.send(wsproto.events.CloseConnection(code=1012)) - self.transport.write(output) - else: - self.send_500_response() - self.transport.close() - - def on_task_complete(self, task: asyncio.Task[None]) -> None: - self.tasks.discard(task) - - # Event handlers - - def handle_connect(self, event: events.Request) -> None: - headers = [(b"host", event.host.encode())] - headers += [(key.lower(), value) for key, value in event.extra_headers] - raw_path, _, query_string = event.target.partition("?") - path = unquote(raw_path) - full_path = self.root_path + path - full_raw_path = self.root_path.encode("ascii") + raw_path.encode("ascii") - self.scope: WebSocketScope = { - "type": "websocket", - "asgi": {"version": self.config.asgi_version, "spec_version": "2.4"}, - "http_version": "1.1", - "scheme": self.scheme, - "server": self.server, - "client": self.client, - "root_path": self.root_path, - "path": full_path, - "raw_path": full_raw_path, - "query_string": query_string.encode("ascii"), - "headers": headers, - "subprotocols": event.subprotocols, - "state": self.app_state.copy(), - "extensions": {"websocket.http.response": {}}, - } - self.queue.put_nowait({"type": "websocket.connect"}) - task = self.loop.create_task(self.run_asgi()) - task.add_done_callback(self.on_task_complete) - self.tasks.add(task) - - def handle_message(self, event: events.TextMessage | events.BytesMessage) -> None: - try: - self.buffer.extend(event) - except FrameTooLargeError: - self.close_sent = True - reason = f"Message exceeds the maximum size ({self.config.ws_max_size} bytes)" - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1009, "reason": reason}) - if not self.transport.is_closing(): - self.transport.write(self.conn.send(wsproto.events.CloseConnection(code=1009, reason=reason))) - self.transport.close() - return - if event.message_finished: - self.queue.put_nowait(self.buffer.to_message()) - self.buffer.clear() - if not self.read_paused: - self.read_paused = True - self.transport.pause_reading() - - def handle_close(self, event: events.CloseConnection) -> None: - if self.conn.state == ConnectionState.REMOTE_CLOSING: - self.transport.write(self.conn.send(event.response())) - self.queue.put_nowait({"type": "websocket.disconnect", "code": event.code, "reason": event.reason}) - self.transport.close() - - def handle_ping(self, event: events.Ping) -> None: - self.transport.write(self.conn.send(event.response())) - - def handle_pong(self, event: events.Pong) -> None: - # Ignore unsolicited pongs and stale pongs whose payload doesn't match the ping currently in flight. - if self.pending_ping_payload is None or bytes(event.payload) != self.pending_ping_payload: - return # pragma: no cover - - self.last_ping_rtt = self.loop.time() - self.ping_sent_at - self.pending_ping_payload = None - # The peer answered in time; cancel the pong deadline and chain the next ping. This `schedule_ping()` call is - # what keeps the keepalive loop running when ping_timeout is set. When ping_timeout is None the next ping is - # already scheduled by `send_keepalive_ping`, so we must not schedule a duplicate here. - if self.pong_timer is not None: - self.pong_timer.cancel() - self.pong_timer = None - self.schedule_ping() - - def start_keepalive(self) -> None: - if self.ping_interval is not None and self.ping_interval > 0: - self.schedule_ping() - - def stop_keepalive(self) -> None: - if self.ping_timer is not None: - self.ping_timer.cancel() - self.ping_timer = None - if self.pong_timer is not None: # pragma: no cover - self.pong_timer.cancel() - self.pong_timer = None - self.pending_ping_payload = None - - def schedule_ping(self) -> None: - assert self.ping_interval is not None - delay = max(0.0, self.ping_interval - self.last_ping_rtt) - self.ping_timer = self.loop.call_later(delay, self.send_keepalive_ping) - - def send_keepalive_ping(self) -> None: - self.ping_timer = None - if self.close_sent or self.transport.is_closing(): # pragma: no cover - return - # Random 4-byte payload identifies this ping; `handle_pong` uses it to ignore stale or unsolicited pongs. - self.pending_ping_payload = struct.pack("!I", random.getrandbits(32)) - self.ping_sent_at = self.loop.time() - self.transport.write(self.conn.send(wsproto.events.Ping(payload=self.pending_ping_payload))) - if self.ping_timeout is not None: - self.pong_timer = self.loop.call_later(self.ping_timeout, self.keepalive_timeout) - else: # pragma: no cover - self.schedule_ping() - - def keepalive_timeout(self) -> None: - self.pong_timer = None - self.pending_ping_payload = None - if self.close_sent or self.transport.is_closing(): # pragma: no cover - return - if self.logger.level <= TRACE_LOG_LEVEL: - prefix = "%s:%d - " % self.client if self.client else "" - self.logger.log(TRACE_LOG_LEVEL, "%sWebSocket keepalive ping timeout", prefix) - reason = "keepalive ping timeout" - self.transport.write(self.conn.send(wsproto.events.CloseConnection(code=1011, reason=reason))) - self.close_sent = True - self.transport.close() - - def send_500_response(self) -> None: - if self.response_started or self.handshake_complete: - return # we cannot send responses anymore - headers: list[tuple[bytes, bytes]] = [ - (b"content-type", b"text/plain; charset=utf-8"), - (b"connection", b"close"), - (b"content-length", b"21"), - ] - output = self.conn.send(wsproto.events.RejectConnection(status_code=500, headers=headers, has_body=True)) - output += self.conn.send(wsproto.events.RejectData(data=b"Internal Server Error")) - self.transport.write(output) - - async def run_asgi(self) -> None: - try: - result = await self.app(self.scope, self.receive, self.send) # type: ignore[func-returns-value] - except ClientDisconnected: - pass # pragma: full coverage - except BaseException: - self.logger.exception("Exception in ASGI application\n") - self.send_500_response() - else: - if not self.handshake_complete: - self.logger.error("ASGI callable returned without completing handshake.") - self.send_500_response() - elif result is not None: - self.logger.error("ASGI callable should return None, but returned '%s'.", result) - self.transport.close() - - async def send(self, message: ASGISendEvent) -> None: - await self.writable.wait() - - if not self.handshake_complete: - if message["type"] == "websocket.accept": - self.logger.info( - '%s - "WebSocket %s" [accepted]', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - ) - subprotocol = message.get("subprotocol") - extra_headers = self.default_headers + list(message.get("headers", [])) - extensions: list[Extension] = [] - if self.config.ws_per_message_deflate: - extensions.append(PerMessageDeflate()) - if not self.transport.is_closing(): - self.handshake_complete = True - output = self.conn.send( - wsproto.events.AcceptConnection( - subprotocol=subprotocol, - extensions=extensions, - extra_headers=extra_headers, - ) - ) - self.transport.write(output) - self.start_keepalive() - - elif message["type"] == "websocket.close": - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1006}) - self.logger.info( - '%s - "WebSocket %s" 403', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - ) - self.handshake_complete = True - self.close_sent = True - event = events.RejectConnection(status_code=403, headers=[]) - output = self.conn.send(event) - self.transport.write(output) - self.transport.close() - - elif message["type"] == "websocket.http.response.start": - # ensure status code is in the valid range - if not (100 <= message["status"] < 600): - msg = "Invalid HTTP status code '%d' in response." - raise RuntimeError(msg % message["status"]) - self.logger.info( - '%s - "WebSocket %s" %d', - get_client_addr(self.scope), - get_path_with_query_string(self.scope), - message["status"], - ) - self.handshake_complete = True - event = events.RejectConnection( - status_code=message["status"], - headers=list(message["headers"]), - has_body=True, - ) - output = self.conn.send(event) - self.transport.write(output) - self.response_started = True - - else: - raise RuntimeError( - "Expected ASGI message 'websocket.accept', 'websocket.close' " - f"or 'websocket.http.response.start' but got '{message['type']}'." - ) - - elif not self.close_sent and not self.response_started: - try: - if message["type"] == "websocket.send": - bytes_data = message.get("bytes") - text_data = message.get("text") - data = text_data if bytes_data is None else bytes_data - output = self.conn.send(wsproto.events.Message(data=data)) # type: ignore - if not self.transport.is_closing(): - self.transport.write(output) - - elif message["type"] == "websocket.close": - self.close_sent = True - code = message.get("code", 1000) - reason = message.get("reason", "") or "" - self.queue.put_nowait({"type": "websocket.disconnect", "code": code, "reason": reason}) - output = self.conn.send(wsproto.events.CloseConnection(code=code, reason=reason)) - if not self.transport.is_closing(): - self.transport.write(output) - self.transport.close() - - else: - raise RuntimeError( - f"Expected ASGI message 'websocket.send' or 'websocket.close', but got '{message['type']}'." - ) - except LocalProtocolError as exc: - raise ClientDisconnected from exc - elif self.response_started: - if message["type"] == "websocket.http.response.body": - body_finished = not message.get("more_body", False) - reject_data = events.RejectData(data=message["body"], body_finished=body_finished) - output = self.conn.send(reject_data) - self.transport.write(output) - - if body_finished: - self.queue.put_nowait({"type": "websocket.disconnect", "code": 1006}) - self.close_sent = True - self.transport.close() - - else: - raise RuntimeError(f"Expected ASGI message 'websocket.http.response.body' but got '{message['type']}'.") - - else: - raise RuntimeError(f"Unexpected ASGI message '{message['type']}', after sending 'websocket.close'.") - - async def receive(self) -> WebSocketEvent: - message = await self.queue.get() - if self.read_paused and self.queue.empty(): - self.read_paused = False - self.transport.resume_reading() - return message diff --git a/functions/python/data-processor/uvicorn/py.typed b/functions/python/data-processor/uvicorn/py.typed deleted file mode 100644 index 8b1378917..000000000 --- a/functions/python/data-processor/uvicorn/py.typed +++ /dev/null @@ -1 +0,0 @@ - diff --git a/functions/python/data-processor/uvicorn/server.py b/functions/python/data-processor/uvicorn/server.py deleted file mode 100644 index 938a6aaa9..000000000 --- a/functions/python/data-processor/uvicorn/server.py +++ /dev/null @@ -1,346 +0,0 @@ -from __future__ import annotations - -import asyncio -import contextlib -import functools -import logging -import os -import platform -import random -import signal -import socket -import sys -import threading -import time -from collections.abc import Generator, Sequence -from email.utils import formatdate -from types import FrameType -from typing import TYPE_CHECKING, TypeAlias - -import click - -from uvicorn._compat import asyncio_run -from uvicorn.config import Config - -if TYPE_CHECKING: - from uvicorn.protocols.http.h11_impl import H11Protocol - from uvicorn.protocols.http.httptools_impl import HttpToolsProtocol - from uvicorn.protocols.websockets.websockets_impl import WebSocketProtocol - from uvicorn.protocols.websockets.websockets_sansio_impl import WebSocketsSansIOProtocol - from uvicorn.protocols.websockets.wsproto_impl import WSProtocol - - Protocols: TypeAlias = H11Protocol | HttpToolsProtocol | WSProtocol | WebSocketProtocol | WebSocketsSansIOProtocol - -HANDLED_SIGNALS = ( - signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. - signal.SIGTERM, # Unix signal 15. Sent by `kill `. -) -if sys.platform == "win32": # pragma: py-not-win32 - HANDLED_SIGNALS += (signal.SIGBREAK,) # Windows signal 21. Sent by Ctrl+Break. - -logger = logging.getLogger("uvicorn.error") - - -class ServerState: - """ - Shared servers state that is available between all protocol instances. - """ - - def __init__(self) -> None: - self.total_requests = 0 - self.connections: set[Protocols] = set() - self.tasks: set[asyncio.Task[None]] = set() - self.default_headers: list[tuple[bytes, bytes]] = [] - - -class Server: - def __init__(self, config: Config) -> None: - self.config = config - self.server_state = ServerState() - - self.started = False - self.should_exit = False - self.force_exit = False - self.last_notified = 0.0 - - self._captured_signals: list[int] = [] - - @functools.cached_property - def limit_max_requests(self) -> int | None: - if self.config.limit_max_requests is None: - return None - return self.config.limit_max_requests + random.randint(0, self.config.limit_max_requests_jitter) - - def run(self, sockets: list[socket.socket] | None = None) -> None: - return asyncio_run(self.serve(sockets=sockets), loop_factory=self.config.get_loop_factory()) - - async def serve(self, sockets: list[socket.socket] | None = None) -> None: - with self.capture_signals(): - await self._serve(sockets) - - async def _serve(self, sockets: list[socket.socket] | None = None) -> None: - process_id = os.getpid() - - config = self.config - if not config.loaded: - config.load() - - self.lifespan = config.lifespan_class(config) - - message = "Started server process [%d]" - color_message = "Started server process [" + click.style("%d", fg="cyan") + "]" - logger.info(message, process_id, extra={"color_message": color_message}) - - await self.startup(sockets=sockets) - if not self.should_exit: - await self.main_loop() - if self.started: - await self.shutdown(sockets=sockets) - - message = "Finished server process [%d]" - color_message = "Finished server process [" + click.style("%d", fg="cyan") + "]" - logger.info(message, process_id, extra={"color_message": color_message}) - - async def startup(self, sockets: list[socket.socket] | None = None) -> None: - await self.lifespan.startup() - if self.lifespan.should_exit: - self.should_exit = True - return - - config = self.config - - def create_protocol( - _loop: asyncio.AbstractEventLoop | None = None, - ) -> asyncio.Protocol: - return config.http_protocol_class( # type: ignore[call-arg] - config=config, - server_state=self.server_state, - app_state=self.lifespan.state, - _loop=_loop, - ) - - loop = asyncio.get_running_loop() - - listeners: Sequence[socket.SocketType] - if sockets is not None: # pragma: full coverage - # Explicitly passed a list of open sockets. - # We use this when the server is run from a Gunicorn worker. - - def _share_socket( - sock: socket.SocketType, - ) -> socket.SocketType: # pragma py-not-win32 - # Windows requires the socket be explicitly shared across - # multiple workers (processes). - from socket import fromshare # type: ignore[attr-defined] - - sock_data = sock.share(os.getpid()) # type: ignore[attr-defined] - return fromshare(sock_data) - - self.servers: list[asyncio.base_events.Server] = [] - for sock in sockets: - is_windows = platform.system() == "Windows" - if config.workers > 1 and is_windows: # pragma: py-not-win32 - sock = _share_socket(sock) # type: ignore[assignment] - server = await loop.create_server(create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog) - self.servers.append(server) - listeners = sockets - - elif config.fd is not None: # pragma: py-win32 - # Use an existing socket, from a file descriptor. - sock = socket.fromfd(config.fd, socket.AF_UNIX, socket.SOCK_STREAM) - server = await loop.create_server(create_protocol, sock=sock, ssl=config.ssl, backlog=config.backlog) - assert server.sockets is not None # mypy - listeners = server.sockets - self.servers = [server] - - elif config.uds is not None: # pragma: py-win32 - # Create a socket using UNIX domain socket. - uds_perms = 0o666 - if os.path.exists(config.uds): - uds_perms = os.stat(config.uds).st_mode # pragma: full coverage - server = await loop.create_unix_server( - create_protocol, path=config.uds, ssl=config.ssl, backlog=config.backlog - ) - os.chmod(config.uds, uds_perms) - assert server.sockets is not None # mypy - listeners = server.sockets - self.servers = [server] - - else: - # Standard case. Create a socket from a host/port pair. - try: - server = await loop.create_server( - create_protocol, - host=config.host, - port=config.port, - ssl=config.ssl, - backlog=config.backlog, - ) - except OSError as exc: - logger.error(exc) - await self.lifespan.shutdown() - sys.exit(1) - - assert server.sockets is not None - listeners = server.sockets - self.servers = [server] - - if sockets is None: - self._log_started_message(listeners) - else: - # We're most likely running multiple workers, so a message has already been - # logged by `config.bind_socket()`. - pass # pragma: full coverage - - self.started = True - - def _log_started_message(self, listeners: Sequence[socket.SocketType]) -> None: - config = self.config - - if config.fd is not None: # pragma: py-win32 - sock = listeners[0] - logger.info( - "Uvicorn running on socket %s (Press CTRL+C to quit)", - sock.getsockname(), - ) - - elif config.uds is not None: # pragma: py-win32 - logger.info("Uvicorn running on unix socket %s (Press CTRL+C to quit)", config.uds) - - else: - addr_format = "%s://%s:%d" - host = "0.0.0.0" if config.host is None else config.host - if ":" in host: - # It's an IPv6 address. - addr_format = "%s://[%s]:%d" - - port = config.port - if port == 0: - port = listeners[0].getsockname()[1] - - protocol_name = "https" if config.ssl else "http" - message = f"Uvicorn running on {addr_format} (Press CTRL+C to quit)" - color_message = "Uvicorn running on " + click.style(addr_format, bold=True) + " (Press CTRL+C to quit)" - logger.info( - message, - protocol_name, - host, - port, - extra={"color_message": color_message}, - ) - - async def main_loop(self) -> None: - counter = 0 - should_exit = await self.on_tick(counter) - while not should_exit: - counter += 1 - counter = counter % 864000 - await asyncio.sleep(0.1) - should_exit = await self.on_tick(counter) - - async def on_tick(self, counter: int) -> bool: - # Update the default headers, once per second. - if counter % 10 == 0: - current_time = time.time() - current_date = formatdate(current_time, usegmt=True).encode() - - if self.config.date_header: - date_header = [(b"date", current_date)] - else: - date_header = [] - - self.server_state.default_headers = date_header + self.config.encoded_headers - - # Callback to `callback_notify` once every `timeout_notify` seconds. - if self.config.callback_notify is not None: - if current_time - self.last_notified > self.config.timeout_notify: # pragma: full coverage - self.last_notified = current_time - await self.config.callback_notify() - - # Determine if we should exit. - if self.should_exit: - return True - - max_requests = self.limit_max_requests - if max_requests is not None and self.server_state.total_requests >= max_requests: - logger.info("Maximum request limit of %d exceeded. Terminating process.", max_requests) - return True - - return False - - async def shutdown(self, sockets: list[socket.socket] | None = None) -> None: - logger.info("Shutting down") - - # Stop accepting new connections. - for server in self.servers: - server.close() - for sock in sockets or []: - sock.close() # pragma: full coverage - - # Request shutdown on all existing connections. - for connection in list(self.server_state.connections): - connection.shutdown() - await asyncio.sleep(0.1) - - # When 3.10 is not supported anymore, use `async with asyncio.timeout(...):`. - try: - await asyncio.wait_for( - self._wait_tasks_to_complete(), - timeout=self.config.timeout_graceful_shutdown, - ) - except asyncio.TimeoutError: - logger.error( - "Cancel %s running task(s), timeout graceful shutdown exceeded", - len(self.server_state.tasks), - ) - for t in self.server_state.tasks: - t.cancel(msg="Task cancelled, timeout graceful shutdown exceeded") - - # Send the lifespan shutdown event, and wait for application shutdown. - if not self.force_exit: - await self.lifespan.shutdown() - - async def _wait_tasks_to_complete(self) -> None: - # Wait for existing connections to finish sending responses. - if self.server_state.connections and not self.force_exit: - msg = "Waiting for connections to close. (CTRL+C to force quit)" - logger.info(msg) - while self.server_state.connections and not self.force_exit: - await asyncio.sleep(0.1) - - # Wait for existing tasks to complete. - if self.server_state.tasks and not self.force_exit: - msg = "Waiting for background tasks to complete. (CTRL+C to force quit)" - logger.info(msg) - while self.server_state.tasks and not self.force_exit: - await asyncio.sleep(0.1) - - for server in self.servers: - await server.wait_closed() - - @contextlib.contextmanager - def capture_signals(self) -> Generator[None, None, None]: - # Signals can only be listened to from the main thread. - if threading.current_thread() is not threading.main_thread(): - yield - return - # always use signal.signal, even if loop.add_signal_handler is available - # this allows to restore previous signal handlers later on - original_handlers = {sig: signal.signal(sig, self.handle_exit) for sig in HANDLED_SIGNALS} - try: - yield - finally: - for sig, handler in original_handlers.items(): - signal.signal(sig, handler) - # If we did gracefully shut down due to a signal, try to - # trigger the expected behaviour now; multiple signals would be - # done LIFO, see https://stackoverflow.com/questions/48434964 - for captured_signal in reversed(self._captured_signals): - signal.raise_signal(captured_signal) - - def handle_exit(self, sig: int, frame: FrameType | None) -> None: - self._captured_signals.append(sig) - if self.should_exit and sig == signal.SIGINT: - self.force_exit = True # pragma: full coverage - else: - self.should_exit = True diff --git a/functions/python/data-processor/uvicorn/supervisors/__init__.py b/functions/python/data-processor/uvicorn/supervisors/__init__.py deleted file mode 100644 index cfceb6b94..000000000 --- a/functions/python/data-processor/uvicorn/supervisors/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -from __future__ import annotations - -from typing import TYPE_CHECKING - -from uvicorn.supervisors.basereload import BaseReload -from uvicorn.supervisors.multiprocess import Multiprocess - -if TYPE_CHECKING: - ChangeReload: type[BaseReload] -else: - try: - from uvicorn.supervisors.watchfilesreload import WatchFilesReload as ChangeReload - except ImportError: # pragma: no cover - from uvicorn.supervisors.statreload import StatReload as ChangeReload - -__all__ = ["Multiprocess", "ChangeReload"] diff --git a/functions/python/data-processor/uvicorn/supervisors/basereload.py b/functions/python/data-processor/uvicorn/supervisors/basereload.py deleted file mode 100644 index 66e34f53f..000000000 --- a/functions/python/data-processor/uvicorn/supervisors/basereload.py +++ /dev/null @@ -1,125 +0,0 @@ -from __future__ import annotations - -import logging -import os -import signal -import sys -import threading -from collections.abc import Callable, Iterator -from pathlib import Path -from socket import socket -from types import FrameType - -import click - -from uvicorn._subprocess import get_subprocess -from uvicorn.config import Config - -HANDLED_SIGNALS = ( - signal.SIGINT, # Unix signal 2. Sent by Ctrl+C. - signal.SIGTERM, # Unix signal 15. Sent by `kill `. -) - -logger = logging.getLogger("uvicorn.error") - - -class BaseReload: - def __init__( - self, - config: Config, - target: Callable[[list[socket] | None], None], - sockets: list[socket], - ) -> None: - self.config = config - self.target = target - self.sockets = sockets - self.should_exit = threading.Event() - self.pid = os.getpid() - self.is_restarting = False - self.reloader_name: str | None = None - - def signal_handler(self, sig: int, frame: FrameType | None) -> None: # pragma: full coverage - """ - A signal handler that is registered with the parent process. - """ - if sys.platform == "win32" and self.is_restarting: - self.is_restarting = False - else: - self.should_exit.set() - - def run(self) -> None: - self.startup() - for changes in self: - if changes: - logger.warning( - "%s detected changes in %s. Reloading...", - self.reloader_name, - ", ".join(map(_display_path, changes)), - ) - self.restart() - - self.shutdown() - - def pause(self) -> None: - if self.should_exit.wait(self.config.reload_delay): - raise StopIteration() - - def __iter__(self) -> Iterator[list[Path] | None]: - return self - - def __next__(self) -> list[Path] | None: - return self.should_restart() - - def startup(self) -> None: - message = f"Started reloader process [{self.pid}] using {self.reloader_name}" - color_message = "Started reloader process [{}] using {}".format( - click.style(str(self.pid), fg="cyan", bold=True), - click.style(str(self.reloader_name), fg="cyan", bold=True), - ) - logger.info(message, extra={"color_message": color_message}) - - for sig in HANDLED_SIGNALS: - signal.signal(sig, self.signal_handler) - - self.process = get_subprocess(config=self.config, target=self.target, sockets=self.sockets) - self.process.start() - - def restart(self) -> None: - if sys.platform == "win32": # pragma: py-not-win32 - self.is_restarting = True - assert self.process.pid is not None - os.kill(self.process.pid, signal.CTRL_C_EVENT) - - # This is a workaround to ensure the Ctrl+C event is processed - sys.stdout.write(" ") # This has to be a non-empty string - sys.stdout.flush() - else: # pragma: py-win32 - self.process.terminate() - self.process.join() - - self.process = get_subprocess(config=self.config, target=self.target, sockets=self.sockets) - self.process.start() - - def shutdown(self) -> None: - if sys.platform == "win32": - self.should_exit.set() # pragma: py-not-win32 - else: - self.process.terminate() # pragma: py-win32 - self.process.join() - - for sock in self.sockets: - sock.close() - - message = f"Stopping reloader process [{str(self.pid)}]" - color_message = "Stopping reloader process [{}]".format(click.style(str(self.pid), fg="cyan", bold=True)) - logger.info(message, extra={"color_message": color_message}) - - def should_restart(self) -> list[Path] | None: - raise NotImplementedError("Reload strategies should override should_restart()") - - -def _display_path(path: Path) -> str: - try: - return f"'{path.relative_to(Path.cwd())}'" - except ValueError: - return f"'{path}'" diff --git a/functions/python/data-processor/uvicorn/supervisors/multiprocess.py b/functions/python/data-processor/uvicorn/supervisors/multiprocess.py deleted file mode 100644 index 3ad98afab..000000000 --- a/functions/python/data-processor/uvicorn/supervisors/multiprocess.py +++ /dev/null @@ -1,223 +0,0 @@ -from __future__ import annotations - -import logging -import os -import signal -import threading -from collections.abc import Callable -from multiprocessing import Pipe -from socket import socket -from typing import Any - -import click - -from uvicorn._subprocess import get_subprocess -from uvicorn.config import Config - -SIGNALS = { - getattr(signal, f"SIG{x}"): x - for x in "INT TERM BREAK HUP QUIT TTIN TTOU USR1 USR2 WINCH".split() - if hasattr(signal, f"SIG{x}") -} - -logger = logging.getLogger("uvicorn.error") - - -class Process: - def __init__( - self, - config: Config, - target: Callable[[list[socket] | None], None], - sockets: list[socket], - ) -> None: - self.real_target = target - - self.parent_conn, self.child_conn = Pipe() - self.process = get_subprocess(config, self.target, sockets) - - def ping(self, timeout: float = 5) -> bool: - self.parent_conn.send(b"ping") - if self.parent_conn.poll(timeout): - self.parent_conn.recv() - return True - return False - - def pong(self) -> None: - self.child_conn.recv() - self.child_conn.send(b"pong") - - def always_pong(self) -> None: - while True: - self.pong() - - def target(self, sockets: list[socket] | None = None) -> Any: # pragma: no cover - if os.name == "nt": # pragma: py-not-win32 - # Windows doesn't support SIGTERM, so we use SIGBREAK instead. - # And then we raise SIGTERM when SIGBREAK is received. - # https://learn.microsoft.com/zh-cn/cpp/c-runtime-library/reference/signal?view=msvc-170 - signal.signal( - signal.SIGBREAK, # type: ignore[attr-defined] - lambda sig, frame: signal.raise_signal(signal.SIGTERM), - ) - - threading.Thread(target=self.always_pong, daemon=True).start() - return self.real_target(sockets) - - def is_alive(self, timeout: float = 5) -> bool: - if not self.process.is_alive(): - return False # pragma: full coverage - - return self.ping(timeout) - - def start(self) -> None: - self.process.start() - - def terminate(self) -> None: - if self.process.exitcode is None: # Process is still running - assert self.process.pid is not None - if os.name == "nt": # pragma: py-not-win32 - # Windows doesn't support SIGTERM. - # So send SIGBREAK, and then in process raise SIGTERM. - os.kill(self.process.pid, signal.CTRL_BREAK_EVENT) # type: ignore[attr-defined] - else: - os.kill(self.process.pid, signal.SIGTERM) - logger.info(f"Terminated child process [{self.process.pid}]") - - self.parent_conn.close() - self.child_conn.close() - - def kill(self) -> None: - # In Windows, the method will call `TerminateProcess` to kill the process. - # In Unix, the method will send SIGKILL to the process. - self.process.kill() - - def join(self) -> None: - logger.info(f"Waiting for child process [{self.process.pid}]") - self.process.join() - - @property - def pid(self) -> int | None: - return self.process.pid - - -class Multiprocess: - def __init__( - self, - config: Config, - target: Callable[[list[socket] | None], None], - sockets: list[socket], - ) -> None: - self.config = config - self.target = target - self.sockets = sockets - - self.processes_num = config.workers - self.processes: list[Process] = [] - - self.should_exit = threading.Event() - - self.signal_queue: list[int] = [] - for sig in SIGNALS: - signal.signal(sig, lambda sig, frame: self.signal_queue.append(sig)) - - def init_processes(self) -> None: - for _ in range(self.processes_num): - process = Process(self.config, self.target, self.sockets) - process.start() - self.processes.append(process) - - def terminate_all(self) -> None: - for process in self.processes: - process.terminate() - - def join_all(self) -> None: - for process in self.processes: - process.join() - - def restart_all(self) -> None: - for idx, process in enumerate(self.processes): - process.terminate() - process.join() - new_process = Process(self.config, self.target, self.sockets) - new_process.start() - self.processes[idx] = new_process - - def run(self) -> None: - message = f"Started parent process [{os.getpid()}]" - color_message = "Started parent process [{}]".format(click.style(str(os.getpid()), fg="cyan", bold=True)) - logger.info(message, extra={"color_message": color_message}) - - self.init_processes() - - while not self.should_exit.wait(0.5): - self.handle_signals() - self.keep_subprocess_alive() - - self.terminate_all() - self.join_all() - - message = f"Stopping parent process [{os.getpid()}]" - color_message = "Stopping parent process [{}]".format(click.style(str(os.getpid()), fg="cyan", bold=True)) - logger.info(message, extra={"color_message": color_message}) - - def keep_subprocess_alive(self) -> None: - if self.should_exit.is_set(): - return # parent process is exiting, no need to keep subprocess alive - - for idx, process in enumerate(self.processes): - if process.is_alive(timeout=self.config.timeout_worker_healthcheck): - continue - - process.kill() # process is hung, kill it - process.join() - - if self.should_exit.is_set(): - return # pragma: full coverage - - logger.info(f"Child process [{process.pid}] died") - process = Process(self.config, self.target, self.sockets) - process.start() - self.processes[idx] = process - - def handle_signals(self) -> None: - for sig in tuple(self.signal_queue): - self.signal_queue.remove(sig) - sig_name = SIGNALS[sig] - sig_handler = getattr(self, f"handle_{sig_name.lower()}", None) - if sig_handler is not None: - sig_handler() - else: # pragma: no cover - logger.debug(f"Received signal {sig_name}, but no handler is defined for it.") - - def handle_int(self) -> None: - logger.info("Received SIGINT, exiting.") - self.should_exit.set() - - def handle_term(self) -> None: - logger.info("Received SIGTERM, exiting.") - self.should_exit.set() - - def handle_break(self) -> None: # pragma: py-not-win32 - logger.info("Received SIGBREAK, exiting.") - self.should_exit.set() - - def handle_hup(self) -> None: # pragma: py-win32 - logger.info("Received SIGHUP, restarting processes.") - self.restart_all() - - def handle_ttin(self) -> None: # pragma: py-win32 - logger.info("Received SIGTTIN, increasing the number of processes.") - self.processes_num += 1 - process = Process(self.config, self.target, self.sockets) - process.start() - self.processes.append(process) - - def handle_ttou(self) -> None: # pragma: py-win32 - logger.info("Received SIGTTOU, decreasing number of processes.") - if self.processes_num <= 1: - logger.info("Already reached one process, cannot decrease the number of processes anymore.") - return - self.processes_num -= 1 - process = self.processes.pop() - process.terminate() - process.join() diff --git a/functions/python/data-processor/uvicorn/supervisors/statreload.py b/functions/python/data-processor/uvicorn/supervisors/statreload.py deleted file mode 100644 index 28693f9cc..000000000 --- a/functions/python/data-processor/uvicorn/supervisors/statreload.py +++ /dev/null @@ -1,52 +0,0 @@ -from __future__ import annotations - -import logging -from collections.abc import Callable, Iterator -from pathlib import Path -from socket import socket - -from uvicorn.config import Config -from uvicorn.supervisors.basereload import BaseReload - -logger = logging.getLogger("uvicorn.error") - - -class StatReload(BaseReload): - def __init__( - self, - config: Config, - target: Callable[[list[socket] | None], None], - sockets: list[socket], - ) -> None: - super().__init__(config, target, sockets) - self.reloader_name = "StatReload" - self.mtimes: dict[Path, float] = {} - - if config.reload_excludes or config.reload_includes: - logger.warning("--reload-include and --reload-exclude have no effect unless watchfiles is installed.") - - def should_restart(self) -> list[Path] | None: - self.pause() - - for file in self.iter_py_files(): - try: - mtime = file.stat().st_mtime - except OSError: # pragma: nocover - continue - - old_time = self.mtimes.get(file) - if old_time is None: - self.mtimes[file] = mtime - continue - elif mtime > old_time: - return [file] - return None - - def restart(self) -> None: - self.mtimes = {} - return super().restart() - - def iter_py_files(self) -> Iterator[Path]: - for reload_dir in self.config.reload_dirs: - for path in list(reload_dir.rglob("*.py")): - yield path.resolve() diff --git a/functions/python/data-processor/uvicorn/supervisors/watchfilesreload.py b/functions/python/data-processor/uvicorn/supervisors/watchfilesreload.py deleted file mode 100644 index 600990595..000000000 --- a/functions/python/data-processor/uvicorn/supervisors/watchfilesreload.py +++ /dev/null @@ -1,86 +0,0 @@ -from __future__ import annotations - -from collections.abc import Callable -from pathlib import Path -from socket import socket - -from watchfiles import watch - -from uvicorn.config import Config -from uvicorn.supervisors.basereload import BaseReload - - -class FileFilter: - def __init__(self, config: Config): - default_includes = ["*.py"] - self.includes = [default for default in default_includes if default not in config.reload_excludes] - self.includes.extend(config.reload_includes) - self.includes = list(set(self.includes)) - - default_excludes = [".*", ".py[cod]", ".sw.*", "~*"] - self.excludes = [default for default in default_excludes if default not in config.reload_includes] - self.exclude_dirs = [] - for e in config.reload_excludes: - p = Path(e) - try: - is_dir = p.is_dir() - except OSError: # pragma: no cover - # gets raised on Windows for values like "*.py" - is_dir = False - - if is_dir: - self.exclude_dirs.append(p) - else: - self.excludes.append(e) # pragma: full coverage - self.excludes = list(set(self.excludes)) - - def __call__(self, path: Path) -> bool: - for include_pattern in self.includes: - if path.match(include_pattern): - if str(path).endswith(include_pattern): - return True # pragma: full coverage - - for exclude_dir in self.exclude_dirs: - if exclude_dir in path.parents: - return False - - for exclude_pattern in self.excludes: - if path.match(exclude_pattern): - return False # pragma: full coverage - - return True - return False - - -class WatchFilesReload(BaseReload): - def __init__( - self, - config: Config, - target: Callable[[list[socket] | None], None], - sockets: list[socket], - ) -> None: - super().__init__(config, target, sockets) - self.reloader_name = "WatchFiles" - self.reload_dirs: list[Path] = [] - for directory in config.reload_dirs: - self.reload_dirs.append(directory) - - self.watch_filter = FileFilter(config) - self.watcher = watch( - *self.reload_dirs, - watch_filter=None, - stop_event=self.should_exit, - # using yield_on_timeout here mostly to make sure tests don't - # hang forever, won't affect the class's behavior - yield_on_timeout=True, - ignore_permission_denied=True, - ) - - def should_restart(self) -> list[Path] | None: - self.pause() - - changes = next(self.watcher) - if changes: - unique_paths = {Path(c[1]) for c in changes} - return [p for p in unique_paths if self.watch_filter(p)] - return None diff --git a/functions/python/data-processor/uvicorn/workers.py b/functions/python/data-processor/uvicorn/workers.py deleted file mode 100644 index 827d091c0..000000000 --- a/functions/python/data-processor/uvicorn/workers.py +++ /dev/null @@ -1,111 +0,0 @@ -from __future__ import annotations - -import asyncio -import logging -import signal -import sys -import warnings -from typing import Any - -from gunicorn.arbiter import Arbiter -from gunicorn.workers.base import Worker - -from uvicorn._compat import asyncio_run -from uvicorn.config import Config -from uvicorn.server import Server - -warnings.warn( - "The `uvicorn.workers` module is deprecated. Please use `uvicorn-worker` package instead.\n" - "For more details, see https://github.com/Kludex/uvicorn-worker.", - DeprecationWarning, -) - - -class UvicornWorker(Worker): - """ - A worker class for Gunicorn that interfaces with an ASGI consumer callable, - rather than a WSGI callable. - """ - - CONFIG_KWARGS: dict[str, Any] = {"loop": "auto", "http": "auto"} - - def __init__(self, *args: Any, **kwargs: Any) -> None: - super().__init__(*args, **kwargs) - - logger = logging.getLogger("uvicorn.error") - logger.handlers = self.log.error_log.handlers - logger.setLevel(self.log.error_log.level) - logger.propagate = False - - logger = logging.getLogger("uvicorn.access") - logger.handlers = self.log.access_log.handlers - logger.setLevel(self.log.access_log.level) - logger.propagate = False - - config_kwargs: dict = { - "app": None, - "log_config": None, - "timeout_keep_alive": self.cfg.keepalive, - "timeout_notify": self.timeout, - "callback_notify": self.callback_notify, - "limit_max_requests": self.max_requests, - "forwarded_allow_ips": self.cfg.forwarded_allow_ips, - } - - if self.cfg.is_ssl: - ssl_kwargs = { - "ssl_keyfile": self.cfg.ssl_options.get("keyfile"), - "ssl_certfile": self.cfg.ssl_options.get("certfile"), - "ssl_keyfile_password": self.cfg.ssl_options.get("password"), - "ssl_version": self.cfg.ssl_options.get("ssl_version"), - "ssl_cert_reqs": self.cfg.ssl_options.get("cert_reqs"), - "ssl_ca_certs": self.cfg.ssl_options.get("ca_certs"), - "ssl_ciphers": self.cfg.ssl_options.get("ciphers"), - } - config_kwargs.update(ssl_kwargs) - - if self.cfg.settings["backlog"].value: - config_kwargs["backlog"] = self.cfg.settings["backlog"].value - - config_kwargs.update(self.CONFIG_KWARGS) - - self.config = Config(**config_kwargs) - - def init_signals(self) -> None: - # Reset signals so Gunicorn doesn't swallow subprocess return codes - # other signals are set up by Server.install_signal_handlers() - # See: https://github.com/Kludex/uvicorn/issues/894 - for s in self.SIGNALS: - signal.signal(s, signal.SIG_DFL) - - signal.signal(signal.SIGUSR1, self.handle_usr1) - # Don't let SIGUSR1 disturb active requests by interrupting system calls - signal.siginterrupt(signal.SIGUSR1, False) - - def _install_sigquit_handler(self) -> None: - """Install a SIGQUIT handler on workers. - - - https://github.com/Kludex/uvicorn/issues/1116 - - https://github.com/benoitc/gunicorn/issues/2604 - """ - - loop = asyncio.get_running_loop() - loop.add_signal_handler(signal.SIGQUIT, self.handle_exit, signal.SIGQUIT, None) - - async def _serve(self) -> None: - self.config.app = self.wsgi - server = Server(config=self.config) - self._install_sigquit_handler() - await server.serve(sockets=self.sockets) - if not server.started: - sys.exit(Arbiter.WORKER_BOOT_ERROR) - - def run(self) -> None: - return asyncio_run(self._serve(), loop_factory=self.config.get_loop_factory()) - - async def callback_notify(self) -> None: - self.notify() - - -class UvicornH11Worker(UvicornWorker): - CONFIG_KWARGS = {"loop": "asyncio", "http": "h11"} diff --git a/functions/python/data-processor/lambda_function.py b/functions/python/lambda_function.py similarity index 100% rename from functions/python/data-processor/lambda_function.py rename to functions/python/lambda_function.py diff --git a/functions/python/data-processor/requirements.txt b/functions/python/requirements.txt similarity index 100% rename from functions/python/data-processor/requirements.txt rename to functions/python/requirements.txt diff --git a/functions/rust/performance-handler/Cargo.lock b/functions/rust/Cargo.lock similarity index 100% rename from functions/rust/performance-handler/Cargo.lock rename to functions/rust/Cargo.lock diff --git a/functions/rust/performance-handler/Cargo.toml b/functions/rust/Cargo.toml similarity index 100% rename from functions/rust/performance-handler/Cargo.toml rename to functions/rust/Cargo.toml diff --git a/functions/rust/performance-handler/target/.rustc_info.json b/functions/rust/performance-handler/target/.rustc_info.json deleted file mode 100644 index 615616a99..000000000 --- a/functions/rust/performance-handler/target/.rustc_info.json +++ /dev/null @@ -1 +0,0 @@ -{"rustc_fingerprint":11425412769582563416,"outputs":{"8997257406952131557":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.so\nlib___.so\nlib___.a\nlib___.so\n/Users/gjcampbell/.rustup/toolchains/stable-aarch64-apple-darwin\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"gnu\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"fp16\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sha2\"\ntarget_feature=\"spe\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"linux\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"unknown\"\nunix\n","stderr":""},"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/Users/gjcampbell/.rustup/toolchains/stable-aarch64-apple-darwin\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dit\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"dpb2\"\ntarget_feature=\"fcma\"\ntarget_feature=\"fhm\"\ntarget_feature=\"flagm\"\ntarget_feature=\"fp16\"\ntarget_feature=\"frintts\"\ntarget_feature=\"jsconv\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"paca\"\ntarget_feature=\"pacg\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rcpc2\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sb\"\ntarget_feature=\"sha2\"\ntarget_feature=\"sha3\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"apple\"\nunix\n","stderr":""},"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.95.0 (59807616e 2026-04-14)\nbinary: rustc\ncommit-hash: 59807616e1fa2540724bfbac14d7976d7e4a3860\ncommit-date: 2026-04-14\nhost: aarch64-apple-darwin\nrelease: 1.95.0\nLLVM version: 22.1.2\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/CACHEDIR.TAG b/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/CACHEDIR.TAG deleted file mode 100644 index 20d7c319c..000000000 --- a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/CACHEDIR.TAG +++ /dev/null @@ -1,3 +0,0 @@ -Signature: 8a477f597d28d172789f06886806bc55 -# This file is a cache directory tag created by cargo. -# For information about cache directory tags see https://bford.info/cachedir/ diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/bin-portfolio-rust-engine b/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/bin-portfolio-rust-engine deleted file mode 100644 index 96db51e89..000000000 --- a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/bin-portfolio-rust-engine +++ /dev/null @@ -1 +0,0 @@ -fb4defbbd2105f58 \ No newline at end of file diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/bin-portfolio-rust-engine.json b/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/bin-portfolio-rust-engine.json deleted file mode 100644 index b42c2c8b8..000000000 --- a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/bin-portfolio-rust-engine.json +++ /dev/null @@ -1 +0,0 @@ -{"rustc":2179919275645516985,"features":"[]","declared_features":"[]","target":12280563477853291191,"profile":15367835066860222016,"path":4942398508502643691,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/dep-bin-portfolio-rust-engine","checksum":false}}],"rustflags":["-C","target-cpu=neoverse-n1"],"config":3877121510173822877,"compile_kind":1912274810809736700} \ No newline at end of file diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5 b/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5 deleted file mode 100755 index b086ab7ed..000000000 Binary files a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5 and /dev/null differ diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5.d b/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5.d deleted file mode 100644 index e6cbe0a14..000000000 --- a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5.d +++ /dev/null @@ -1,5 +0,0 @@ -/Users/gjcampbell/Repos/personal_site_gh/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5.d: src/main.rs - -/Users/gjcampbell/Repos/personal_site_gh/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/deps/portfolio_rust_engine-12684ba1146441f5: src/main.rs - -src/main.rs: diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/portfolio-rust-engine.d b/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/portfolio-rust-engine.d deleted file mode 100644 index 7de96fc27..000000000 --- a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/portfolio-rust-engine.d +++ /dev/null @@ -1 +0,0 @@ -/Users/gjcampbell/Repos/personal_site_gh/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/portfolio-rust-engine: /Users/gjcampbell/Repos/personal_site_gh/functions/rust/performance-handler/src/main.rs diff --git a/functions/rust/performance-handler/target/lambda/portfolio-rust-engine/bootstrap b/functions/rust/performance-handler/target/lambda/portfolio-rust-engine/bootstrap deleted file mode 100755 index b086ab7ed..000000000 Binary files a/functions/rust/performance-handler/target/lambda/portfolio-rust-engine/bootstrap and /dev/null differ diff --git a/functions/rust/performance-handler/target/release/.cargo-lock b/functions/rust/performance-handler/target/release/.cargo-lock deleted file mode 100644 index e69de29bb..000000000 diff --git a/functions/rust/performance-handler/src/main.rs b/functions/rust/src/main.rs similarity index 100% rename from functions/rust/performance-handler/src/main.rs rename to functions/rust/src/main.rs diff --git a/functions/rust/target/.rustc_info.json b/functions/rust/target/.rustc_info.json new file mode 100644 index 000000000..c200a4263 --- /dev/null +++ b/functions/rust/target/.rustc_info.json @@ -0,0 +1 @@ +{"rustc_fingerprint":11425412769582563416,"outputs":{"17747080675513052775":{"success":true,"status":"","code":0,"stdout":"rustc 1.95.0 (59807616e 2026-04-14)\nbinary: rustc\ncommit-hash: 59807616e1fa2540724bfbac14d7976d7e4a3860\ncommit-date: 2026-04-14\nhost: aarch64-apple-darwin\nrelease: 1.95.0\nLLVM version: 22.1.2\n","stderr":""},"7971740275564407648":{"success":true,"status":"","code":0,"stdout":"___\nlib___.rlib\nlib___.dylib\nlib___.dylib\nlib___.a\nlib___.dylib\n/Users/gjcampbell/.rustup/toolchains/stable-aarch64-apple-darwin\noff\npacked\nunpacked\n___\ndebug_assertions\npanic=\"unwind\"\nproc_macro\ntarget_abi=\"\"\ntarget_arch=\"aarch64\"\ntarget_endian=\"little\"\ntarget_env=\"\"\ntarget_family=\"unix\"\ntarget_feature=\"aes\"\ntarget_feature=\"crc\"\ntarget_feature=\"dit\"\ntarget_feature=\"dotprod\"\ntarget_feature=\"dpb\"\ntarget_feature=\"dpb2\"\ntarget_feature=\"fcma\"\ntarget_feature=\"fhm\"\ntarget_feature=\"flagm\"\ntarget_feature=\"fp16\"\ntarget_feature=\"frintts\"\ntarget_feature=\"jsconv\"\ntarget_feature=\"lor\"\ntarget_feature=\"lse\"\ntarget_feature=\"neon\"\ntarget_feature=\"paca\"\ntarget_feature=\"pacg\"\ntarget_feature=\"pan\"\ntarget_feature=\"pmuv3\"\ntarget_feature=\"ras\"\ntarget_feature=\"rcpc\"\ntarget_feature=\"rcpc2\"\ntarget_feature=\"rdm\"\ntarget_feature=\"sb\"\ntarget_feature=\"sha2\"\ntarget_feature=\"sha3\"\ntarget_feature=\"ssbs\"\ntarget_feature=\"vh\"\ntarget_has_atomic=\"128\"\ntarget_has_atomic=\"16\"\ntarget_has_atomic=\"32\"\ntarget_has_atomic=\"64\"\ntarget_has_atomic=\"8\"\ntarget_has_atomic=\"ptr\"\ntarget_os=\"macos\"\ntarget_pointer_width=\"64\"\ntarget_vendor=\"apple\"\nunix\n","stderr":""}},"successes":{}} \ No newline at end of file diff --git a/functions/rust/performance-handler/target/CACHEDIR.TAG b/functions/rust/target/CACHEDIR.TAG similarity index 100% rename from functions/rust/performance-handler/target/CACHEDIR.TAG rename to functions/rust/target/CACHEDIR.TAG diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.cargo-lock b/functions/rust/target/debug/.cargo-lock similarity index 100% rename from functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.cargo-lock rename to functions/rust/target/debug/.cargo-lock diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/dep-bin-portfolio-rust-engine b/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/dep-test-bin-portfolio-rust-engine similarity index 100% rename from functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/dep-bin-portfolio-rust-engine rename to functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/dep-test-bin-portfolio-rust-engine diff --git a/functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/invoked.timestamp b/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/invoked.timestamp similarity index 100% rename from functions/rust/performance-handler/target/aarch64-unknown-linux-gnu/release/.fingerprint/portfolio-rust-engine-12684ba1146441f5/invoked.timestamp rename to functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/invoked.timestamp diff --git a/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/test-bin-portfolio-rust-engine b/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/test-bin-portfolio-rust-engine new file mode 100644 index 000000000..d65f58e2e --- /dev/null +++ b/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/test-bin-portfolio-rust-engine @@ -0,0 +1 @@ +3b24766a91ce7e17 \ No newline at end of file diff --git a/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/test-bin-portfolio-rust-engine.json b/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/test-bin-portfolio-rust-engine.json new file mode 100644 index 000000000..04ff241d5 --- /dev/null +++ b/functions/rust/target/debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/test-bin-portfolio-rust-engine.json @@ -0,0 +1 @@ +{"rustc":2179919275645516985,"features":"[]","declared_features":"[]","target":12280563477853291191,"profile":15057526963834790232,"path":4942398508502643691,"deps":[],"local":[{"CheckDepInfo":{"dep_info":"debug/.fingerprint/portfolio-rust-engine-d47120c580687ac1/dep-test-bin-portfolio-rust-engine","checksum":false}}],"rustflags":[],"config":8247474407144887393,"compile_kind":0} \ No newline at end of file diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1 b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1 new file mode 100755 index 000000000..7b662e9c6 Binary files /dev/null and b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1 differ diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.2dkmkgsvdr2ov1ipeyp0krqpt.0nvg6fj.rcgu.o b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.2dkmkgsvdr2ov1ipeyp0krqpt.0nvg6fj.rcgu.o new file mode 100644 index 000000000..fc4e9b2e8 Binary files /dev/null and b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.2dkmkgsvdr2ov1ipeyp0krqpt.0nvg6fj.rcgu.o differ diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.2h05d2zlgl8eskqkfx5phavdg.0nvg6fj.rcgu.o b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.2h05d2zlgl8eskqkfx5phavdg.0nvg6fj.rcgu.o new file mode 100644 index 000000000..f56082be4 Binary files /dev/null and b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.2h05d2zlgl8eskqkfx5phavdg.0nvg6fj.rcgu.o differ diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.7z3gckst69j96ne1vzsvfd4l2.0nvg6fj.rcgu.o b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.7z3gckst69j96ne1vzsvfd4l2.0nvg6fj.rcgu.o new file mode 100644 index 000000000..29784bc95 Binary files /dev/null and b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.7z3gckst69j96ne1vzsvfd4l2.0nvg6fj.rcgu.o differ diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.849hrjk0mnyfs4wljccmpcguz.0nvg6fj.rcgu.o b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.849hrjk0mnyfs4wljccmpcguz.0nvg6fj.rcgu.o new file mode 100644 index 000000000..6ae70bfdc Binary files /dev/null and b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.849hrjk0mnyfs4wljccmpcguz.0nvg6fj.rcgu.o differ diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.9ubxthysrx6okb0s4g3lvvex2.0nvg6fj.rcgu.o b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.9ubxthysrx6okb0s4g3lvvex2.0nvg6fj.rcgu.o new file mode 100644 index 000000000..4e0bf1c84 Binary files /dev/null and b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.9ubxthysrx6okb0s4g3lvvex2.0nvg6fj.rcgu.o differ diff --git a/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.d b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.d new file mode 100644 index 000000000..3f7924037 --- /dev/null +++ b/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.d @@ -0,0 +1,5 @@ +/Users/gjcampbell/Repos/personal_site_gh/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1.d: src/main.rs + +/Users/gjcampbell/Repos/personal_site_gh/functions/rust/target/debug/deps/portfolio_rust_engine-d47120c580687ac1: src/main.rs + +src/main.rs: diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/2dkmkgsvdr2ov1ipeyp0krqpt.o b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/2dkmkgsvdr2ov1ipeyp0krqpt.o new file mode 100644 index 000000000..fc4e9b2e8 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/2dkmkgsvdr2ov1ipeyp0krqpt.o differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/2h05d2zlgl8eskqkfx5phavdg.o b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/2h05d2zlgl8eskqkfx5phavdg.o new file mode 100644 index 000000000..f56082be4 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/2h05d2zlgl8eskqkfx5phavdg.o differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/7z3gckst69j96ne1vzsvfd4l2.o b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/7z3gckst69j96ne1vzsvfd4l2.o new file mode 100644 index 000000000..29784bc95 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/7z3gckst69j96ne1vzsvfd4l2.o differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/849hrjk0mnyfs4wljccmpcguz.o b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/849hrjk0mnyfs4wljccmpcguz.o new file mode 100644 index 000000000..6ae70bfdc Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/849hrjk0mnyfs4wljccmpcguz.o differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/9ubxthysrx6okb0s4g3lvvex2.o b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/9ubxthysrx6okb0s4g3lvvex2.o new file mode 100644 index 000000000..4e0bf1c84 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/9ubxthysrx6okb0s4g3lvvex2.o differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/dep-graph.bin b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/dep-graph.bin new file mode 100644 index 000000000..b5d83b2e0 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/dep-graph.bin differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/query-cache.bin b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/query-cache.bin new file mode 100644 index 000000000..47a34a4a3 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/query-cache.bin differ diff --git a/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/work-products.bin b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/work-products.bin new file mode 100644 index 000000000..56aa17701 Binary files /dev/null and b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6-e26442bj3nabkn6rocw4y15tg/work-products.bin differ diff --git a/functions/python/data-processor/annotated_doc/py.typed b/functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6.lock old mode 100644 new mode 100755 similarity index 100% rename from functions/python/data-processor/annotated_doc/py.typed rename to functions/rust/target/debug/incremental/portfolio_rust_engine-2x8a1jcaw3lms/s-hjh4cwe3k1-0bnyem6.lock