Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,18 +9,20 @@ jobs:
deploy:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
- uses: actions/checkout@v5
with:
python-version: "3.9"
persist-credentials: false
- name: Install deps
uses: knowsuchagency/poetry-install@v1
env:
POETRY_VIRTUALENVS_CREATE: false
uses: astral-sh/setup-uv@v7
with:
enable-cache: false
python-version: "3.12"
version: "latest"
- name: Set verison
run: poetry version "${{ github.ref_name }}"
- name: Release package
run: uv version "${{ github.ref_name }}"
- name: Build
run: uv build
- name: Publish
env:
POETRY_PYPI_TOKEN_PYPI: ${{ secrets.PYPI_TOKEN }}
run: poetry publish --build
UV_PUBLISH_TOKEN: ${{ secrets.PYPI_TOKEN }}
run: uv publish
45 changes: 23 additions & 22 deletions .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -12,36 +12,37 @@ jobs:
- mypy
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
- name: Install poetry
run: pipx install poetry
- name: Set up Python
uses: actions/setup-python@v4
- uses: actions/checkout@v5
with:
python-version: "3.11"
cache: "poetry"
persist-credentials: false
- id: setup-uv
uses: astral-sh/setup-uv@v7
with:
enable-cache: true
cache-suffix: 3.11
version: "latest"
python-version: 3.11
- name: Install deps
run: poetry install --all-extras
run: uv sync --all-extras
- name: Run lint check
run: poetry run pre-commit run -a ${{ matrix.cmd }}
run: uv run pre-commit run -a ${{ matrix.cmd }}
pytest:
strategy:
matrix:
py_version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
py_version: ["3.10", "3.11", "3.12", "3.13"]
runs-on: "ubuntu-latest"
steps:
- uses: actions/checkout@v2
- name: Set up Python
uses: actions/setup-python@v2
- uses: actions/checkout@v5
with:
persist-credentials: false
- id: setup-uv
uses: astral-sh/setup-uv@v7
with:
python-version: "${{ matrix.py_version }}"
- name: Update pip
run: python -m pip install -U pip
- name: Install poetry
run: python -m pip install poetry
enable-cache: true
cache-suffix: ${{ matrix.py_version }}
version: "latest"
python-version: ${{ matrix.py_version }}
- name: Install deps
run: poetry install
env:
POETRY_VIRTUALENVS_CREATE: false
run: uv sync --all-extras
- name: Run pytest check
run: poetry run pytest -vv -n auto --cov="taskiq_pipelines" .
run: uv run pytest -vv -n auto --cov="taskiq_pipelines" .
6 changes: 3 additions & 3 deletions .pre-commit-config.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,13 @@ repos:
hooks:
- id: black
name: Format with Black
entry: poetry run black
entry: uv run black
language: system
types: [python]

- id: ruff
name: Run ruff lints
entry: poetry run ruff
entry: uv run ruff
language: system
pass_filenames: false
types: [python]
Expand All @@ -35,6 +35,6 @@ repos:

- id: mypy
name: Validate types with MyPy
entry: poetry run mypy
entry: uv run mypy
language: system
types: [python]
991 changes: 0 additions & 991 deletions poetry.lock

This file was deleted.

63 changes: 38 additions & 25 deletions pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,12 +1,18 @@
[tool.poetry]
[project]
name = "taskiq-pipelines"
# The version is set automatically by the CI/CD pipeline
version = "0.0.0"
description = "Taskiq pipelines for task chaining."
authors = ["Pavel Kirilin <[email protected]>"]
authors = [{ name = "Pavel Kirilin", email = "[email protected]" }]
requires-python = ">=3.10,<4"
readme = "README.md"
repository = "https://github.com/taskiq-python/taskiq-pipelines"
license = "LICENSE"
license = "MIT"
keywords = [
"taskiq",
"pipelines",
"tasks",
"distributed",
"async",
]
classifiers = [
"Typing :: Typed",
"Programming Language :: Python",
Expand All @@ -22,26 +28,34 @@ classifiers = [
"Topic :: System :: Networking",
"Development Status :: 3 - Alpha",
]
homepage = "https://github.com/taskiq-python/taskiq-pipelines"
keywords = ["taskiq", "pipelines", "tasks", "distributed", "async"]
dependencies = [
"taskiq>=0.11.12, <1",
"typing-extensions>=4.3.0,<5",
"pydantic>=2,<3",
]

[tool.poetry.dependencies]
python = "^3.9"
taskiq = ">=0.11.12, <1"
typing-extensions = "^4.3.0"
pydantic = "^2"
[project.urls]
Homepage = "https://github.com/taskiq-python/taskiq-pipelines"
Repository = "https://github.com/taskiq-python/taskiq-pipelines"

[tool.poetry.group.dev.dependencies]
pytest = "^8"
black = { version = "^25", allow-prereleases = true }
pytest-cov = "^6"
anyio = "^4"
pre-commit = "^4"
mypy = "^1"
pytest-xdist = { version = "^3", extras = ["psutil"] }
ruff = "^0.9.9"
[dependency-groups]
dev = [
"pytest>=8,<9",
"black>=25,<26",
"pytest-cov>=6,<7",
"anyio>=4,<5",
"pre-commit>=4,<5",
"mypy>=1,<2",
"pytest-xdist[psutil]>=3,<4",
"ruff>=0.9.9,<0.10",
]

[build-system]
requires = ["hatchling"]
build-backend = "hatchling.build"

[tool.mypy]
python_version = "3.10"
strict = true
ignore_missing_imports = true
allow_subclassing_any = true
Expand All @@ -57,11 +71,8 @@ warn_unused_ignores = false
profile = "black"
multi_line_output = 3

[build-system]
requires = ["poetry-core>=1.0.0"]
build-backend = "poetry.core.masonry.api"

[tool.ruff]
target-version = "py310"
# List of enabled rulsets.
# See https://docs.astral.sh/ruff/rules/ for more information.
lint.select = [
Expand Down Expand Up @@ -90,6 +101,8 @@ lint.select = [
"ERA", # Checks for commented out code
"PL", # PyLint checks
"RUF", # Specific to Ruff checks
"FA102", # Future annotations
"UP", # Pyupgrade
]
lint.ignore = [
"D105", # Missing docstring in magic method
Expand Down
5 changes: 2 additions & 3 deletions taskiq_pipelines/abc.py
Original file line number Diff line number Diff line change
@@ -1,15 +1,14 @@
from abc import ABC, abstractmethod
from typing import Any, Dict, Type
from typing import Any, ClassVar

from taskiq import AsyncBroker, TaskiqResult
from typing_extensions import ClassVar


class AbstractStep(ABC):
"""Abstract pipeline step."""

_step_name: str
_known_steps: ClassVar[Dict[str, Type["AbstractStep"]]] = {}
_known_steps: ClassVar[dict[str, type["AbstractStep"]]] = {}

def __init_subclass__(cls, step_name: str, **kwargs: Any) -> None:
super().__init_subclass__(**kwargs)
Expand Down
4 changes: 2 additions & 2 deletions taskiq_pipelines/exceptions.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
from typing import ClassVar, Union
from typing import ClassVar

from taskiq import TaskiqError

Expand All @@ -16,7 +16,7 @@ class StepError(PipelineError):
_STEP_NAME: ClassVar[str]

task_id: str
error: Union[BaseException, None]
error: BaseException | None


class MappingError(StepError):
Expand Down
14 changes: 7 additions & 7 deletions taskiq_pipelines/middleware.py
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
from logging import getLogger
from typing import Any, List, Optional
from typing import Any

import pydantic
from taskiq import TaskiqMessage, TaskiqMiddleware, TaskiqResult
Expand Down Expand Up @@ -38,16 +38,16 @@ async def post_save( # noqa: PLR0911
return
current_step_num = int(message.labels[CURRENT_STEP])
if PIPELINE_DATA not in message.labels:
logger.warning("Pipline data not found. Execution flow is broken.")
logger.warning("Pipeline data not found. Execution flow is broken.")
return
pipeline_data = message.labels[PIPELINE_DATA]
parsed_data = self.broker.serializer.loadb(pipeline_data)
try:
steps_data = pydantic.TypeAdapter(List[DumpedStep]).validate_python(
steps_data = pydantic.TypeAdapter(list[DumpedStep]).validate_python(
parsed_data,
)
except ValueError as err:
logger.warning("Cannot parse pipline_data: %s", err, exc_info=True)
logger.warning("Cannot parse pipeline_data: %s", err, exc_info=True)
return
if current_step_num + 1 >= len(steps_data):
logger.debug("Pipeline is completed.")
Expand Down Expand Up @@ -99,11 +99,11 @@ async def on_error(
return
current_step_num = int(message.labels[CURRENT_STEP])
if PIPELINE_DATA not in message.labels:
logger.warning("Pipline data not found. Execution flow is broken.")
logger.warning("Pipeline data not found. Execution flow is broken.")
return
pipe_data = message.labels[PIPELINE_DATA]
try:
steps = pydantic.TypeAdapter(List[DumpedStep]).validate_json(pipe_data)
steps = pydantic.TypeAdapter(list[DumpedStep]).validate_json(pipe_data)
except ValueError:
return
if current_step_num == len(steps) - 1:
Expand All @@ -113,7 +113,7 @@ async def on_error(
async def fail_pipeline(
self,
last_task_id: str,
abort: Optional[BaseException] = None,
abort: BaseException | None = None,
) -> None:
"""
This function aborts pipeline.
Expand Down
Loading