Jump to content

SRE/Data Persistence/Documentation/Python

From Wikitech

Python development tips

This page documents Python development tips for the Data Persistence team and SREs in general.

If you starting a new project you might also consider https://gitlab.wikimedia.org/repos/data_persistence/service-template

General tips

  • Use type hints and run mypy based on the configuration below
  • Use ruff with the configuration below
  • Use Pydantic on new projects for stronger type checking
  • Use pytest with mock for testing
    • Use caplog and syrupy to capture and check logs

Best practices

  • Prefer using logging with levels and timestamps
    • In complex scripts it can be difficult to understand who-logged-what: show the tool or function or target name frequently or in each log
  • Use progress bars or terminal manipulation sparingly
    • Either send that to stderr or disable it when stdout is redirected so that stdout can be captured in a "clean" state

pyproject.toml

Create pyproject.toml as: 

# Tools are sorted by name


[tool.black]
line-length = 120


[tool.flake8]
max-line-length = 120


[tool.mypy]
# Fine-grained mypy configuration. You can incrementally enable stricter checks as you improve type hints.
check_untyped_defs = true        
# disallow_any_generics = true
disallow_incomplete_defs = true
disallow_untyped_calls = true
follow_imports = "skip"
ignore_missing_imports = true
no_implicit_optional = true
pretty = true
show_error_context = true
strict_equality = true
#strict = true
warn_redundant_casts = true
# warn_return_any = true
warn_unused_configs = true
warn_unused_ignores = true
# disallow_any_expr = true
disallow_any_explicit = true


[tool.pylsp-mypy]
# Enable mypy integration in the Python LSP server
enabled = true
live_mode = true
# strict = true

 
[tool.pytest.ini_options]
norecursedirs = ["build", "dist", ".git", ".tox"]


[tool.ruff]
# Minimum python version to support
target-version = "py311"

# Exclude commonly ignored directories
exclude = [
    ".direnv",
    ".eggs",
    ".git",
    ".git-rewrite",
    ".mypy_cache",
    ".nox",
    ".pants.d",
    ".pyenv",
    ".pytest_cache",
    ".pytype",
    ".ruff_cache",
    ".tox",
    ".venv",
    ".vscode",
    "__pypackages__",
    "_build",
    "build",
    "dist",
    "site-packages",
    "venv",
]

line-length = 120

[tool.ruff.lint]
# You can incrementally enable stricter checks as needed
select = [
    "E",  # Pycodestyle errors
    "F",  # Pyflakes
    "ERA",  # Eradicate
    "YTT",  # Flake8 2020
    "A",  # Flake8 builtins
    "T10",  # Flake8 debugger
    "EXE",  # Flake8 executable
    "G",  # Flake8 logging format
    "PIE",  # Flake8 pie
    "T20",  # Flake8 print
    "SLOT",  # Flake8 slots
    "TID",  # Flake8 tidy imports
    "I",  # Isort
    "W",  # Pycodestyle warnings

    # "S",  # Flake8 bandit                            
    # "B",  # Flake8 bugbear                           
    # "PL",  # Pylint                                   
    #
    # "ANN",  # Flake8 annotations                     
    # "BLE",  # Flake8 blind except                    
    # "FBT",  # Flake8 boolean trap                    
    # "COM",  # Flake8 commas
    # "C4",  # Flake8 comprehensions
    # "DTZ",  # Flake8 datetimez
    # "ISC",  # Flake8 implicit string concat          
    # "INP",  # Flake8 no pep420                       
    # "PT",  # Flake8 pytest style                     
    # "SLF",  # Flake8 self - private member access     
    # "SIM",  # Flake8 simplify                        
    # "ARG",  # Flake8 unused arguments                
    # "PTH",  # Flake8 use pathlib                      
    # "RUF",  # Ruff specific rules                    
    # "TRY",  # Tryceratops                             
]

ignore = [
    "ANN001",  # Missing type annotation for function argument
    "ANN201",  # Missing return type annotation for public function
    "ANN202",  # Missing return type annotation for private function
    "ANN204",  # Missing return type annotation for special method `__init__`
    "ANN401",  # Allow Any type hint
    "C408",    # Unnecessary `dict()` call (rewrite as a literal)
    "COM812",  # Trailing commas enforced by the formatter
    "D200",    # One-line docstring should fit on one line
    "D205",    # 1 blank line required between summary line and description
    "D212",    # Multi-line docstring summary should start at the first line
    "D400",    # First line should end with a period
    "D415",    # First line should end with a period, question mark, or exclamation point
    "ERA001",  # Commented-out code
    "G004",    # Logging statement uses f-string
    "I001",    # Import block is un-sorted
    "PLR",     # Ignore Pylint refactor rules - we run pylint directly
    "PTH123",  # Do not force to use Path in all cases
    "Q000",    # Allow to use the built-in open() function
    "S101",    # Use of `assert` detected
    "S603",    # Subprocess calls without shell=True
    "SIM108",  # Don't force ternary operator always
    "T201",    # Allow print() calls
    "TRY003",  # Allow exception classes that don't define a common message
    "TRY400",  # Allow to log just the error message and not the full exception traceback

]

[tool.ruff.lint.per-file-ignores]
"tests/*" = [
    "ANN",     # Ignore type hints checks in tests
    "ARG001",  # Unused function argument
    "B011",    # Do not `assert False`
    "E501",    # Long lines
    "INP001",  # File `tests/unit/test_import.py` is part of an implicit namespace package.
    "PT015",   # Assertion always fails, replace with `pytest.fail()`
    "S101",    # Allow asserts
    "S106",    # Allow hardcoded fake passwords in tests
]

[tool.ruff.lint.flake8-bandit]
check-typed-exception = true

[tool.ruff.format]
docstring-code-format = true

This replaces the need for dedicated conf files for mypy and tox.

Security

Where possible consider using only dependencies from Debian instead of pip/poetry install.

Retrieved from "https://wikitech.wikimedia.org/w/index.php?title=SRE/Data_Persistence/Documentation/Python&oldid=2413830"