Update mypy to 0.910

This also involved installing some new packages for the type stubs for a
few of the major third-party libraries.

I also had to change some of the imports in some model files in strange
ways, I'm not sure why some of these were necessary. I suspect this
might be a bug in mypy, but I'm not sure if I'll be able to build a
reproduction of it to be able to report it.

tildes/consumers/post_processing_script_runner.py

@@ -3,6 +3,8 @@
 
 """Consumer that runs processing scripts on posts."""
 
+from typing import Type, Union
+
 from sqlalchemy import desc
 from sqlalchemy.sql.expression import or_
 
@@ -21,6 +23,8 @@ class PostProcessingScriptRunner(EventStreamConsumer):
 
     def process_message(self, message: Message) -> None:
         """Process a message from the stream."""
+        wrapper_class: Union[Type[TopicScriptingWrapper], Type[CommentScriptingWrapper]]
+
         if "topic_id" in message.fields:
             post = (
                 self.db_session.query(Topic)

tildes/consumers/topic_metadata_generator.py

@@ -63,6 +63,9 @@ class TopicMetadataGenerator(EventStreamConsumer):
     @staticmethod
     def _generate_text_metadata(topic: Topic) -> Dict[str, Any]:
         """Generate metadata for a text topic (word count and excerpt)."""
+        if not topic.rendered_html:
+            return {}
+
         extracted_text = extract_text_from_html(topic.rendered_html)
 
         # create a short excerpt by truncating the extracted string
@@ -80,6 +83,9 @@ class TopicMetadataGenerator(EventStreamConsumer):
 
     def _generate_link_metadata(self, topic: Topic) -> Dict[str, Any]:
         """Generate metadata for a link topic (domain)."""
+        if not topic.link:
+            return {}
+
         parsed_domain = get_domain_from_url(topic.link)
 
         if self._domain_is_ip_address(parsed_domain):

tildes/mypy.ini

@@ -1,5 +1,6 @@
 [mypy]
 mypy_path = /opt/tildes/stubs/
+exclude = ^(tests|alembic)/
 disallow_untyped_defs = true
 ignore_missing_imports = true
 no_implicit_optional = true
@@ -13,5 +14,3 @@ warn_unused_ignores = true
 [mypy-tasks]
 disallow_untyped_defs = false
 
-[mypy-tests.*]
-disallow_untyped_defs = false

tildes/requirements-dev.in

@@ -8,4 +8,8 @@ pyramid-debugtoolbar
 pytest
 pytest-mock
 testing.redis
+types-bleach
+types-python-dateutil
+types-redis
+types-requests
 webtest

tildes/requirements-dev.txt

@@ -38,7 +38,7 @@ markupsafe==1.1.1
 marshmallow==3.9.0
 mccabe==0.6.1
 mypy-extensions==0.4.3
-mypy==0.790
+mypy==0.910
 packaging==20.4
 parso==0.7.1
 pastedeploy==2.1.1
@@ -104,7 +104,11 @@ traitlets==5.0.5
 transaction==3.0.0
 translationstring==1.4
 typed-ast==1.4.1
-typing-extensions==3.7.4.3
+types-bleach==3.3.3
+types-python-dateutil==0.1.4
+types-redis==3.5.4
+types-requests==2.25.0
+typing-extensions==3.10.0.0
 urllib3==1.25.11
 venusian==3.0.0
 waitress==1.4.4

tildes/tildes/lib/database.py

@@ -157,14 +157,18 @@ class RecurrenceRule(TypeDecorator):
 
     impl = Text
 
-    def process_bind_param(self, value: rrule, dialect: Dialect) -> str:
+    def process_bind_param(
+        self, value: Optional[rrule], dialect: Dialect
+    ) -> Optional[str]:
         """Convert the rrule value to a string to store it."""
         if value is None:
             return value
 
         return rrule_to_str(value)
 
-    def process_result_value(self, value: str, dialect: Dialect) -> rrule:
+    def process_result_value(
+        self, value: Optional[str], dialect: Dialect
+    ) -> Optional[rrule]:
         """Convert the stored string to an rrule."""
         if value is None:
             return value

tildes/tildes/lib/event_stream.py

@@ -6,7 +6,7 @@
 import os
 from abc import abstractmethod
 from configparser import ConfigParser
-from typing import Any, Dict, List, Sequence
+from typing import Any, Dict, List, Optional, Sequence
 
 from prometheus_client import CollectorRegistry, Counter, start_http_server
 from redis import Redis, ResponseError
@@ -47,7 +47,7 @@ class EventStreamConsumer:
     necessary. It relies on the environment variable INI_FILE being set.
     """
 
-    METRICS_PORT = None
+    METRICS_PORT: Optional[int] = None
 
     def __init__(
         self,

tildes/tildes/lib/markdown.py

@@ -5,7 +5,18 @@
 
 import re
 from functools import partial
-from typing import Any, Callable, Iterator, List, Match, Optional, Pattern, Tuple
+from typing import (
+    Any,
+    Callable,
+    Dict,
+    Iterator,
+    List,
+    Match,
+    Optional,
+    Pattern,
+    Tuple,
+    Union,
+)
 
 import bleach
 from bs4 import BeautifulSoup
@@ -94,7 +105,7 @@ ALLOWED_HTML_TAGS = (
 )
 ALLOWED_LINK_PROTOCOLS = ("gemini", "http", "https", "mailto")
 
-ALLOWED_HTML_ATTRIBUTES_DEFAULT = {
+ALLOWED_HTML_ATTRIBUTES_DEFAULT: Dict[str, Union[List[str], Callable]] = {
     "a": ["href", "title"],
     "details": ["open"],
     "ol": ["start"],

tildes/tildes/models/comment/comment_bookmark.py

@@ -7,9 +7,10 @@ from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import text
 
 from tildes.models import DatabaseModel
-from tildes.models.comment import Comment
 from tildes.models.user import User
 
+from .comment import Comment
+
 
 class CommentBookmark(DatabaseModel):
     """Model for a comment bookmark."""

tildes/tildes/models/comment/comment_tree.py

@@ -272,7 +272,7 @@ class CommentInTree(ObjectProxy):
         self.replies: List[CommentInTree] = []
         self.has_visible_descendant = False
         self.num_children = 0
-        self.depth: Optional[int] = None
+        self.depth = 0
 
     @property
     def has_uncollapsed_descendant(self) -> bool:

tildes/tildes/models/group/group_subscription.py

@@ -11,7 +11,7 @@ from sqlalchemy.sql.expression import text
 
 from tildes.metrics import incr_counter
 from tildes.models import DatabaseModel
-from tildes.models.user import User
+from tildes.models.user.user import User
 
 from .group import Group
 

tildes/tildes/models/group/group_wiki_page.py

@@ -20,7 +20,7 @@ from tildes.lib.html import add_anchors_to_headings
 from tildes.lib.markdown import convert_markdown_to_safe_html
 from tildes.lib.string import convert_to_url_slug
 from tildes.models import DatabaseModel
-from tildes.models.user import User
+from tildes.models.user.user import User
 from tildes.schemas.group_wiki_page import GroupWikiPageSchema, PAGE_NAME_MAX_LENGTH
 from tildes.typing import AclType
 

tildes/tildes/models/topic/topic_bookmark.py

@@ -7,9 +7,10 @@ from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import text
 
 from tildes.models import DatabaseModel
-from tildes.models.topic import Topic
 from tildes.models.user import User
 
+from .topic import Topic
+
 
 class TopicBookmark(DatabaseModel):
     """Model for a topic bookmark."""

tildes/tildes/models/topic/topic_ignore.py

@@ -7,9 +7,10 @@ from sqlalchemy.orm import relationship
 from sqlalchemy.sql.expression import text
 
 from tildes.models import DatabaseModel
-from tildes.models.topic import Topic
 from tildes.models.user import User
 
+from .topic import Topic
+
 
 class TopicIgnore(DatabaseModel):
     """Model for an ignored topic."""

tildes/tildes/models/topic/topic_schedule.py

@@ -25,10 +25,11 @@ from tildes.lib.database import RecurrenceRule, TagList
 from tildes.lib.datetime import utc_now
 from tildes.models import DatabaseModel
 from tildes.models.group import Group
-from tildes.models.topic import Topic
 from tildes.models.user import User
 from tildes.schemas.topic import TITLE_MAX_LENGTH
 
+from .topic import Topic
+
 
 class TopicSchedule(DatabaseModel):
     """Model for scheduled topics (auto-posted, often repeatedly on a schedule).

tildes/tildes/models/user/user_group_settings.py

@@ -12,7 +12,8 @@ from sqlalchemy.orm import relationship
 from tildes.enums import TopicSortOption
 from tildes.models import DatabaseModel
 from tildes.models.group import Group
-from tildes.models.user import User
+
+from .user import User
 
 
 class UserGroupSettings(DatabaseModel):

tildes/tildes/models/user/user_permissions.py

@@ -10,7 +10,8 @@ from sqlalchemy.orm import relationship
 from tildes.enums import UserPermission, UserPermissionType
 from tildes.models import DatabaseModel
 from tildes.models.group import Group
-from tildes.models.user import User
+
+from .user import User
 
 
 class UserPermissions(DatabaseModel):