New distribution [0.9.5]

 * revised hostname related models (FQDN max-length)
 * updated documentations & demos

Dockerfile

@@ -4,7 +4,7 @@ LABEL org.opencontainers.image.title="darc" \
       org.opencontainers.image.description="Darkweb Crawler Project" \
       org.opencontainers.image.url="https://darc.jarryshaw.me/" \
       org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
-      org.opencontainers.image.version="0.9.4.post2" \
+      org.opencontainers.image.version="0.9.5" \
       org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
 
 STOPSIGNAL SIGINT

darc/__init__.py

@@ -40,4 +40,4 @@
 from darc.sites import register as register_sites  # pylint: disable=unused-import
 
 __all__ = ['darc']
-__version__ = '0.9.4.post2'
+__version__ = '0.9.5'

darc/model/tasks/hostname.py

@@ -16,7 +16,7 @@
 
 from typing import TYPE_CHECKING
 
-from peewee import DateTimeField, TextField
+from peewee import CharField, DateTimeField
 
 from darc.model.abc import BaseModel
 
@@ -30,6 +30,6 @@ class HostnameQueueModel(BaseModel):
     """Hostname task queue."""
 
     #: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`).
-    hostname: str = TextField()
+    hostname: str = CharField(max_length=255, unique=True)  # a valid FQDN is at most 255 characters
     #: Timestamp of last update.
     timestamp: 'datetime' = DateTimeField()

darc/model/web/hostname.py

@@ -14,7 +14,7 @@
 
 from typing import TYPE_CHECKING
 
-from peewee import DateTimeField, TextField
+from peewee import CharField, DateTimeField
 
 from darc._compat import cached_property
 from darc._typing import SPHINX_BUILD
@@ -65,8 +65,10 @@ class HostnameModel(BaseModel):
     #: :attr:`URLModel.hostname <darc.model.web.url.URLModel.hostname>`.
     urls: 'List[URLModel]'
 
-    #: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`).
-    hostname: str = TextField()
+    #: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`). The maximum length of
+    #: the host name and of the fully qualified domain name (FQDN) is 63 bytes per
+    #: label and 255 characters per FQDN.
+    hostname: str = CharField(max_length=255, unique=True)  # a valid FQDN is at most 255 characters
     #: Proxy type (c.f. :attr:`link.proxy <darc.link.Link.proxy>`).
     proxy: Proxy = IntEnumField(choices=Proxy)
 

debug.dockerfile

@@ -15,7 +15,7 @@ LABEL org.opencontainers.image.title="darc" \
       org.opencontainers.image.description="Darkweb Crawler Project" \
       org.opencontainers.image.url="https://darc.jarryshaw.me/" \
       org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
-      org.opencontainers.image.version="0.9.4.post2" \
+      org.opencontainers.image.version="0.9.5" \
       org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
 #EXPOSE 9050
 

demo/model.py

@@ -1,6 +1,7 @@
 # -*- coding: utf-8 -*-
 # pylint: disable=ungrouped-imports
 
+import enum
 import os
 from typing import TYPE_CHECKING
 
@@ -10,12 +11,67 @@
 
 if TYPE_CHECKING:
     from datetime import datetime
-    from typing import Any, Dict, List
+    from enum import IntEnum
+    from typing import Any, Dict, List, Optional
 
 # database client
 DB = playhouse.db_url.connect(os.getenv('DB_URL', 'mysql://127.0.0.1'))
 
 
+class IntEnumField(peewee.IntegerField):
+    """:class:`enum.IntEnum` data field."""
+
+    #: The original :class:`enum.IntEnum` class.
+    choices: 'IntEnum'
+
+    # def db_value(self, value: 'Optional[IntEnum]') -> 'Optional[str]':  # pylint: disable=inconsistent-return-statements
+    #     """Dump the value for database storage.
+
+    #     Args:
+    #         val: Original enumeration object.
+
+    #     Returns:
+    #         Integral representation of the enumeration.
+
+    #     """
+    #     if value is not None:
+    #         return value
+
+    def python_value(self, value: 'Optional[int]') -> 'Optional[IntEnum]':  # pylint: disable=inconsistent-return-statements
+        """Load the value from database storage.
+
+        Args:
+            value: Integral representation of the enumeration.
+
+        Returns:
+            Original enumeration object.
+
+        """
+        if value is not None:
+            return self.choices(value)  # type: ignore
+        return None
+
+
+class Proxy(enum.IntEnum):
+    """Proxy types supported by :mod:`darc`.
+
+    .. _tor2web: https://onion.sh/
+    """
+
+    #: No proxy.
+    NULL = enum.auto()
+    #: Tor proxy.
+    TOR = enum.auto()
+    #: I2P proxy.
+    I2P = enum.auto()
+    #: ZeroNet proxy.
+    ZERONET = enum.auto()
+    #: Freenet proxy.
+    FREENET = enum.auto()
+    #: Proxied Tor (`tor2web`_, no proxy).
+    TOR2WEB = enum.auto()
+
+
 def table_function(model_class: peewee.Model) -> str:
     """Generate table name dynamically.
 
@@ -81,9 +137,9 @@ class HostnameModel(BaseModel):
     """Data model for a hostname record."""
 
     #: Hostname (c.f. :attr:`link.host <darc.link.Link.host>`).
-    hostname: str = peewee.TextField()
+    hostname: str = peewee.CharField(max_length=255, unique=True)  # a valid FQDN is at most 255 characters
     #: Proxy type (c.f. :attr:`link.proxy <darc.link.Link.proxy>`).
-    proxy: str = peewee.CharField(max_length=8)
+    proxy: 'Proxy' = IntEnumField(choices=Proxy)
 
     #: Timestamp of first ``new_host`` submission.
     discovery: 'datetime' = peewee.DateTimeField()

docker/Dockerfile

@@ -4,7 +4,7 @@ LABEL org.opencontainers.image.title="darc" \
       org.opencontainers.image.description="Darkweb Crawler Project" \
       org.opencontainers.image.url="https://darc.jarryshaw.me/" \
       org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
-      org.opencontainers.image.version="0.9.4.post2" \
+      org.opencontainers.image.version="0.9.5" \
       org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
 
 STOPSIGNAL SIGINT

docker/debug.dockerfile

@@ -15,7 +15,7 @@ LABEL org.opencontainers.image.title="darc" \
       org.opencontainers.image.description="Darkweb Crawler Project" \
       org.opencontainers.image.url="https://darc.jarryshaw.me/" \
       org.opencontainers.image.source="https://github.com/JarryShaw/darc" \
-      org.opencontainers.image.version="0.9.4.post2" \
+      org.opencontainers.image.version="0.9.5" \
       org.opencontainers.image.licenses='BSD 3-Clause "New" or "Revised" License'
 #EXPOSE 9050
 

docs/source/conf.py

@@ -37,7 +37,7 @@
 author = 'Jarry Shaw'
 
 # The full version, including alpha/beta/rc tags
-release = '0.9.4.post2'
+release = '0.9.5'
 
 
 # -- General configuration ---------------------------------------------------

docs/source/demo/model.rst

@@ -11,5 +11,10 @@ Assuming the database is using |peewee|_ as ORM and
 .. _peewee: https://docs.peewee-orm.com/
 .. _MySQL: https://mysql.com/
 
+.. important::
+
+   For more updated, battlefield-tested version of the
+   data models, please refer to :mod:`darc.model.web`.
+
 .. literalinclude:: ../../../demo/model.py
    :language: python

setup.py

@@ -26,7 +26,7 @@
 version_info = sys.version_info[:2]
 
 # version string
-__version__ = '0.9.4.post2'
+__version__ = '0.9.5'
 
 # setup attributes
 attrs = dict(