From a40921f628a01b339f7e633da4668ca2a0b589d7 Mon Sep 17 00:00:00 2001 From: Nagico Date: Wed, 28 Dec 2022 00:29:31 +0800 Subject: [PATCH 1/4] Support MongoDB with djongo backend --- README.rst | 72 +++++++++++++++------------ dj_database_url.py | 65 ++++++++++++++++++++---- test_dj_database_url.py | 108 ++++++++++++++++++++++++++++++++++++++++ 3 files changed, 203 insertions(+), 42 deletions(-) diff --git a/README.rst b/README.rst index 2938bcf..5d7f64b 100644 --- a/README.rst +++ b/README.rst @@ -151,37 +151,39 @@ and should instead be passed as: URL schema ---------- -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| Engine | Django Backend | URL | -+======================+===============================================+==================================================+ -| PostgreSQL | ``django.db.backends.postgresql`` [1]_ | ``postgres://USER:PASSWORD@HOST:PORT/NAME`` [2]_ | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| PostGIS | ``django.contrib.gis.db.backends.postgis`` | ``postgis://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| MSSQL | ``sql_server.pyodbc`` | ``mssql://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| MSSQL [5]_ | ``mssql`` | ``mssqlms://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| MySQL | ``django.db.backends.mysql`` | ``mysql://USER:PASSWORD@HOST:PORT/NAME`` [2]_ | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| MySQL (GIS) | ``django.contrib.gis.db.backends.mysql`` | ``mysqlgis://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| SQLite | ``django.db.backends.sqlite3`` | ``sqlite:///PATH`` [3]_ | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| SpatiaLite | ``django.contrib.gis.db.backends.spatialite`` | ``spatialite:///PATH`` [3]_ | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| Oracle | ``django.db.backends.oracle`` | ``oracle://USER:PASSWORD@HOST:PORT/NAME`` [4]_ | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| Oracle (GIS) | ``django.contrib.gis.db.backends.oracle`` | ``oraclegis://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| Redshift | ``django_redshift_backend`` | ``redshift://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| CockroachDB | ``django_cockroachdb`` | ``cockroach://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| Timescale [6]_ | ``timescale.db.backends.postgresql`` | ``timescale://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ -| Timescale (GIS) [6]_ | ``timescale.db.backend.postgis`` | ``timescalegis://USER:PASSWORD@HOST:PORT/NAME`` | -+----------------------+-----------------------------------------------+--------------------------------------------------+ ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| Engine | Django Backend | URL | ++======================+===============================================+=======================================================+ +| PostgreSQL | ``django.db.backends.postgresql`` [1]_ | ``postgres://USER:PASSWORD@HOST:PORT/NAME`` [2]_ | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| PostGIS | ``django.contrib.gis.db.backends.postgis`` | ``postgis://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| MSSQL | ``sql_server.pyodbc`` | ``mssql://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| MSSQL [5]_ | ``mssql`` | ``mssqlms://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| MySQL | ``django.db.backends.mysql`` | ``mysql://USER:PASSWORD@HOST:PORT/NAME`` [2]_ | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| MySQL (GIS) | ``django.contrib.gis.db.backends.mysql`` | ``mysqlgis://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| SQLite | ``django.db.backends.sqlite3`` | ``sqlite:///PATH`` [3]_ | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| SpatiaLite | ``django.contrib.gis.db.backends.spatialite`` | ``spatialite:///PATH`` [3]_ | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| Oracle | ``django.db.backends.oracle`` | ``oracle://USER:PASSWORD@HOST:PORT/NAME`` [4]_ | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| Oracle (GIS) | ``django.contrib.gis.db.backends.oracle`` | ``oraclegis://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| Redshift | ``django_redshift_backend`` | ``redshift://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| CockroachDB | ``django_cockroachdb`` | ``cockroach://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| Timescale [6]_ | ``timescale.db.backends.postgresql`` | ``timescale://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| Timescale (GIS) [6]_ | ``timescale.db.backend.postgis`` | ``timescalegis://USER:PASSWORD@HOST:PORT/NAME`` | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ +| MongoDB [7]_ | ``djongo`` | ``mongodb[+srv]://USER:PASSWORD@HOST:PORT/NAME`` [8]_ | ++----------------------+-----------------------------------------------+-------------------------------------------------------+ .. [1] The django.db.backends.postgresql backend is named django.db.backends.postgresql_psycopg2 in older releases. For backwards compatibility, the old name still works in newer versions. (The new name does not work in older versions). @@ -199,7 +201,13 @@ URL schema and provide a full DSN string or TNS name in ``NAME`` part. .. [5] Microsoft official `mssql-django `_ adapter. .. [6] Using the django-timescaledb Package which must be installed. - +.. [7] Using the `djongo Package `_ which must be installed. + The URL support both ``mongodb`` and ``mongodb+srv`` protocols. +.. [8] You can add extra query parameters to the URL. For example, to set the ``enforceSchema`` and ``replicaSet`` parameter, you can use the following URL: + ``mongodb://user:password@host:port/dbname?enforceSchema=true&replicaSet=rs0`` + More information about the extra query parameters can be found in the `djongo documentation `_ + and the `mongodb uri format `_. + Remember convert ``: / ? # [ ] @`` using `percent encoding `_. Contributing ------------ diff --git a/dj_database_url.py b/dj_database_url.py index 8a5e141..8db97a0 100644 --- a/dj_database_url.py +++ b/dj_database_url.py @@ -1,6 +1,6 @@ import os import urllib.parse as urlparse -from typing import Any, Dict, Optional, Union +from typing import Any, Dict, Optional, Union, Sequence, Literal from typing_extensions import TypedDict @@ -23,6 +23,8 @@ urlparse.uses_netloc.append("cockroach") urlparse.uses_netloc.append("timescale") urlparse.uses_netloc.append("timescalegis") +urlparse.uses_netloc.append("mongodb") +urlparse.uses_netloc.append("mongodb+srv") DEFAULT_ENV = "DATABASE_URL" @@ -45,6 +47,8 @@ "cockroach": "django_cockroachdb", "timescale": "timescale.db.backends.postgresql", "timescalegis": "timescale.db.backends.postgis", + "mongodb": "djongo", + "mongodb+srv": "djongo", } @@ -64,6 +68,9 @@ class DBConfig(TypedDict, total=False): TEST: Dict[str, Any] TIME_ZONE: str USER: str + # MongoDB (djongo backend): + CLIENT: Optional[Dict[str, Any]] + ENFORCE_SCHEMA: bool def config( @@ -144,12 +151,18 @@ def parse( % (spliturl.scheme, ", ".join(sorted(SCHEMES.keys()))) ) - port = ( - str(spliturl.port) - if spliturl.port - and engine in (SCHEMES["oracle"], SCHEMES["mssql"], SCHEMES["mssqlms"]) - else spliturl.port - ) + try: + port = ( + str(spliturl.port) + if spliturl.port + and engine in (SCHEMES["oracle"], SCHEMES["mssql"], SCHEMES["mssqlms"]) + else spliturl.port + ) + except Exception as e: + if engine == "djongo": # compatible with multiple host:port + port = None + else: + raise e # Update with environment configuration. parsed_config.update( @@ -165,9 +178,7 @@ def parse( ) if test_options: parsed_config.update( - { - 'TEST': test_options, - } + {'TEST': test_options,} ) # Pass the query string into OPTIONS. @@ -199,4 +210,38 @@ def parse( if engine: parsed_config["ENGINE"] = engine + # MongoDB + if engine == "djongo": + if "enforceSchema" in options: + # Remove the enforceSchema option from the options dict + parsed_config["ENFORCE_SCHEMA"] = ( + options.pop("enforceSchema").lower() == "true" + ) + + if spliturl.query == "": + host = url + else: + host = f"{url.split('?')[0]}?{urlparse.urlencode(options)}" + parsed_config["CLIENT"] = {"host": host} + + # default database + if parsed_config['NAME'] == '': + parsed_config['NAME'] = 'db' + + # pop unnecessary options + remove_key_list: Sequence[ + Literal['USER', 'PASSWORD', 'HOST', 'PORT', 'OPTIONS'] + ] = [ + 'USER', + 'PASSWORD', + 'HOST', + 'PORT', + 'OPTIONS', + ] # cannot use list[str] directly: + # https://github.com/python/mypy/issues/7178#issuecomment-1208364397 + + for key in remove_key_list: + if key in parsed_config: + parsed_config.pop(key) + return parsed_config diff --git a/test_dj_database_url.py b/test_dj_database_url.py index 0ed8b02..0371e31 100644 --- a/test_dj_database_url.py +++ b/test_dj_database_url.py @@ -531,6 +531,114 @@ def test_sqlite_memory_persistent_connection_variables(self): assert "CONN_MAX_AGE" not in url assert "CONN_HEALTH_CHECKS" not in url + def test_mongodb_parsing_least_args(self): + url = "mongodb://10.0.0.100" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "db") + self.assertEqual(url["CLIENT"]["host"], "mongodb://10.0.0.100") + + def test_mongodb_parsing_full_args(self): + url = "mongodb://foo:bar@10.0.0.100:27017/database?enforceSchema=true&retryWrites=true&w=majority" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "database") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb://foo:bar@10.0.0.100:27017/database" + "?retryWrites=true&w=majority", + ) + self.assertTrue(url["ENFORCE_SCHEMA"]) + + def test_mongodb_unix_socket_parsing(self): + url = "mongodb://%2Fvar%2Frun%2Fmongo/foo" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "foo") + self.assertEqual(url["CLIENT"]["host"], "mongodb://%2Fvar%2Frun%2Fmongo/foo") + + def test_mongodb_parsing_with_special_characters(self): + url = "mongodb://%23user:%23password@mongo.example.com/%23database" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "#database") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb://%23user:%23password@mongo.example.com/%23database", + ) + + def test_mongodb_replica_set_with_members_on_different_machines(self): + url = "mongodb://db1.example.net,db2.example.com/?replicaSet=test" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "db") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb://db1.example.net,db2.example.com/?replicaSet=test", + ) + + def test_mongodb_replica_set_with_members_on_same_machine(self): + url = "mongodb://localhost,localhost:27018,localhost:27019/?replicaSet=test" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "db") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb://localhost,localhost:27018,localhost:27019/?replicaSet=test", + ) + + def test_mongodb_replica_set_with_read_distribution(self): + url = "mongodb://example1.com,example2.com,example3.com/?replicaSet=test&readPreference=secondary" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "db") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb://example1.com,example2.com,example3.com/" + "?replicaSet=test&readPreference=secondary", + ) + + def test_mongodb_shared_cluster(self): + url = "mongodb://router1.example.com:27017,router2.example2.com:27017,router3.example3.com:27017/" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "db") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb://router1.example.com:27017,router2.example2.com:27017," + "router3.example3.com:27017/", + ) + + def test_mongodb_srv_parsing_least_args(self): + url = "mongodb+srv://server.example.com/database" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "database") + self.assertEqual( + url["CLIENT"]["host"], "mongodb+srv://server.example.com/database" + ) + + def test_mongodb_srv_parsing_full_args(self): + url = "mongodb+srv://server.example.com/?enforceSchema=false&retryWrites=true&w=majority" + url = dj_database_url.parse(url) + + self.assertEqual(url["ENGINE"], "djongo") + self.assertEqual(url["NAME"], "db") + self.assertEqual( + url["CLIENT"]["host"], + "mongodb+srv://server.example.com/?retryWrites=true&w=majority", + ) + self.assertFalse(url["ENFORCE_SCHEMA"]) + @mock.patch.dict( os.environ, {"DATABASE_URL": "postgres://user:password@instance.amazonaws.com:5431/d8r8?"}, From 1d9e78624ce9bd38e767b29846067520acca31e5 Mon Sep 17 00:00:00 2001 From: Nagico Date: Wed, 28 Dec 2022 00:34:14 +0800 Subject: [PATCH 2/4] Fix mypy error caused by from Literal --- dj_database_url.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/dj_database_url.py b/dj_database_url.py index 8db97a0..3cf8a15 100644 --- a/dj_database_url.py +++ b/dj_database_url.py @@ -1,8 +1,11 @@ import os import urllib.parse as urlparse -from typing import Any, Dict, Optional, Union, Sequence, Literal +from typing import Any, Dict, Optional, Union, Sequence -from typing_extensions import TypedDict +# Support Python 3.7. +# `try: from typing import Literal` causes: +# error: Module 'typing' has no attribute 'Literal' [attr-defined] +from typing_extensions import TypedDict, Literal # Register database schemes in URLs. urlparse.uses_netloc.append("postgres") From 0ae2f732d3b731f1bff17b00db9cef5b930d45e1 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 27 Dec 2022 16:43:03 +0000 Subject: [PATCH 3/4] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- dj_database_url.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/dj_database_url.py b/dj_database_url.py index 3cf8a15..375c321 100644 --- a/dj_database_url.py +++ b/dj_database_url.py @@ -1,11 +1,11 @@ import os import urllib.parse as urlparse -from typing import Any, Dict, Optional, Union, Sequence +from typing import Any, Dict, Optional, Sequence, Union # Support Python 3.7. # `try: from typing import Literal` causes: # error: Module 'typing' has no attribute 'Literal' [attr-defined] -from typing_extensions import TypedDict, Literal +from typing_extensions import Literal, TypedDict # Register database schemes in URLs. urlparse.uses_netloc.append("postgres") @@ -181,7 +181,9 @@ def parse( ) if test_options: parsed_config.update( - {'TEST': test_options,} + { + 'TEST': test_options, + } ) # Pass the query string into OPTIONS. From bc8da3ddd59753aa4b7703be37fe58d3e12087db Mon Sep 17 00:00:00 2001 From: Nagico Date: Wed, 28 Dec 2022 00:55:34 +0800 Subject: [PATCH 4/4] Add invalid port exception --- dj_database_url.py | 2 +- test_dj_database_url.py | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/dj_database_url.py b/dj_database_url.py index 375c321..55959a2 100644 --- a/dj_database_url.py +++ b/dj_database_url.py @@ -165,7 +165,7 @@ def parse( if engine == "djongo": # compatible with multiple host:port port = None else: - raise e + raise ValueError(f'Port parse error: {e}') # Update with environment configuration. parsed_config.update( diff --git a/test_dj_database_url.py b/test_dj_database_url.py index 0371e31..8f38df4 100644 --- a/test_dj_database_url.py +++ b/test_dj_database_url.py @@ -653,6 +653,10 @@ def test_bad_url_parsing(self): with self.assertRaisesRegex(ValueError, "No support for 'foo'. We support: "): dj_database_url.parse("foo://bar") + def test_bad_port_parsing(self): + with self.assertRaisesRegex(Exception, "Port parse error"): + dj_database_url.parse("mysql://foo:bar@localhost:65536/db") + if __name__ == "__main__": unittest.main()