PXP-8825 & BRH 206

.github/workflows/docs.yaml

@@ -31,6 +31,8 @@ jobs:
         poetry install -vv --no-interaction
         poetry show -vv
     - name: Build docs
+      env:
+        USE_AGG_MDS: true # so the aggregate MDS docs are added
       run: poetry run python run.py openapi
 
     - uses: stefanzweifel/[email protected]

.secrets.baseline

@@ -3,7 +3,7 @@
     "files": null,
     "lines": null
   },
-  "generated_at": "2022-10-03T18:10:28Z",
+  "generated_at": "2022-10-12T21:15:02Z",
   "plugins_used": [
     {
       "name": "AWSKeyDetector"
@@ -70,15 +70,15 @@
       {
         "hashed_secret": "6eae3a5b062c6d0d79f070c26e6d62486b40cb46",
         "is_verified": false,
-        "line_number": 36,
+        "line_number": 62,
         "type": "Secret Keyword"
       }
     ],
     "docs/metadata_adapters.md": [
       {
         "hashed_secret": "bf7e894868fd96c11edf05ef7d23122cbfa22e7e",
         "is_verified": false,
-        "line_number": 60,
+        "line_number": 204,
         "type": "Hex High Entropy String"
       }
     ],
@@ -106,14 +106,6 @@
         "type": "Hex High Entropy String"
       }
     ],
-    "tests/test_agg_mds_adapters.py": [
-      {
-        "hashed_secret": "143e9f2aca10dbd2711cb96047f4016f095e5709",
-        "is_verified": false,
-        "line_number": 3898,
-        "type": "Hex High Entropy String"
-      }
-    ],
     "tests/test_migrations.py": [
       {
         "hashed_secret": "4dcba4ad1d671981e2d211ebe56da8a5b40f14ef",

README.md

@@ -17,6 +17,119 @@ The server is built with [FastAPI](https://fastapi.tiangolo.com/) and packaged w
 
 The documentation can be browsed in the [docs](docs) folder, and key documents are linked below.
 
-* [Detailed API Documentation](http://petstore.swagger.io/?url=https://raw.githubusercontent.com/uc-cdis/metadata-service/master/docs/openapi.yaml)
-* [Development and deployment](docs/dev.md)
-* [Aggregate Metadata Service](docs/agg_mds.md)
+The aggregated MDS APIs and scripts copy metadata from one or many metadata services into a single data store. This enables a metadata service to act as a central API for browsing Metadata using clients such as the Ecosystem browser.
+
+The aggregate metadata APIs and migrations are disabled by default unless `USE_AGG_MDS=true` is specified. The `AGG_MDS_NAMESPACE` should also be defined for shared Elasticserach environments so that a unique index is used per-instance.
+
+The aggregate cache is built using Elasticsearch. See the `docker-compose.yaml` file (specifically the `aggregate_migration` service) for details regarding how aggregate data is populated.
+
+## Installation
+
+Install required software:
+
+* [PostgreSQL](PostgreSQL) 9.6 or above
+* [Python](https://www.python.org/downloads/) 3.9 or above
+* [Poetry](https://poetry.eustace.io/docs/#installation)
+
+Then use `poetry install` to install the dependencies. Before that,
+a [virtualenv](https://virtualenv.pypa.io/) is recommended.
+If you don't manage your own, Poetry will create one for you
+during `poetry install`, and you must activate it by:
+
+```bash
+poetry shell
+```
+
+## Development
+
+Create a file `.env` in the root directory of the checkout:
+(uncomment to override the default)
+
+```python
+# DB_HOST = "..."           # default: localhost
+# DB_PORT = ...             # default: 5432
+# DB_USER = "..."           # default: current user
+# DB_PASSWORD = "..."       # default: empty
+# DB_DATABASE = "..."       # default: current user
+# USE_AGG_MDS = "..."       # default: false
+# AGG_MDS_NAMESPACE = "..." # default: default_namespace
+# GEN3_ES_ENDPOINT = "..."  # default: empty
+```
+
+Run database schema migration:
+
+```bash
+alembic upgrade head
+```
+
+Run the server with auto-reloading:
+
+```bash
+python run.py
+```
+
+Try out the API at: <http://localhost:8000/docs>.
+
+## Run tests
+
+Please note that the name of the test database is prepended with "test_", you
+need to create that database first:
+
+```bash
+psql
+CREATE DATABASE test_metadata;
+```
+
+```bash
+pytest --cov=src --cov=migrations/versions tests
+```
+
+## Develop with Docker
+
+Use Docker compose:
+
+```bash
+docker-compose up
+```
+
+Run database schema migration as well:
+
+```bash
+docker-compose exec app alembic upgrade head
+```
+
+Run tests:
+
+```bash
+docker-compose exec app pytest --cov=src --cov=migrations/versions tests
+```
+
+### Aggregate MDS
+testing populate:
+```bash
+python src/mds/populate.py --config <config file> --hostname localhost --port 9200
+```
+view the loaded data
+```bash
+http://localhost:8000/aggregate/metadata?limit=1000
+```
+
+## Deployment
+
+For production, use [gunicorn](https://gunicorn.org/):
+
+```bash
+gunicorn mds.asgi:app -k uvicorn.workers.UvicornWorker -c gunicorn.conf.py
+```
+
+Or use the Docker image built from the `Dockerfile`, using environment variables
+with the same name to configure the server.
+
+Other than database configuration, please also set:
+
+```bash
+DEBUG=0
+ADMIN_LOGINS=alice:123,bob:456
+```
+
+Except that, don't use `123` or `456` as the password.

docker-compose.yml

@@ -36,7 +36,7 @@ services:
     environment:
       - USE_AGG_MDS=true
       - GEN3_ES_ENDPOINT=http://esproxy-service:9200
-    command: sh -c 'while [[ "$$(curl --connect-timeout 2 -s -o /dev/null -w ''%{http_code}'' $$GEN3_ES_ENDPOINT)" != "200" ]]; do echo "wait for " $$GEN3_ES_ENDPOINT; sleep 5; done; echo es backend is available;/env/bin/python /src/src/mds/populate.py --config /src/tests/config.json'
+    command: bash -c 'while [[ "$$(curl --connect-timeout 2 -s -o /dev/null -w ''%{http_code}'' $$GEN3_ES_ENDPOINT)" != "200" ]]; do echo "wait for " $$GEN3_ES_ENDPOINT; sleep 5; done; echo es backend is available;/env/bin/python /src/src/mds/populate.py --config /src/tests/config.json'
   db:
     image: postgres
     environment: