Skip to content

Commit 2e8d123

Browse files
AvTheBlackBirdAvTheBlackBird
committed
reset history
1 parent 75c31fb commit 2e8d123

File tree

16 files changed

+226
-58
lines changed

16 files changed

+226
-58
lines changed

‎CITATION.cff

+33
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
# This CITATION.cff file was generated with cffinit.
2+
# Visit https://bit.ly/cffinit to generate yours today!
3+
4+
cff-version: 1.2.0
5+
title: >-
6+
Tirtha - An Automated Platform to Crowdsource Images and
7+
Create 3D Models of Heritage Sites
8+
message: 'If you use this software, please cite it as below.'
9+
type: software
10+
authors:
11+
- family-names: Vishwakarma
12+
given-names: Anubhav
13+
orcid: 'https://orcid.org/0009-0004-9982-4268'
14+
- family-names: Shivottam
15+
given-names: Jyotirmaya
16+
orcid: 'https://orcid.org/0000-0002-2688-9421'
17+
- family-names: Mishra
18+
given-names: Subhankar
19+
orcid: 'https://orcid.org/0000-0002-9910-7291'
20+
identifiers:
21+
- type: doi
22+
value: 10.1145/3611314.3615904
23+
description: Conference Paper
24+
repository-code: 'https://github.com/smlab-niser/tirtha-public'
25+
url: 'https://smlab.niser.ac.in/project/tirtha/'
26+
keywords:
27+
- open source
28+
- digital heritage
29+
- crowdsourcing
30+
- 3D dataset
31+
- photogrammetry
32+
license: AGPL-3.0
33+
version: 0.1.0

‎Dockerfile

+36-14
Original file line numberDiff line numberDiff line change
@@ -1,35 +1,57 @@
11
# syntax=docker/dockerfile:1
22

33
# Stage 1: Base image
4-
FROM postgres
4+
FROM nvidia/cuda:11.8.0-devel-ubuntu22.04
55

6-
ENV DEBIAN_FRONTEND=noninteractive POSTGRES_PASSWORD=test1234
6+
# create environment such that no prompt occurs during the build
7+
ENV DEBIAN_FRONTEND=noninteractive
78

9+
# create the working directory
810
WORKDIR /
911

1012
# Install dependencies
1113
RUN apt-get update \
12-
&& apt-get install -y git tmux nala libpq-dev python3.11-dev nginx rabbitmq-server libpq-dev python3.11-venv
14+
&& apt-get install -y nano curl git tmux wget unzip libopencv-dev libpq-dev nginx rabbitmq-server python3.11-dev python3.11-venv python3-pip
1315

14-
RUN git clone https://github.com/smlab-niser/tirtha-public.git && cd /tirtha-public && git submodule update --init --recursive
16+
# Clone the tirtha repository and also get the submodules
17+
RUN git clone https://github.com/smlab-niser/tirtha-public.git
18+
RUN cd /tirtha-public && git config --global http.postBuffer 524288000 && git submodule update --init --recursive
19+
20+
# Initiate the python virtual environment and download the dependencies
21+
# ---------------------------------------------------------------------------------
1522
RUN python3.11 -m venv venv \
16-
&& nala install -y python3-pip && /venv/bin/pip install --upgrade pip setuptools wheel \
17-
&& cd /tirtha-public && /venv/bin/pip install -r ./requirements.txt --default-timeout=2000 \
18-
&& /venv/bin/pip install -e /tirtha-public/tirtha_bk/nn_models/nsfw_model/ && /venv/bin/pip install protobuf==3.20.3 && nala install -y wget unzip && wget https://smlab.niser.ac.in/project/tirtha/static/artifacts/MR2021.1.0.zip \
19-
&& unzip MR2021.1.0.zip && mv ./bin21/ /tirtha-public/tirtha_bk/bin21/ && rm ./MR2021.1.0.zip && wget https://smlab.niser.ac.in/project/tirtha/static/artifacts/ckpt_kadid10k.pt \
23+
&& /venv/bin/pip install --upgrade pip setuptools wheel \
24+
&& cd /tirtha-public \
25+
&& /venv/bin/pip install -r ./requirements.txt --default-timeout=2000 \
26+
&& /venv/bin/pip install -e /tirtha-public/tirtha_bk/nn_models/nsfw_model/ && /venv/bin/pip install protobuf==3.20.3
27+
28+
29+
RUN wget https://smlab.niser.ac.in/project/tirtha/static/artifacts/MR2021.1.0.zip \
30+
&& unzip MR2021.1.0.zip && mv ./bin21/ /tirtha-public/tirtha_bk/bin21/ && rm ./MR2021.1.0.zip
31+
32+
RUN wget https://smlab.niser.ac.in/project/tirtha/static/artifacts/ckpt_kadid10k.pt \
2033
&& mv ./ckpt_kadid10k.pt /tirtha-public/tirtha_bk/nn_models/MANIQA/
2134

22-
RUN nala install -y npm systemctl nano && npm install -g obj2gltf && wget https://github.com/zeux/meshoptimizer/releases/download/v0.20/gltfpack-ubuntu.zip \
35+
RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.39.7/install.sh | bash
36+
37+
RUN . /root/.bashrc && nvm install node && nvm use node && npm install -g obj2gltf
38+
39+
RUN wget https://github.com/zeux/meshoptimizer/releases/download/v0.20/gltfpack-ubuntu.zip \
2340
&& unzip gltfpack-ubuntu.zip && chmod +x gltfpack && mv gltfpack /usr/local/bin/ && rm /gltfpack-ubuntu.zip
2441

2542
RUN mv /tirtha-public/tirtha_bk/config/tirtha.docker.nginx /tirtha-public/tirtha_bk/config/tirtha.nginx \
2643
&& mv /tirtha-public/tirtha_bk/tirtha_bk/local_settings.docker.py /tirtha-public/tirtha_bk/tirtha_bk/local_settings.py \
2744
&& mv /tirtha-public/tirtha_bk/gunicorn/gunicorn.conf.docker.py /tirtha-public/tirtha_bk/gunicorn/gunicorn.conf.py
45+
# ---------------------------------------------------------------------------------
2846

29-
# COPY ./tirtha-public/tirtha_bk/gunicorn/gunicorn.conf.py /tirtha-public/tirtha_bk/gunicorn/
30-
31-
COPY ./init_db.sh /docker-entrypoint-initdb.d
32-
# COPY ./tirtha /var/www/tirtha
47+
# copy the entrypoint executable from local system to the container
3348
COPY ./start.sh /
3449

35-
RUN chmod +x /docker-entrypoint-initdb.d/init_db.sh && chmod +x /start.sh
50+
# copy the tirtha folder to the container
51+
COPY ./tirtha /var/www/tirtha
52+
53+
# make it executable
54+
RUN chmod +x /start.sh
55+
56+
# entrypoint to run the executable during start of the container
57+
ENTRYPOINT [ "/start.sh" ]

‎README.md

+30
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
## Docker support for [Project Tirtha](https://github.com/smlab-niser/tirtha-public)
2+
3+
4+
1. Requirements:
5+
* Docker [engine](https://docs.docker.com/engine/install/) and [compose](https://docs.docker.com/compose/install/).
6+
* Nvidia drivers installed.
7+
* [Nvidia container toolkit](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html).
8+
9+
10+
2. Size of the image is:
11+
12+
| REPOSITORY | SIZE |
13+
| ------------ | ------- |
14+
| tirtha-web | 21.1GB |
15+
| tirtha-db | 431MB |
16+
17+
18+
3. Steps to build the image:
19+
20+
3.1 User can change the environment variables such as the IP of the host machine, and database configuration. Important is change of Host Ip if user is trying to use the dockerfile through the ssh session. NOTE: By default, if user is using this in host pc itself nothing is required to change and password can be seen in the environment file.
21+
22+
3.2 Use following command ```sudo docker compose up```, to build and run the image tirtha-web and tirtha-deb.
23+
24+
3.3 After image build is complete, user can attach the shell with either of the containers. For the web-site tirtha-web container is required. And it can be attached with the this command. ```sudo docker exec -it tirtha-web-1 bash```
25+
26+
27+
4. Users can check Tirtha-related logs at /var/www/tirtha/logs/ and logs for system packages, like RabbitMQ, using journalctl. Postgres logs are available using ```sudo docker logs --details container_name```.
28+
29+
30+
5. Docker container will use ports 8000(gunicorn), 15672(rabbitmq) and 5432(postgres) on the host system. Ensure these ports are free on the host system.

‎compose.yml

+26-9
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,35 @@
11
services:
2-
cuda:
3-
container_name: cuda
4-
image: nvidia/cuda:12.3.1-runtime-ubuntu22.04
5-
restart: always
6-
tirtha_repo:
7-
container_name: tirtha_repo
2+
web:
83
build: .
94
restart: always
5+
command: /bin/bash
6+
tty: true
7+
deploy:
8+
resources:
9+
reservations:
10+
devices:
11+
- driver: nvidia
12+
# NOTE: Untested with multiple GPUs
13+
# count: 1 # GPU count | Default: all GPUs
14+
capabilities: [gpu]
1015
volumes:
11-
- type: bind
16+
- type: bind # user can change the source file path according to their preferences. NOTE: source file path is of the host pc. And target is from the docker container.
1217
source: ./tirtha
13-
target: /var/www/tirtha
18+
target: /var/www/tirtha
19+
env_file:
20+
- ./docker.env
1421
ports:
1522
- "8000:8000"
1623
- "15672:15672"
1724
depends_on:
18-
- cuda
25+
- db
26+
27+
db:
28+
build: ./postgres
29+
restart: always
30+
environment:
31+
- POSTGRES_PASSWORD=docker # changing password here will not reflect in the environment of database container, as it is getting changed in environment file of the database. It is just given to avoid runtime error.
32+
env_file:
33+
- ./docker.env
34+
ports:
35+
- "8001:5432" # user can change the host machine port. NOTE: the docker port should not be changed. 5432 is a default port for container based on postgres image.

‎docker.env

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
GUNICORN_PORT=8000
2+
POSTGRES_PORT_HOST=8001
3+
RMQ_WEB_POST_HOST=15672
4+
HOST_IP=0.0.0.0 # CHANGEME: Replace with your actual host IP
5+
6+
DB_NAME=dbtirtha
7+
DB_USER=dbtirthauser
8+
DB_PWD=docker
9+
DB_HOST=db
10+
DB_PORT=5432
11+
POSTGRES_PASSWORD=docker
12+
13+
RMQ_USER=rmqtirthauser
14+
RMQ_PWD=rmqtirthapwd
15+
RMQ_VHOST=rmqtirtha
16+
17+
SECRET_KEY="" # CHANGEME: Generate a new secret key using a tool or website, or run `python -c 'from django.core.management.utils import get_random_secret_key; print(get_random_secret_key())'` if python is available
18+
DEBUG=True # CHANGEME: Set to False for deployment / production
19+
PRE_URL=""
20+
TIME_ZONE="Asia/Kolkata" # CHANGEME:
21+
DEFAULT_USER_NAME="Tirtha Admin"
22+
DEFAULT_USER_MAIL=[email protected]
23+
DJANGO_SUPERUSER_PASSWORD=test
24+
DJANGO_SUPERUSER_NAME=dbadmin
25+
DJANGO_SUPERUSER_EMAIL=""
26+
27+
GOOGLE_LOGIN=False # Set to True if using Google Login
28+
GOOGLE_CLIENT_ID="" # CHANGEME: Add the client ID, if you have one
29+
30+
ARK_NAAN=999999 # CHANGEME: Integer | NOTE: NAAN - 999999 does not exist; CHECK: https://arks.org/about/testing-arks/
31+
ARK_SHOULDER="/a" # CHANGEME: | CHECK: https://arks.org/about/testing-arks/
32+
BASE_URL="http://$HOST_IP:$GUNICORN_PORT"

‎init_db.sh

-13
This file was deleted.

‎postgres/Dockerfile

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
FROM postgres
2+
WORKDIR /
3+
COPY ./init_db.sh /docker-entrypoint-initdb.d
4+
RUN chmod +x /docker-entrypoint-initdb.d/init_db.sh

‎postgres/init_db.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
set -e
3+
4+
psql -v ON_ERROR_STOP=1 --username "postgres" <<-EOSQL
5+
CREATE DATABASE $DB_NAME;
6+
CREATE USER $DB_USER WITH PASSWORD '$DB_PWD';
7+
ALTER ROLE $DB_USER SET client_encoding TO 'utf8';
8+
ALTER ROLE $DB_USER SET default_transaction_isolation TO 'read committed';
9+
ALTER ROLE $DB_USER SET timezone TO 'UTC';
10+
ALTER DATABASE $DB_NAME OWNER TO $DB_USER;
11+
GRANT ALL PRIVILEGES ON DATABASE $DB_NAME TO $DB_USER;
12+
GRANT CREATE ON SCHEMA public TO $DB_USER;
13+
EOSQL

‎start.sh

+52-22
Original file line numberDiff line numberDiff line change
@@ -1,23 +1,53 @@
1-
#
2-
cp /tirtha-public/tirtha_bk/config/tirtha.nginx /etc/nginx/sites-available/tirtha
3-
ln -s /etc/nginx/sites-available/tirtha /etc/nginx/sites-enabled/
4-
systemctl enable rabbitmq-server
5-
systemctl start rabbitmq-server
6-
rabbitmq-plugins enable rabbitmq_management
7-
rabbitmqctl add_user rmqtirthauser rmqtirthapwd
8-
rabbitmqctl add_vhost rmqtirtha
9-
rabbitmqctl set_user_tags rmqtirthauser administrator
10-
rabbitmqctl set_permissions -p rmqtirtha rmqtirthauser ".*" ".*" ".*"
11-
rabbitmqctl eval 'application:get_env(rabbit, consumer_timeout).'
12-
echo "consumer_timeout = 31556952000" | tee -a /etc/rabbitmq/rabbitmq.conf
13-
systemctl restart rabbitmq-server
14-
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/403.html /var/www/tirtha/errors/
15-
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/404.html /var/www/tirtha/errors/
16-
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/500.html /var/www/tirtha/errors/
17-
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/503.html /var/www/tirtha/errors/
18-
chmod -R 755 /var/www/tirtha/
19-
chown -R $(whoami):$(whoami) /var/www/tirtha/
1+
#!/bin/bash
2+
3+
if [ ! -f /.docker_intialized ]; then
4+
# Your initialization script
5+
echo "Running first-time setup..."
6+
cp /tirtha-public/tirtha_bk/config/tirtha.nginx /etc/nginx/sites-available/tirtha
7+
ln -s /etc/nginx/sites-available/tirtha /etc/nginx/sites-enabled/
8+
9+
# rabbitmq
10+
service rabbitmq-server start
11+
rabbitmq-plugins enable rabbitmq_management
12+
rabbitmqctl add_user $RMQ_USER $RMQ_PWD
13+
rabbitmqctl add_vhost $RMQ_VHOST
14+
rabbitmqctl set_user_tags $RMQ_USER administrator
15+
rabbitmqctl set_permissions -p $RMQ_VHOST $RMQ_USER ".*" ".*" ".*"
16+
rabbitmqctl eval 'application:get_env(rabbit, consumer_timeout).'
17+
echo "consumer_timeout = 31556952000" | tee -a /etc/rabbitmq/rabbitmq.conf
18+
service rabbitmq-server restart
19+
20+
# copying the error templates
21+
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/403.html /var/www/tirtha/errors/
22+
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/404.html /var/www/tirtha/errors/
23+
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/500.html /var/www/tirtha/errors/
24+
cp tirtha-public/tirtha_bk/tirtha/templates/tirtha/503.html /var/www/tirtha/errors/
25+
26+
# giving permissions
27+
chmod -R 755 /var/www/tirtha/
28+
chown -R $(whoami):$(whoami) /var/www/tirtha/
29+
30+
# django configuration
31+
source /venv/bin/activate
32+
python /tirtha-public/tirtha_bk/manage.py makemigrations tirtha
33+
python /tirtha-public/tirtha_bk/manage.py collectstatic --no-input
34+
python /tirtha-public/tirtha_bk/manage.py migrate
35+
DJANGO_SUPERUSER_PASSWORD=$DJANGO_SUPERUSER_PASSWORD python /tirtha-public/tirtha_bk/manage.py createsuperuser --no-input --username $DJANGO_SUPERUSER_NAME --email "$DJANGO_SUPERUSER_EMAIL"
36+
# Create a flag to prevent re-execution
37+
touch /.docker_intialized
38+
fi
39+
40+
cd /tirtha-public/tirtha_bk/
2041
source /venv/bin/activate
21-
python /tirtha-public/tirtha_bk/manage.py makemigrations tirtha
22-
python /tirtha-public/tirtha_bk/manage.py collectstatic
23-
python /tirtha-public/tirtha_bk/manage.py migrate
42+
43+
# running celery in a tmux session
44+
tmux new-session -d -s celery_session || tmux attach-session -t celery_session
45+
tmux send-keys -t celery_session "celery -A tirtha worker -l INFO --max-tasks-per-child=1 -P threads --beat" C-m
46+
47+
# starting RabbitMQ for celery
48+
service rabbitmq-server start
49+
50+
# starting the frontend | browse to 0.0.0.0:8000 in a browser
51+
gunicorn --bind 0.0.0.0:$GUNICORN_PORT tirtha_bk.wsgi
52+
53+
exec "$@"

‎tirtha/archive/archives/.gitkeep

Whitespace-only changes.

‎tirtha/archive/db_backups/.gitkeep

Whitespace-only changes.

‎tirtha/errors/.gitkeep

Whitespace-only changes.

‎tirtha/prod/logs/ImageOps/.gitkeep

Whitespace-only changes.

‎tirtha/prod/logs/MeshOps/.gitkeep

Whitespace-only changes.

‎tirtha/prod/media/models/.gitkeep

Whitespace-only changes.

‎tirtha/prod/static/.gitkeep

Whitespace-only changes.

0 commit comments

Comments
 (0)