Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace verdi database commands with verdi storage #5228

Merged
merged 6 commits into from
Nov 16, 2021
Merged

Replace verdi database commands with verdi storage #5228

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1e27f3b
Replace `verdi database` commands with `verdi storage`
ramirezfranciscof Nov 10, 2021
63ca7e4
Apply suggestions from code review
ramirezfranciscof Nov 12, 2021
c5eb984
Add modifications of code review
ramirezfranciscof Nov 12, 2021
55c8b7e
Second batch of PR modifications.
ramirezfranciscof Nov 12, 2021
49d1097
Merge branch 'develop' into repocli_base
ramirezfranciscof Nov 15, 2021
bf8ad01
Quickfix
ramirezfranciscof Nov 16, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions aiida/cmdline/commands/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,5 +32,6 @@
cmd_setup,
cmd_shell,
cmd_status,
cmd_storage,
cmd_user,
)
221 changes: 56 additions & 165 deletions aiida/cmdline/commands/cmd_database.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,95 +8,60 @@
# For further information please visit http://www.aiida.net #
###########################################################################
"""`verdi database` commands."""
# pylint: disable=unused-argument
sphuber marked this conversation as resolved.
Show resolved Hide resolved

import click

from aiida.backends.general.migrations.duplicate_uuids import TABLES_UUID_DEDUPLICATION
from aiida.cmdline.commands.cmd_verdi import verdi
from aiida.cmdline.params import options
from aiida.cmdline.utils import decorators, echo
from aiida.common import exceptions
from aiida.cmdline.utils import decorators


@verdi.group('database')
def verdi_database():
"""Inspect and manage the database."""
"""Inspect and manage the database.

.. deprecated:: v2.0.0
"""


@verdi_database.command('version')
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'The same information is now available through `verdi status`.\n'
)
def database_version():
"""Show the version of the database.

The database version is defined by the tuple of the schema generation and schema revision.
"""
from aiida.manage.manager import get_manager

manager = get_manager()
manager._load_backend(schema_check=False) # pylint: disable=protected-access
backend_manager = manager.get_backend_manager()

echo.echo('Generation: ', bold=True, nl=False)
echo.echo(backend_manager.get_schema_generation_database())
echo.echo('Revision: ', bold=True, nl=False)
echo.echo(backend_manager.get_schema_version_backend())
.. deprecated:: v2.0.0
"""


@verdi_database.command('migrate')
@options.FORCE()
def database_migrate(force):
"""Migrate the database to the latest schema version."""
from aiida.engine.daemon.client import get_daemon_client
from aiida.manage.manager import get_manager

client = get_daemon_client()
if client.is_daemon_running:
echo.echo_critical('Migration aborted, the daemon for the profile is still running.')

manager = get_manager()
profile = manager.get_profile()
backend = manager._load_backend(schema_check=False) # pylint: disable=protected-access

if force:
try:
backend.migrate()
except (exceptions.ConfigurationError, exceptions.DatabaseMigrationError) as exception:
echo.echo_critical(str(exception))
return

echo.echo_warning('Migrating your database might take a while and is not reversible.')
echo.echo_warning('Before continuing, make sure you have completed the following steps:')
echo.echo_warning('')
echo.echo_warning(' 1. Make sure you have no active calculations and workflows.')
echo.echo_warning(' 2. If you do, revert the code to the previous version and finish running them first.')
echo.echo_warning(' 3. Stop the daemon using `verdi daemon stop`')
echo.echo_warning(' 4. Make a backup of your database and repository')
echo.echo_warning('')
echo.echo_warning('', nl=False)

expected_answer = 'MIGRATE NOW'
confirm_message = 'If you have completed the steps above and want to migrate profile "{}", type {}'.format(
profile.name, expected_answer
)

try:
response = click.prompt(confirm_message)
while response != expected_answer:
response = click.prompt(confirm_message)
except click.Abort:
echo.echo('\n')
echo.echo_critical('Migration aborted, the data has not been affected.')
else:
try:
backend.migrate()
except (exceptions.ConfigurationError, exceptions.DatabaseMigrationError) as exception:
echo.echo_critical(str(exception))
else:
echo.echo_success('migration completed')
@click.pass_context
@decorators.deprecated_command(
'This command has been deprecated and will be removed soon (in v3.0). '
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is it intentional that this will be removed in 3.0 and the rest in 2.1?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, this is intentional. I personally would prefer to leave the deprecations for longer (not sure if until 3.0, but probably more than 2.1 since I suspect this will be out relatively soon), but this is what was agreed in the last meeting (see notes).

BTW: I'm still not convinced about exposing this information to the user since this is mostly for internal reference (i.e. remembering when to take it out), but I didn't know where else to track this since the .. deprecated:: vA.B.C addition to the docstring is for recording when it was deprecated and not up to when. Any suggestions?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't agree that it is for internal remembering. Since it affects the user exactly when it is going to be removed, it seems like very relevant information for them. Only thing missing is when the version that it will be removed in is expected to be released.

'Please call `verdi storage migrate` instead.\n'
)
def database_migrate(ctx, force):
"""Migrate the database to the latest schema version.

.. deprecated:: v2.0.0
"""
from aiida.cmdline.commands.cmd_storage import backend_migrate
ctx.forward(backend_migrate)


@verdi_database.group('integrity')
def verdi_database_integrity():
"""Check the integrity of the database and fix potential issues."""
"""Check the integrity of the database and fix potential issues.

.. deprecated:: v2.0.0
"""


@verdi_database_integrity.command('detect-duplicate-uuid')
Expand All @@ -110,6 +75,10 @@ def verdi_database_integrity():
@click.option(
'-a', '--apply-patch', is_flag=True, help='Actually apply the proposed changes instead of performing a dry run.'
)
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'For remaining available integrity checks, use `verdi storage integrity` instead.\n'
)
def detect_duplicate_uuid(table, apply_patch):
"""Detect and fix entities with duplicate UUIDs.

Expand All @@ -119,123 +88,45 @@ def detect_duplicate_uuid(table, apply_patch):
constraint on UUIDs on the database level. However, this would leave databases created before this patch with
duplicate UUIDs in an inconsistent state. This command will run an analysis to detect duplicate UUIDs in a given
table and solve it by generating new UUIDs. Note that it will not delete or merge any rows.
"""
from aiida.backends.general.migrations.duplicate_uuids import deduplicate_uuids
from aiida.manage.manager import get_manager

manager = get_manager()
manager._load_backend(schema_check=False) # pylint: disable=protected-access

try:
messages = deduplicate_uuids(table=table, dry_run=not apply_patch)
except Exception as exception: # pylint: disable=broad-except
echo.echo_critical(f'integrity check failed: {str(exception)}')
else:
for message in messages:
echo.echo_report(message)

if apply_patch:
echo.echo_success('integrity patch completed')
else:
echo.echo_success('dry-run of integrity patch completed')
.. deprecated:: v2.0.0
"""


@verdi_database_integrity.command('detect-invalid-links')
@decorators.with_dbenv()
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'For remaining available integrity checks, use `verdi storage integrity` instead.\n'
)
def detect_invalid_links():
"""Scan the database for invalid links."""
from tabulate import tabulate

from aiida.manage.database.integrity.sql.links import INVALID_LINK_SELECT_STATEMENTS
from aiida.manage.manager import get_manager
"""Scan the database for invalid links.

integrity_violated = False

backend = get_manager().get_backend()

for check in INVALID_LINK_SELECT_STATEMENTS:

result = backend.execute_prepared_statement(check.sql, check.parameters)

if result:
integrity_violated = True
echo.echo_warning(f'{check.message}:\n')
echo.echo(tabulate(result, headers=check.headers))

if not integrity_violated:
echo.echo_success('no integrity violations detected')
else:
echo.echo_critical('one or more integrity violations detected')
.. deprecated:: v2.0.0
"""


@verdi_database_integrity.command('detect-invalid-nodes')
@decorators.with_dbenv()
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'For remaining available integrity checks, use `verdi storage integrity` instead.\n'
)
def detect_invalid_nodes():
"""Scan the database for invalid nodes."""
from tabulate import tabulate

from aiida.manage.database.integrity.sql.nodes import INVALID_NODE_SELECT_STATEMENTS
from aiida.manage.manager import get_manager

integrity_violated = False
"""Scan the database for invalid nodes.

backend = get_manager().get_backend()

for check in INVALID_NODE_SELECT_STATEMENTS:

result = backend.execute_prepared_statement(check.sql, check.parameters)

if result:
integrity_violated = True
echo.echo_warning(f'{check.message}:\n')
echo.echo(tabulate(result, headers=check.headers))

if not integrity_violated:
echo.echo_success('no integrity violations detected')
else:
echo.echo_critical('one or more integrity violations detected')
.. deprecated:: v2.0.0
"""


@verdi_database.command('summary')
@decorators.deprecated_command(
'This command has been deprecated and no longer has any effect. It will be removed soon from the CLI (in v2.1).\n'
'Please call `verdi storage info` instead.\n'
)
def database_summary():
"""Summarise the entities in the database."""
from aiida.cmdline import is_verbose
from aiida.orm import Comment, Computer, Group, Log, Node, QueryBuilder, User
data = {}

# User
query_user = QueryBuilder().append(User, project=['email'])
data['Users'] = {'count': query_user.count()}
if is_verbose():
data['Users']['emails'] = query_user.distinct().all(flat=True)

# Computer
query_comp = QueryBuilder().append(Computer, project=['label'])
data['Computers'] = {'count': query_comp.count()}
if is_verbose():
data['Computers']['labels'] = query_comp.distinct().all(flat=True)

# Node
count = QueryBuilder().append(Node).count()
data['Nodes'] = {'count': count}
if is_verbose():
node_types = QueryBuilder().append(Node, project=['node_type']).distinct().all(flat=True)
data['Nodes']['node_types'] = node_types
process_types = QueryBuilder().append(Node, project=['process_type']).distinct().all(flat=True)
data['Nodes']['process_types'] = [p for p in process_types if p]

# Group
query_group = QueryBuilder().append(Group, project=['type_string'])
data['Groups'] = {'count': query_group.count()}
if is_verbose():
data['Groups']['type_strings'] = query_group.distinct().all(flat=True)

# Comment
count = QueryBuilder().append(Comment).count()
data['Comments'] = {'count': count}

# Log
count = QueryBuilder().append(Log).count()
data['Logs'] = {'count': count}

echo.echo_dictionary(data, sort_keys=False, fmt='yaml')
"""Summarise the entities in the database.

.. deprecated:: v2.0.0
"""
37 changes: 26 additions & 11 deletions aiida/cmdline/commands/cmd_status.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ class ServiceStatus(enum.IntEnum):
@click.option('--no-rmq', is_flag=True, help='Do not check RabbitMQ status')
def verdi_status(print_traceback, no_rmq):
"""Print status of AiiDA services."""
# pylint: disable=broad-except,too-many-statements,too-many-branches
# pylint: disable=broad-except,too-many-statements,too-many-branches,too-many-locals,
from aiida import __version__
from aiida.cmdline.utils.daemon import delete_stale_pid_file, get_daemon_status
from aiida.common.utils import Capturing
Expand All @@ -68,16 +68,17 @@ def verdi_status(print_traceback, no_rmq):
print_status(ServiceStatus.UP, 'config', AIIDA_CONFIG_FOLDER)

manager = get_manager()
profile = manager.get_profile()

if profile is None:
print_status(ServiceStatus.WARNING, 'profile', 'no profile configured yet')
echo.echo_report('Configure a profile by running `verdi quicksetup` or `verdi setup`.')
return

try:
profile = manager.get_profile()

if profile is None:
print_status(ServiceStatus.WARNING, 'profile', 'no profile configured yet')
echo.echo_report('Configure a profile by running `verdi quicksetup` or `verdi setup`.')
return

print_status(ServiceStatus.UP, 'profile', profile.name)

except Exception as exc:
message = 'Unable to read AiiDA profile'
print_status(ServiceStatus.ERROR, 'profile', message, exception=exc, print_traceback=print_traceback)
Expand All @@ -95,21 +96,35 @@ def verdi_status(print_traceback, no_rmq):
print_status(ServiceStatus.UP, 'repository', repository_status)

# Getting the postgres status by trying to get a database cursor
database_data = [profile.database_name, profile.database_username, profile.database_hostname, profile.database_port]
backend_manager = manager.get_backend_manager()
dbgen = backend_manager.get_schema_generation_database()
dbver = backend_manager.get_schema_version_backend()
database_data = [
profile.database_name, dbgen, dbver, profile.database_username, profile.database_hostname, profile.database_port
]
try:
with override_log_level(): # temporarily suppress noisy logging
backend = manager.get_backend()
backend.cursor()

except IncompatibleDatabaseSchema:
message = 'Database schema version is incompatible with the code: run `verdi database migrate`.'
message = f'Database schema {dbgen} / {dbver} (generation/version) is incompatible with the code. '
message += 'Run `verdi database migrate` to solve this.'
print_status(ServiceStatus.DOWN, 'postgres', message)
exit_code = ExitCode.CRITICAL

except Exception as exc:
message = 'Unable to connect to database `{}` as {}@{}:{}'.format(*database_data)
message = 'Unable to connect to database `{}` with schema {} / {} (generation/version) as {}@{}:{}'.format(
*database_data
)
print_status(ServiceStatus.DOWN, 'postgres', message, exception=exc, print_traceback=print_traceback)
exit_code = ExitCode.CRITICAL

else:
print_status(ServiceStatus.UP, 'postgres', 'Connected to database `{}` as {}@{}:{}'.format(*database_data))
message = 'Connected to database `{}` with schema {} / {} (generation/version) as {}@{}:{}'.format(
*database_data
)
print_status(ServiceStatus.UP, 'postgres', message)

# Getting the rmq status
if not no_rmq:
Expand Down
Loading