From ba42640202be5c6f6311f075f11c041bc9558b44 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 13:09:39 +0100 Subject: [PATCH 1/7] Remove the download and snapshot tasks These tasks are coupled to flight-direct and have been moved into: https://github.com/alces-software/forge-packages/pull/23 --- lib/tasks/packages.rake | 96 ----------------------------------------- 1 file changed, 96 deletions(-) diff --git a/lib/tasks/packages.rake b/lib/tasks/packages.rake index a45814c..12ee288 100644 --- a/lib/tasks/packages.rake +++ b/lib/tasks/packages.rake @@ -7,25 +7,6 @@ require 'highline/import' require_relative File.join(ENV['FL_ROOT'], 'lib/flight_direct/version.rb') namespace :packages do - desc 'Downloads all the rake packages' - task download: :environment do - raise 'The ANVIL_UPSTREAM has not been set' unless ENV['ANVIL_UPSTREAM'] - raise 'The ANVIL_LOCAL_DIR has not been set' unless ENV['ANVIL_LOCAL_DIR'] - packages = JSON.parse( - Net::HTTP.get(URI("#{ENV['ANVIL_UPSTREAM']}/v1/packages")), - object_class: OpenStruct - ) - Parallel.map(packages.data, in_threads: 10) do |metadata| - uri = URI.parse(metadata.attributes.packageUrl) - puts "Downloading: #{uri.to_s}" - path = package_path(URI.unescape(uri.path)) - FileUtils.mkdir_p File.dirname(path) - File.open(path, "wb") do |save_file| - open(uri.to_s) { |line| save_file.write(line.read) } - end - end - end - desc 'Import packages from a local source' task import: :environment do raise 'The ANVIL_LOCAL_DIR has not been set' unless ENV['ANVIL_LOCAL_DIR'] @@ -42,62 +23,6 @@ namespace :packages do ERROR end - desc 'Download and import the packages' - task snapshot: :environment do - exit_if_db_exists('snapshot') - raise 'The ANVIL_BASE_URL has not been set' unless ENV['ANVIL_BASE_URL'] - ENV['ANVIL_UPSTREAM'] ||= 'https://forge-api.alces-flight.com' - ENV['ANVIL_LOCAL_DIR'] ||= File.expand_path(File.join( - File.dirname(__FILE__), '..', '..', 'public' - )) - - # Downloads the flight direct bootstrap script - puts 'Downloading FlightDirect bootstrap' - bootstrap_url = 'https://raw.githubusercontent.com/alces-software/flight-direct/master/scripts/bootstrap.sh' - bootstrap_path = File.join(ENV['ANVIL_LOCAL_DIR'], - 'flight-direct', - 'bootstrap.sh') - download(bootstrap_url, bootstrap_path) - - # Sets the anvil_url in the bootstrap script - bootstrap_content = File.read(bootstrap_path) - new_bootstrap_content = bootstrap_content.gsub( - /# anvil_url=/, "anvil_url=#{ENV['ANVIL_BASE_URL']}" - ) - FileUtils.rm(bootstrap_path) - File.write(bootstrap_path, new_bootstrap_content) - - # Downloads the installed version of Flight Direct from S3 - puts 'Downloading FlightDirect tarball' - fd_url = "https://s3-eu-west-1.amazonaws.com/flight-direct/releases/el7/flight-direct-#{FlightDirect::VERSION}.tar.gz" - fd_path = File.join(ENV['ANVIL_LOCAL_DIR'], - 'flight-direct/flight-direct.tar.gz') - download(fd_url, fd_path) - - # Downloads the git packages - ['clusterware-sessions', 'clusterware-storage', - 'gridware-packages-main', 'packager-base', 'gridware-depots' - ].each do |repo| - url = "https://github.com/alces-software/#{repo}.git" - source = "/tmp/repos/#{repo}" - target = File.join(ENV['ANVIL_LOCAL_DIR'], 'git', "#{repo}.tar.gz") - print `rm -rf #{source} #{target}` - print `mkdir -p #{File.dirname(target)}` - puts `git clone #{url} #{source}` - puts `tar --warning=no-file-changed -C #{source} -czf #{target} .` - end - # Renames packager-base to be the volatile repo - FileUtils.mv File.join(ENV['ANVIL_LOCAL_DIR'], 'git', 'packager-base.tar.gz'), - File.join(ENV['ANVIL_LOCAL_DIR'], 'git', 'gridware-packages-volatile.tar.gz') - - Rake::Task['db:setup'].invoke - puts 'Downloading packages...' - Rake::Task['packages:download'].invoke - puts 'Importing the packages...' - Rake::Task['packages:import'].invoke - puts 'Done' - end - def package_path(relative_path) File.join(ENV['ANVIL_LOCAL_DIR'], 'packages', relative_path) end @@ -107,19 +32,6 @@ namespace :packages do File.join(ENV['ANVIL_BASE_URL'], relative_path) end - def exit_if_db_exists(action) - ActiveRecord::Base.connection - $stderr.puts <<~ERROR.squish - `rake packages:#{action}` can not be ran once the db has been setup. - Either run `rake db:drop` to delete the database, or perform the - #{action} manually - ERROR - exit 1 - rescue ActiveRecord::NoDatabaseError - # The database doesn't exist so do nothing - return - end - def add_package_from_zip_path(zip_path) user = User.where(name: 'alces').first_or_create url = extract_package_url(zip_path) @@ -127,12 +39,4 @@ namespace :packages do user: user, package_url: url, file: zip_path ).save end - - def download(url, path) - FileUtils.mkdir_p File.dirname(path) - case io = open(url) - when StringIO then File.open(path, 'w') { |f| f.write(io.read) } - when Tempfile then FileUtils.mv(io.path, path) - end - end end From 4083c5188534e4302b541215f6ffb2ccfed1a8ab Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 14:33:01 +0100 Subject: [PATCH 2/7] Simplify the env var checks --- lib/tasks/packages.rake | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/tasks/packages.rake b/lib/tasks/packages.rake index 12ee288..35f7eaa 100644 --- a/lib/tasks/packages.rake +++ b/lib/tasks/packages.rake @@ -9,8 +9,9 @@ require_relative File.join(ENV['FL_ROOT'], 'lib/flight_direct/version.rb') namespace :packages do desc 'Import packages from a local source' task import: :environment do - raise 'The ANVIL_LOCAL_DIR has not been set' unless ENV['ANVIL_LOCAL_DIR'] - raise 'The ANVIL_BASE_URL has not been set' unless ENV['ANVIL_BASE_URL'] + ['ANVIL_LOCAL_DIR', 'ANVIL_BASE_URL'].each do |env| + raise "The #{env} has not been set" unless ENV[env] + end files = Dir[package_path('**/*.zip')] files.define_singleton_method(:delete_if_saveable) do self.delete_if { |f| add_package_from_zip_path(f) } From 352f2eb41224fb5f61d03ce50b8b29ea4f554bca Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 15:09:53 +0100 Subject: [PATCH 3/7] Remove the old flight-direct start script This script has been moved into flight-cache packages --- scripts/start-anvil.sh | 6 ------ 1 file changed, 6 deletions(-) delete mode 100644 scripts/start-anvil.sh diff --git a/scripts/start-anvil.sh b/scripts/start-anvil.sh deleted file mode 100644 index 0a733b7..0000000 --- a/scripts/start-anvil.sh +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/bash -source /etc/profile.d/flight-direct.sh -source $FL_ROOT/etc/runtime.sh -anvil_dir="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" -cd $anvil_dir -bundle exec rails server -p 80 -b 0.0.0.0 -e snapshot From d572779f70a0537d32d1246f413e84d6bb4fed98 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 15:33:50 +0100 Subject: [PATCH 4/7] Switch the `LOCAL_DIR` envvar to `IMPORT_DIR` The environment variable has been changed as anvil no longer controls the snapshot. Instead it only needs a means to add packages from a directory This does require messing around with setting the package URL correctly (as `packages` must be within its path). However setting the package_url is going to be changed at a future date --- lib/tasks/packages.rake | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/tasks/packages.rake b/lib/tasks/packages.rake index 35f7eaa..eea6746 100644 --- a/lib/tasks/packages.rake +++ b/lib/tasks/packages.rake @@ -9,7 +9,7 @@ require_relative File.join(ENV['FL_ROOT'], 'lib/flight_direct/version.rb') namespace :packages do desc 'Import packages from a local source' task import: :environment do - ['ANVIL_LOCAL_DIR', 'ANVIL_BASE_URL'].each do |env| + ['ANVIL_IMPORT_DIR', 'ANVIL_BASE_URL'].each do |env| raise "The #{env} has not been set" unless ENV[env] end files = Dir[package_path('**/*.zip')] @@ -25,12 +25,12 @@ namespace :packages do end def package_path(relative_path) - File.join(ENV['ANVIL_LOCAL_DIR'], 'packages', relative_path) + File.join(ENV['ANVIL_IMPORT_DIR'], relative_path) end def extract_package_url(absolute_path) - relative_path = absolute_path&.sub(ENV['ANVIL_LOCAL_DIR'], '') - File.join(ENV['ANVIL_BASE_URL'], relative_path) + relative_path = absolute_path&.sub(ENV['ANVIL_IMPORT_DIR'], '') + File.join(ENV['ANVIL_BASE_URL'], 'packages', relative_path) end def add_package_from_zip_path(zip_path) From 60db4b08d93848d9a27558766baeeed61866ee25 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 15:59:45 +0100 Subject: [PATCH 5/7] Update the README --- README.md | 98 ++++++++++++++----------------------------------------- 1 file changed, 25 insertions(+), 73 deletions(-) diff --git a/README.md b/README.md index 7a7356a..0d931ec 100644 --- a/README.md +++ b/README.md @@ -47,81 +47,33 @@ this isn't very important right now. In general we need to think about versionin content items such as Gridware and customizers, and also in terms of Clusterware version compatibilities.) -# Creating a Database Snapshot +# Importing local packages into the database +It is possible to import packages into the database from a directory. The +directory is searched recursively for all `.zip` files. The packages are +directly added to the database without going through the upload and thus +do not require sign in credentials. -The database snapshot can be triggered manually using: -``` -# cd ./anvil -# rake packages:snapshot -Which IP/domain are the packages hosted on? -> www.example.com -``` - -Running this command with no other system configurations sets up the -default server, however it still needs to be told which ip/ domain the -packages will be hosted on. See environment setup for more details - -NOTE: The answer should not include the protocol as it will default to -`http`. However the underlining `ANVIL_BASE_URL` needs to be fully -qualified including the protocol. - -## Environment Setup - -Running the `snapshot` rake command does not alter your environment setup, -it does however use the following environment variables internally - -``` -ANVIL_LOCAL_DIR: The base directory for the download. Typically this would - be hosted on the rails in built `public` directory or - hosted by apache/ enginx. Anvil will store the files in - a `packages` sub directory - DEFAULT: /path/to/anvil/public - -ANVIL_UPSTREAM: The upstream anvil database to take the snapshot on. It - defaults to the main production database. - DEFAULT: https://forge-api.alces-flight.com - -ANVIL_BASE_URL: A fully qualified URL (inc. protocol) to where the - packages will be hosted. This will be stored in the db - with the package metadata. Thus it does not need to be - permanetly set within the environment. There is no default -``` - -## Drop DB and Running the Snapshot Manually - -If the above environment variable are set, then `rake package:snapshot` -will automatically download and import the database in a single set. - -The automatic snapshot will not work if the database already exists as -it can not recover from any errors. Instead the snapshot should be -preformed step by step using the commands bellow. - -It does require the above environment variables to be set manually as well, -refer to `lib/rake/packages.rb` for further details. - -``` -rake snapshot:download -rake snashot:import -``` - -Alternatively the database could be dropped with: -``` -rake db:drop -``` - -## Running the Server - -You are now ready to start the `anvil` rails server with the command bellow. -This will likely launch in development mode if no further configurations are -made. This means the `public` directory should be statically served without -any futher configurations. - -However there is no reason why the packages need to be hosted on this -machine, the packages can be hosted anywhere as long as the -`ANVIL_BASE_URL` was set correctly when the import occurred. -NOTE: Changes to the base url after the fact will be ignored +In order for the packages to be added correctly, the base url and package +directory must be set as environment variables: ``` -bin/rails server -p 80 -b 0.0.0.0 +ANVIL_BASE_URL: A fully qualified URL (inc. protocol) to where the + packages will be hosted. This will be stored in the db + with the package metadata. Thus it does not need to be + permanetly set within the environment. + +ANVIL_IMPORT_DIR: The directory the packages are imported from. The import + command does not concern itself with the hosting of the + packages. It assumes the packages will be found at: + http://$ANVIL_BASE_URL/packages/ + + Typically packages will be hosted from within the anvil + public directory. Thus ANVIL_IMPORT_DIR would normally + be set to: //anvil/public/packages + + However a different static file server could be used + (e.g. nginx). In this case the import dir should be set + accordingly. It is the responsibility of the user to + ensure the package is reachable at the above address. ``` From 4829e987e42772932734600f28f9609668220652 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 16:01:02 +0100 Subject: [PATCH 6/7] Add the command into the README --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 0d931ec..3f77ce5 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,11 @@ directory is searched recursively for all `.zip` files. The packages are directly added to the database without going through the upload and thus do not require sign in credentials. +The import is preformed use rake: +``` +rake packages:import +``` + In order for the packages to be added correctly, the base url and package directory must be set as environment variables: From 1bb4ab5d30ee03c769ddc387ac2a0c906d679885 Mon Sep 17 00:00:00 2001 From: william Date: Thu, 18 Oct 2018 16:22:40 +0100 Subject: [PATCH 7/7] Remove the `anvil` forge-package A separate forge-package is proving to be problematic as requires knowledge about flight direct. Instead the `flight-cache` package can build `anvil` into itself directly. This also has the added benefit of locking the anvil version used by each `flight-cache` package --- package/build.sh | 41 ----------------------------------------- package/install.sh | 4 ---- package/metadata.json | 10 ---------- 3 files changed, 55 deletions(-) delete mode 100755 package/build.sh delete mode 100755 package/install.sh delete mode 100644 package/metadata.json diff --git a/package/build.sh b/package/build.sh deleted file mode 100755 index 02946df..0000000 --- a/package/build.sh +++ /dev/null @@ -1,41 +0,0 @@ -#!/bin/bash - -set -e - -package_name='anvil' - -if [ -f ./${package_name}.zip ]; then - echo "Replacing existing ${package_name}.zip in this directory" - rm ./${package_name}.zip -fi - -# Determines the metadata path -metadata="$(pwd)/metadata.json" - -# Determines the version number from the metadata -gem install json -version=$(cat << EOF | ruby -require 'json' -metadata = JSON.parse(File.read('$metadata')) -print metadata['attributes']['version'] -EOF -) - -temp_dir=$(mktemp -d /tmp/${package_name}-build-XXXXX) - -cp -r * $temp_dir - -anvil_dir="$temp_dir/data/opt/anvil" -mkdir -p $anvil_dir - -pushd .. > /dev/null -git archive $version | tar -x -C "${temp_dir}"/data/opt/anvil -popd > /dev/null - -pushd "${temp_dir}" > /dev/null -zip -r ${package_name}.zip * -popd > /dev/null - -mv "${temp_dir}"/${package_name}.zip . - -rm -rf "${temp_dir}" diff --git a/package/install.sh b/package/install.sh deleted file mode 100755 index 7d9dac4..0000000 --- a/package/install.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash - -cp -R data/* "${FL_ROOT}" - diff --git a/package/metadata.json b/package/metadata.json deleted file mode 100644 index 3b8c159..0000000 --- a/package/metadata.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "type": "packages", - "attributes": { - "name": "anvil", - "version": "1.0.0", - "summary": "Contains the anvil source code", - "dependencies": [ - ] - } -}