From f0e1cc4e9985f00f661cd666ac8331535f46a22a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Fri, 6 Aug 2021 10:49:15 +0800 Subject: [PATCH 01/27] feat: Implement basic credential --- go.mod | 1 + go.sum | 2 ++ utils.go | 16 ++++++++++++++++ 3 files changed, 19 insertions(+) diff --git a/go.mod b/go.mod index 56b813c..49530ae 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/beyondstorage/go-service-ftp go 1.15 require ( + github.com/beyondstorage/go-credential v0.0.0-20210804082910-7c63d94f2826 github.com/beyondstorage/go-endpoint v1.1.0 github.com/beyondstorage/go-integration-test/v4 v4.2.0 github.com/beyondstorage/go-storage/v4 v4.4.0 diff --git a/go.sum b/go.sum index 4b2b1f8..4a22bb5 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,7 @@ github.com/Xuanwo/templateutils v0.1.0 h1:WpkWOqQtIQ2vAIpJLa727DdN8WtxhUkkbDGa6UhntJY= github.com/Xuanwo/templateutils v0.1.0/go.mod h1:OdE0DJ+CJxDBq6psX5DPV+gOZi8bhuHuVUpPCG++Wb8= +github.com/beyondstorage/go-credential v0.0.0-20210804082910-7c63d94f2826 h1:9cHyxD1JKUHWzWtDgaoLvzsR1JeEwguuL9IYSN7tk3k= +github.com/beyondstorage/go-credential v0.0.0-20210804082910-7c63d94f2826/go.mod h1:A7WYzKKk479gERCs5/5jWi0kNN24bqeMjD8IJik4JBA= github.com/beyondstorage/go-endpoint v1.1.0 h1:cpjmQdrAMyaLoT161NIFU/eXcsuMI3xViycid5/mBZg= github.com/beyondstorage/go-endpoint v1.1.0/go.mod h1:P2hknaGrziOJJKySv/XnAiVw/d3v12/LZu2gSxEx4nM= github.com/beyondstorage/go-integration-test/v4 v4.2.0 h1:h2+SLmlDqjfBg+NzVcDr6VCmcD7I2xG+mqMzDlaCG+0= diff --git a/utils.go b/utils.go index 275995e..d86bc48 100644 --- a/utils.go +++ b/utils.go @@ -10,6 +10,7 @@ import ( "github.com/jlaffaye/ftp" mime "github.com/qingstor/go-mime" + credential "github.com/beyondstorage/go-credential" endpoint "github.com/beyondstorage/go-endpoint" ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/services" @@ -81,6 +82,21 @@ func newStorager(pairs ...types.Pair) (store *Storage, err error) { store.workDir = opt.WorkDir } + if opt.HasCredential { + cp, err := credential.Parse(opt.Credential) + if err != nil { + return nil, err + } + switch cp.Protocol() { + case credential.ProtocolBasic: + user, pass := cp.Basic() + store.password = pass + store.user = user + default: + return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} + } + } + err = store.connect() if err != nil { return nil, err From cc46bbeac1053723d2453e878794d4aa1a1a3a66 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Sat, 7 Aug 2021 19:35:15 +0800 Subject: [PATCH 02/27] Create integration-test.yml --- .github/workflows/integration-test.yml | 210 +++++++++++++++++++++++++ 1 file changed, 210 insertions(+) create mode 100644 .github/workflows/integration-test.yml diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml new file mode 100644 index 0000000..9996876 --- /dev/null +++ b/.github/workflows/integration-test.yml @@ -0,0 +1,210 @@ +name: "Integration Test" + +on: [push, pull_request] + +jobs: + integration_test: + name: Integration Test + runs-on: ${{ matrix.os }} + + strategy: + matrix: + go: ["1.15", "1.16"] + os: [ubuntu-latest] + + steps: + - name: Set up Go 1.x + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go }} + + - name: Run vsftpd Server + run: | + sudo apt-get install vsftpd + sudo service vsftpd start + #sudo sed -e 's/anonymous_enable=YES/anonymous_enable=No/g' /etc/vsftpd.conf + sudo cat /etc/vsftpd.conf || /etc/vsftpd.conf <<- EOF + # Example config file /etc/vsftpd.conf + # + # The default compiled in settings are fairly paranoid. This sample file + # loosens things up a bit, to make the ftp daemon more usable. + # Please see vsftpd.conf.5 for all compiled in defaults. + # + # READ THIS: This example file is NOT an exhaustive list of vsftpd options. + # Please read the vsftpd.conf.5 manual page to get a full idea of vsftpd's + # capabilities. + # + # + # Run standalone? vsftpd can run either from an inetd or as a standalone + # daemon started from an initscript. + listen=NO + #listen_port=21 + # + # This directive enables listening on IPv6 sockets. By default, listening + # on the IPv6 "any" address (::) will accept connections from both IPv6 + # and IPv4 clients. It is not necessary to listen on *both* IPv4 and IPv6 + # sockets. If you want that (perhaps because you want to listen on specific + # addresses) then you must run two copies of vsftpd with two configuration + # files. + listen_ipv6=YES + # + # Allow anonymous FTP? (Disabled by default). + anonymous_enable=NO + # + # Uncomment this to allow local users to log in. + local_enable=YES + # + # Uncomment this to enable any form of FTP write command. + write_enable=YES + # + # Default umask for local users is 077. You may wish to change this to 022, + # if your users expect that (022 is used by most other ftpd's) + local_umask=777 + # + # Uncomment this to allow the anonymous FTP user to upload files. This only + # has an effect if the above global write enable is activated. Also, you will + # obviously need to create a directory writable by the FTP user. + anon_upload_enable=YES + # + # Uncomment this if you want the anonymous FTP user to be able to create + # new directories. + anon_mkdir_write_enable=YES + # + # Activate directory messages - messages given to remote users when they + # go into a certain directory. + dirmessage_enable=YES + # + # If enabled, vsftpd will display directory listings with the time + # in your local time zone. The default is to display GMT. The + # times returned by the MDTM FTP command are also affected by this + # option. + use_localtime=YES + # + # Activate logging of uploads/downloads. + xferlog_enable=YES + # + # Make sure PORT transfer connections originate from port 20 (ftp-data). + connect_from_port_20=YES + # + # If you want, you can arrange for uploaded anonymous files to be owned by + # a different user. Note! Using "root" for uploaded files is not + # recommended! + #chown_uploads=YES + #chown_username=whoever + # + # You may override where the log file goes if you like. The default is shown + # below. + #xferlog_file=/var/log/vsftpd.log + # + # If you want, you can have your log file in standard ftpd xferlog format. + # Note that the default log file location is /var/log/xferlog in this case. + #xferlog_std_format=YES + # + # You may change the default value for timing out an idle session. + #idle_session_timeout=600 + # + # You may change the default value for timing out a data connection. + #data_connection_timeout=120 + # + # It is recommended that you define on your system a unique user which the + # ftp server can use as a totally isolated and unprivileged user. + #nopriv_user=ftpsecure + # + # Enable this and the server will recognise asynchronous ABOR requests. Not + # recommended for security (the code is non-trivial). Not enabling it, + # however, may confuse older FTP clients. + #async_abor_enable=YES + # + # By default the server will pretend to allow ASCII mode but in fact ignore + # the request. Turn on the below options to have the server actually do ASCII + # mangling on files when in ASCII mode. + # Beware that on some FTP servers, ASCII support allows a denial of service + # attack (DoS) via the command "SIZE /big/file" in ASCII mode. vsftpd + # predicted this attack and has always been safe, reporting the size of the + # raw file. + # ASCII mangling is a horrible feature of the protocol. + #ascii_upload_enable=YES + #ascii_download_enable=YES + # + # You may fully customise the login banner string: + #ftpd_banner=Welcome to blah FTP service. + # + # You may specify a file of disallowed anonymous e-mail addresses. Apparently + # useful for combatting certain DoS attacks. + #deny_email_enable=YES + # (default follows) + #banned_email_file=/etc/vsftpd.banned_emails + # + # You may restrict local users to their home directories. See the FAQ for + # the possible risks in this before using chroot_local_user or + # chroot_list_enable below. + chroot_local_user=YES + user_sub_token=$USER + local_root=/home/$USER/ftp + # + # You may specify an explicit list of local users to chroot() to their home + # directory. If chroot_local_user is YES, then this list becomes a list of + # users to NOT chroot(). + # (Warning! chroot'ing can be very dangerous. If using chroot, make sure that + # the user does not have write access to the top level directory within the + # chroot) + #chroot_local_user=YES + #chroot_list_enable=YES + # (default follows) + #chroot_list_file=/etc/vsftpd.chroot_list + # + # You may activate the "-R" option to the builtin ls. This is disabled by + # default to avoid remote users being able to cause excessive I/O on large + # sites. However, some broken FTP clients such as "ncftp" and "mirror" assume + # the presence of the "-R" option, so there is a strong case for enabling it. + #ls_recurse_enable=YES + # + # Customization + # + # Some of vsftpd's settings don't fit the filesystem layout by + # default. + # + # This option should be the name of a directory which is empty. Also, the + # directory should not be writable by the ftp user. This directory is used + # as a secure chroot() jail at times vsftpd does not require filesystem + # access. + #secure_chroot_dir=/var/run/vsftpd/empty + # + # This string is the name of the PAM service vsftpd will use. + pam_service_name=vsftpd + # + # This option specifies the location of the RSA certificate to use for SSL + # encrypted connections. + rsa_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem + rsa_private_key_file=/etc/ssl/private/ssl-cert-snakeoil.key + ssl_enable=NO + + # + # Uncomment this to indicate that vsftpd use a utf8 filesystem. + utf8_filesystem=YES + userlist_enable=YES + userlist_file=/etc/vsftpd.userlist + userlist_deny=NO + EOF + sudo service vsftpd restart + sudo adduser testuser < Date: Sat, 7 Aug 2021 19:43:08 +0800 Subject: [PATCH 03/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9996876..ceaf729 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -187,9 +187,7 @@ jobs: userlist_deny=NO EOF sudo service vsftpd restart - sudo adduser testuser < Date: Sat, 7 Aug 2021 19:43:59 +0800 Subject: [PATCH 04/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ceaf729..4877612 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -207,5 +207,5 @@ jobs: - name: Prepare ENV run: mv Makefile.env.example Makefile.env - - name: Test + - name: Integration Test run: make integration_test From e0ae1946607965761867da4e9109ecaaa4b64b33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Sat, 7 Aug 2021 19:48:30 +0800 Subject: [PATCH 05/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4877612..0148087 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -187,7 +187,8 @@ jobs: userlist_deny=NO EOF sudo service vsftpd restart - echo password123:? | sudo adduser testuser --stdin + sudo adduser testuser + echo password123:? | sudo passwd testuser --stdin echo "testuser" | sudo tee -a /etc/vsftpd.userlist sudo mkdir /home/testuser/ftp sudo chown nobody:nogroup /home/testuser/ftp From 558ef01a7577318fd1d8e430e71da81d71e895f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Sat, 7 Aug 2021 19:50:14 +0800 Subject: [PATCH 06/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 0148087..e473777 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -188,7 +188,7 @@ jobs: EOF sudo service vsftpd restart sudo adduser testuser - echo password123:? | sudo passwd testuser --stdin + echo 'password123:?' | sudo passwd testuser -stdin echo "testuser" | sudo tee -a /etc/vsftpd.userlist sudo mkdir /home/testuser/ftp sudo chown nobody:nogroup /home/testuser/ftp From 365a2af86e0ed2a2553f5bc113370c4db73c48a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Sat, 7 Aug 2021 19:57:06 +0800 Subject: [PATCH 07/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index e473777..9359e74 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -21,6 +21,7 @@ jobs: - name: Run vsftpd Server run: | sudo apt-get install vsftpd + sudo apt-get install expect sudo service vsftpd start #sudo sed -e 's/anonymous_enable=YES/anonymous_enable=No/g' /etc/vsftpd.conf sudo cat /etc/vsftpd.conf || /etc/vsftpd.conf <<- EOF @@ -188,7 +189,10 @@ jobs: EOF sudo service vsftpd restart sudo adduser testuser - echo 'password123:?' | sudo passwd testuser -stdin + spawn sudo passwd testuser + send "password123:?" + interact + expect eof echo "testuser" | sudo tee -a /etc/vsftpd.userlist sudo mkdir /home/testuser/ftp sudo chown nobody:nogroup /home/testuser/ftp From 9758f5382d16ab9936bcdc6944f823058f1d61f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Sat, 7 Aug 2021 20:00:33 +0800 Subject: [PATCH 08/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9359e74..59ee787 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -189,10 +189,12 @@ jobs: EOF sudo service vsftpd restart sudo adduser testuser - spawn sudo passwd testuser - send "password123:?" - interact - expect eof + expect -c "\n\ + spawn sudo passwd testuser\n\ + send "password123:?"\n\ + interact\n\ + expect eof\n\ + " echo "testuser" | sudo tee -a /etc/vsftpd.userlist sudo mkdir /home/testuser/ftp sudo chown nobody:nogroup /home/testuser/ftp From 7b28a8e69b1537aa40e0a91bac880dded00331cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:24:33 +0800 Subject: [PATCH 09/27] fix: Add fallback check for list --- .github/workflows/integration-test.yml | 186 +------------------------ storage.go | 14 +- tests/utils_test.go | 5 +- 3 files changed, 16 insertions(+), 189 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 59ee787..31a7ec9 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -18,192 +18,8 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Run vsftpd Server + - name: Run ftp Server run: | - sudo apt-get install vsftpd - sudo apt-get install expect - sudo service vsftpd start - #sudo sed -e 's/anonymous_enable=YES/anonymous_enable=No/g' /etc/vsftpd.conf - sudo cat /etc/vsftpd.conf || /etc/vsftpd.conf <<- EOF - # Example config file /etc/vsftpd.conf - # - # The default compiled in settings are fairly paranoid. This sample file - # loosens things up a bit, to make the ftp daemon more usable. - # Please see vsftpd.conf.5 for all compiled in defaults. - # - # READ THIS: This example file is NOT an exhaustive list of vsftpd options. - # Please read the vsftpd.conf.5 manual page to get a full idea of vsftpd's - # capabilities. - # - # - # Run standalone? vsftpd can run either from an inetd or as a standalone - # daemon started from an initscript. - listen=NO - #listen_port=21 - # - # This directive enables listening on IPv6 sockets. By default, listening - # on the IPv6 "any" address (::) will accept connections from both IPv6 - # and IPv4 clients. It is not necessary to listen on *both* IPv4 and IPv6 - # sockets. If you want that (perhaps because you want to listen on specific - # addresses) then you must run two copies of vsftpd with two configuration - # files. - listen_ipv6=YES - # - # Allow anonymous FTP? (Disabled by default). - anonymous_enable=NO - # - # Uncomment this to allow local users to log in. - local_enable=YES - # - # Uncomment this to enable any form of FTP write command. - write_enable=YES - # - # Default umask for local users is 077. You may wish to change this to 022, - # if your users expect that (022 is used by most other ftpd's) - local_umask=777 - # - # Uncomment this to allow the anonymous FTP user to upload files. This only - # has an effect if the above global write enable is activated. Also, you will - # obviously need to create a directory writable by the FTP user. - anon_upload_enable=YES - # - # Uncomment this if you want the anonymous FTP user to be able to create - # new directories. - anon_mkdir_write_enable=YES - # - # Activate directory messages - messages given to remote users when they - # go into a certain directory. - dirmessage_enable=YES - # - # If enabled, vsftpd will display directory listings with the time - # in your local time zone. The default is to display GMT. The - # times returned by the MDTM FTP command are also affected by this - # option. - use_localtime=YES - # - # Activate logging of uploads/downloads. - xferlog_enable=YES - # - # Make sure PORT transfer connections originate from port 20 (ftp-data). - connect_from_port_20=YES - # - # If you want, you can arrange for uploaded anonymous files to be owned by - # a different user. Note! Using "root" for uploaded files is not - # recommended! - #chown_uploads=YES - #chown_username=whoever - # - # You may override where the log file goes if you like. The default is shown - # below. - #xferlog_file=/var/log/vsftpd.log - # - # If you want, you can have your log file in standard ftpd xferlog format. - # Note that the default log file location is /var/log/xferlog in this case. - #xferlog_std_format=YES - # - # You may change the default value for timing out an idle session. - #idle_session_timeout=600 - # - # You may change the default value for timing out a data connection. - #data_connection_timeout=120 - # - # It is recommended that you define on your system a unique user which the - # ftp server can use as a totally isolated and unprivileged user. - #nopriv_user=ftpsecure - # - # Enable this and the server will recognise asynchronous ABOR requests. Not - # recommended for security (the code is non-trivial). Not enabling it, - # however, may confuse older FTP clients. - #async_abor_enable=YES - # - # By default the server will pretend to allow ASCII mode but in fact ignore - # the request. Turn on the below options to have the server actually do ASCII - # mangling on files when in ASCII mode. - # Beware that on some FTP servers, ASCII support allows a denial of service - # attack (DoS) via the command "SIZE /big/file" in ASCII mode. vsftpd - # predicted this attack and has always been safe, reporting the size of the - # raw file. - # ASCII mangling is a horrible feature of the protocol. - #ascii_upload_enable=YES - #ascii_download_enable=YES - # - # You may fully customise the login banner string: - #ftpd_banner=Welcome to blah FTP service. - # - # You may specify a file of disallowed anonymous e-mail addresses. Apparently - # useful for combatting certain DoS attacks. - #deny_email_enable=YES - # (default follows) - #banned_email_file=/etc/vsftpd.banned_emails - # - # You may restrict local users to their home directories. See the FAQ for - # the possible risks in this before using chroot_local_user or - # chroot_list_enable below. - chroot_local_user=YES - user_sub_token=$USER - local_root=/home/$USER/ftp - # - # You may specify an explicit list of local users to chroot() to their home - # directory. If chroot_local_user is YES, then this list becomes a list of - # users to NOT chroot(). - # (Warning! chroot'ing can be very dangerous. If using chroot, make sure that - # the user does not have write access to the top level directory within the - # chroot) - #chroot_local_user=YES - #chroot_list_enable=YES - # (default follows) - #chroot_list_file=/etc/vsftpd.chroot_list - # - # You may activate the "-R" option to the builtin ls. This is disabled by - # default to avoid remote users being able to cause excessive I/O on large - # sites. However, some broken FTP clients such as "ncftp" and "mirror" assume - # the presence of the "-R" option, so there is a strong case for enabling it. - #ls_recurse_enable=YES - # - # Customization - # - # Some of vsftpd's settings don't fit the filesystem layout by - # default. - # - # This option should be the name of a directory which is empty. Also, the - # directory should not be writable by the ftp user. This directory is used - # as a secure chroot() jail at times vsftpd does not require filesystem - # access. - #secure_chroot_dir=/var/run/vsftpd/empty - # - # This string is the name of the PAM service vsftpd will use. - pam_service_name=vsftpd - # - # This option specifies the location of the RSA certificate to use for SSL - # encrypted connections. - rsa_cert_file=/etc/ssl/certs/ssl-cert-snakeoil.pem - rsa_private_key_file=/etc/ssl/private/ssl-cert-snakeoil.key - ssl_enable=NO - - # - # Uncomment this to indicate that vsftpd use a utf8 filesystem. - utf8_filesystem=YES - userlist_enable=YES - userlist_file=/etc/vsftpd.userlist - userlist_deny=NO - EOF - sudo service vsftpd restart - sudo adduser testuser - expect -c "\n\ - spawn sudo passwd testuser\n\ - send "password123:?"\n\ - interact\n\ - expect eof\n\ - " - echo "testuser" | sudo tee -a /etc/vsftpd.userlist - sudo mkdir /home/testuser/ftp - sudo chown nobody:nogroup /home/testuser/ftp - sudo chmod a-w /home/testuser/ftp - sudo ls -al /home/testuser/ftp - sudo mkdir /home/testuser/ftp/upload - sudo chown testuser:testuser /home/testuser/ftp/upload - sudo ls -al /home/testuser/ftp - #echo "vsftpd test file" | sudo tee /home/testuser/ftp/upload/test.txt - name: Checkout repository uses: actions/checkout@v2 diff --git a/storage.go b/storage.go index 3dbf778..fdd19ab 100644 --- a/storage.go +++ b/storage.go @@ -124,9 +124,19 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o rp := s.getAbsPath(path) fl, err := s.connection.List(rp) if err != nil { - return nil, err + flst, err := s.connection.List(filepath.Dir(rp)) + if err != nil { + return nil, err + } + for i := range flst { + if filepath.Base(rp) == flst[i].Name { + fl = []*ftp.Entry{flst[i]} + break + } + } + } - if 0 == len(fl) { + if len(fl) == 0 { return nil, services.ErrObjectNotExist } var fe *ftp.Entry = fl[0] diff --git a/tests/utils_test.go b/tests/utils_test.go index 0a0b203..2eac385 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -3,14 +3,15 @@ package tests import ( "testing" - ftp "github.com/beyondstorage/go-service-ftp" + _ "github.com/beyondstorage/go-service-ftp" + "github.com/beyondstorage/go-storage/v4/services" "github.com/beyondstorage/go-storage/v4/types" ) func initTest(t *testing.T) (store types.Storager) { t.Log("Setup test for ftp") - store, err := ftp.NewStorager() + store, err := services.NewStoragerFromString("ftp://?endpoint=tcp:127.0.0.1:2121&credential=basic:user:password") if err != nil { t.Errorf("create storager: %v", err) } From 9d6440d956a50b16687915cac05af2c88b513a4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:50:32 +0800 Subject: [PATCH 10/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 43 ++++++++++++++++++++++++-- 1 file changed, 40 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 31a7ec9..063368f 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -18,16 +18,53 @@ jobs: with: go-version: ${{ matrix.go }} - - name: Run ftp Server + - name: Download ftpserver release + uses: robinraju/release-downloader@v1.1 + with: + repository: fclairamb/ftpserver + tag: v0.10.0 + fileName: ftpserver-linux-amd64 + + - name: Prepare config file run: | - + cat ftpserver.json || ftpserver.json <<- EOF + { + "version": 1, + "accesses": [ + { + "sync_and_delete": { + "enable": true + }, + "user": "user", + "pass": "password", + "fs": "os", + "params": { + "basePath": "./tmp" + } + } + ], + "passive_transfer_port_range": { + "start": 2122, + "end": 2130 + } + } + EOF + + - name: Create test dir + run: mkdir tmp + + - name: Start ftp server + run: | + chmod 777 ftpserver-linux-amd64 + ./ftpserver-linux-amd64 + - name: Checkout repository uses: actions/checkout@v2 - name: Build run: make build - - name: Prepare ENV + - name: Prepare env run: mv Makefile.env.example Makefile.env - name: Integration Test From a8479b1be31e632528701e15529a25dd0558d3c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:51:45 +0800 Subject: [PATCH 11/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 063368f..4671520 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,7 +27,7 @@ jobs: - name: Prepare config file run: | - cat ftpserver.json || ftpserver.json <<- EOF + cat ftpserver.json | ftpserver.json <<- EOF { "version": 1, "accesses": [ @@ -47,8 +47,8 @@ jobs: "start": 2122, "end": 2130 } - } - EOF + } + EOF - name: Create test dir run: mkdir tmp From c3c9753b0700972b735a71f3317d158d785a80f5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:52:48 +0800 Subject: [PATCH 12/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 4671520..6bc484c 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -47,8 +47,8 @@ jobs: "start": 2122, "end": 2130 } - } - EOF + } + EOF - name: Create test dir run: mkdir tmp From 6ddc53e9bbc13f074486cfd4a48c75a174a760fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:53:37 +0800 Subject: [PATCH 13/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 6bc484c..644f487 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,6 +27,7 @@ jobs: - name: Prepare config file run: | + touch ftpserver.json cat ftpserver.json | ftpserver.json <<- EOF { "version": 1, From 6378205c4099abb1b2918b8cb5536a5f31bcbe1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:57:03 +0800 Subject: [PATCH 14/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 644f487..ec60959 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,7 +27,7 @@ jobs: - name: Prepare config file run: | - touch ftpserver.json + touch "ftpserver.json" cat ftpserver.json | ftpserver.json <<- EOF { "version": 1, From e4c2ca0108073cedc99cfab143e632026dbbcc61 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 10:59:31 +0800 Subject: [PATCH 15/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index ec60959..02ffa39 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -18,6 +18,9 @@ jobs: with: go-version: ${{ matrix.go }} + - name: Checkout repository + uses: actions/checkout@v2 + - name: Download ftpserver release uses: robinraju/release-downloader@v1.1 with: @@ -27,7 +30,7 @@ jobs: - name: Prepare config file run: | - touch "ftpserver.json" + > ftpserver.json cat ftpserver.json | ftpserver.json <<- EOF { "version": 1, @@ -59,9 +62,6 @@ jobs: chmod 777 ftpserver-linux-amd64 ./ftpserver-linux-amd64 - - name: Checkout repository - uses: actions/checkout@v2 - - name: Build run: make build From 670058fb57cb50af22dd7fbe02b97b75de1b9e95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:01:12 +0800 Subject: [PATCH 16/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 02ffa39..2e9f853 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -28,9 +28,12 @@ jobs: tag: v0.10.0 fileName: ftpserver-linux-amd64 + - name: Create config file + run: \> ftpserver.json + - name: Prepare config file run: | - > ftpserver.json + cat ftpserver.json | ftpserver.json <<- EOF { "version": 1, From 045ff611a54b41fc58c18d65114d7240566340e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:02:17 +0800 Subject: [PATCH 17/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 2e9f853..8bf6e7a 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -29,11 +29,10 @@ jobs: fileName: ftpserver-linux-amd64 - name: Create config file - run: \> ftpserver.json + run: touch ftpserver.json - name: Prepare config file run: | - cat ftpserver.json | ftpserver.json <<- EOF { "version": 1, From 839db93ec8269c71c8e75e2bd7964e8ad36ad852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:03:45 +0800 Subject: [PATCH 18/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 8bf6e7a..afc7904 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -33,7 +33,7 @@ jobs: - name: Prepare config file run: | - cat ftpserver.json | ftpserver.json <<- EOF + cat > ftpserver.json <<- EOF { "version": 1, "accesses": [ From cfc24fa9011c674e93e825bf5925666e851f6ed8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:07:57 +0800 Subject: [PATCH 19/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index afc7904..49c3765 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,7 +27,10 @@ jobs: repository: fclairamb/ftpserver tag: v0.10.0 fileName: ftpserver-linux-amd64 - + + - name: Install + run: sudo apt install screen + - name: Create config file run: touch ftpserver.json @@ -62,7 +65,7 @@ jobs: - name: Start ftp server run: | chmod 777 ftpserver-linux-amd64 - ./ftpserver-linux-amd64 + screen -S ftpserver ./ftpserver-linux-amd64 - name: Build run: make build From 01689370bd1315857015f5b9379ac62240de1a2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:11:28 +0800 Subject: [PATCH 20/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 49c3765..de31baa 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,6 +65,7 @@ jobs: - name: Start ftp server run: | chmod 777 ftpserver-linux-amd64 + script /dev/null screen -S ftpserver ./ftpserver-linux-amd64 - name: Build From 120e477b0c39d3a3fe6c94198278c11685f5847d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:14:03 +0800 Subject: [PATCH 21/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index de31baa..fd88c63 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,8 +65,8 @@ jobs: - name: Start ftp server run: | chmod 777 ftpserver-linux-amd64 - script /dev/null screen -S ftpserver ./ftpserver-linux-amd64 + script /dev/null - name: Build run: make build From 0ad807338429ca1c68645c007cd4d3561a0b99e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:16:53 +0800 Subject: [PATCH 22/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index fd88c63..f77ac15 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,8 +65,8 @@ jobs: - name: Start ftp server run: | chmod 777 ftpserver-linux-amd64 - screen -S ftpserver ./ftpserver-linux-amd64 - script /dev/null + screen -dS ftpserver ./ftpserver-linux-amd64 + - name: Build run: make build From 328535b983da637696c99151d9c632aa9f599013 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?npofsi=20=7C=20N/P=E7=A1=85?= Date: Sun, 8 Aug 2021 11:18:55 +0800 Subject: [PATCH 23/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index f77ac15..f8677f7 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -65,7 +65,7 @@ jobs: - name: Start ftp server run: | chmod 777 ftpserver-linux-amd64 - screen -dS ftpserver ./ftpserver-linux-amd64 + screen -dmS ftpserver ./ftpserver-linux-amd64 - name: Build From 48d7b348c79bb3260a47849719c3c318a4f51a00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Mon, 9 Aug 2021 18:45:19 +0800 Subject: [PATCH 24/27] update: Integration test --- .github/workflows/integration-test.yml | 46 ++++++-------------------- Makefile.env.example | 4 ++- tests/README.md | 2 ++ tests/ftpserver.json | 20 +++++++++++ tests/utils_test.go | 7 +++- 5 files changed, 41 insertions(+), 38 deletions(-) create mode 100644 tests/ftpserver.json diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index f8677f7..9894334 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -27,52 +27,26 @@ jobs: repository: fclairamb/ftpserver tag: v0.10.0 fileName: ftpserver-linux-amd64 - - - name: Install - run: sudo apt install screen - - - name: Create config file - run: touch ftpserver.json - + - name: Prepare config file - run: | - cat > ftpserver.json <<- EOF - { - "version": 1, - "accesses": [ - { - "sync_and_delete": { - "enable": true - }, - "user": "user", - "pass": "password", - "fs": "os", - "params": { - "basePath": "./tmp" - } - } - ], - "passive_transfer_port_range": { - "start": 2122, - "end": 2130 - } - } - EOF - + run: mv ./tests/ftpserver.json ./ftpserver.json + - name: Create test dir run: mkdir tmp - + - name: Start ftp server run: | chmod 777 ftpserver-linux-amd64 - screen -dmS ftpserver ./ftpserver-linux-amd64 - - + ./ftpserver-linux-amd64 & + - name: Build run: make build - name: Prepare env - run: mv Makefile.env.example Makefile.env + run: | + export STORAGE_FTP_INTEGRATION_TEST=on + export STORAGE_FTP_CREDENTIAL=basic:user:password + export STORAGE_FTP_ENDPOINT=tcp:127.0.0.1:2121 - name: Integration Test run: make integration_test diff --git a/Makefile.env.example b/Makefile.env.example index 8b82f8b..fee84c8 100644 --- a/Makefile.env.example +++ b/Makefile.env.example @@ -1 +1,3 @@ -export STORAGE_FTP_INTEGRATION_TEST=on \ No newline at end of file +export STORAGE_FTP_INTEGRATION_TEST=on +export STORAGE_FTP_CREDENTIAL=basic:user:password +export STORAGE_FTP_ENDPOINT=tcp:127.0.0.1:2121 \ No newline at end of file diff --git a/tests/README.md b/tests/README.md index e3eee8a..40354b3 100644 --- a/tests/README.md +++ b/tests/README.md @@ -20,6 +20,8 @@ Set following environment variables: ```shell export STORAGE_FTP_INTEGRATION_TEST=on +export STORAGE_FTP_CREDENTIAL=basic:user:password +export STORAGE_FTP_ENDPOINT=tcp:127.0.0.1:2121 ``` Run tests diff --git a/tests/ftpserver.json b/tests/ftpserver.json new file mode 100644 index 0000000..1be5a93 --- /dev/null +++ b/tests/ftpserver.json @@ -0,0 +1,20 @@ +{ + "version": 1, + "accesses": [ + { + "sync_and_delete": { + "enable": true + }, + "user": "user", + "pass": "password", + "fs": "os", + "params": { + "basePath": "./tmp" + } + } + ], + "passive_transfer_port_range": { + "start": 2122, + "end": 2130 + } +} diff --git a/tests/utils_test.go b/tests/utils_test.go index 2eac385..063f6fc 100644 --- a/tests/utils_test.go +++ b/tests/utils_test.go @@ -1,9 +1,11 @@ package tests import ( + "os" "testing" _ "github.com/beyondstorage/go-service-ftp" + ps "github.com/beyondstorage/go-storage/v4/pairs" "github.com/beyondstorage/go-storage/v4/services" "github.com/beyondstorage/go-storage/v4/types" ) @@ -11,7 +13,10 @@ import ( func initTest(t *testing.T) (store types.Storager) { t.Log("Setup test for ftp") - store, err := services.NewStoragerFromString("ftp://?endpoint=tcp:127.0.0.1:2121&credential=basic:user:password") + store, err := services.NewStorager("ftp", + ps.WithCredential(os.Getenv("STORAGE_FTP_CREDENTIAL")), + ps.WithEndpoint(os.Getenv("STORAGE_FTP_ENDPOINT")), + ) if err != nil { t.Errorf("create storager: %v", err) } From e904e8826cf2a0aad9e95ca3e57ddc961a6f38f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Mon, 9 Aug 2021 18:54:39 +0800 Subject: [PATCH 25/27] fix: remove irrelated codes --- storage.go | 12 +----------- utils.go | 24 +++++++++++------------- 2 files changed, 12 insertions(+), 24 deletions(-) diff --git a/storage.go b/storage.go index fdd19ab..735ccac 100644 --- a/storage.go +++ b/storage.go @@ -124,17 +124,7 @@ func (s *Storage) stat(ctx context.Context, path string, opt pairStorageStat) (o rp := s.getAbsPath(path) fl, err := s.connection.List(rp) if err != nil { - flst, err := s.connection.List(filepath.Dir(rp)) - if err != nil { - return nil, err - } - for i := range flst { - if filepath.Base(rp) == flst[i].Name { - fl = []*ftp.Entry{flst[i]} - break - } - } - + return nil, err } if len(fl) == 0 { return nil, services.ErrObjectNotExist diff --git a/utils.go b/utils.go index d86bc48..0637233 100644 --- a/utils.go +++ b/utils.go @@ -82,19 +82,17 @@ func newStorager(pairs ...types.Pair) (store *Storage, err error) { store.workDir = opt.WorkDir } - if opt.HasCredential { - cp, err := credential.Parse(opt.Credential) - if err != nil { - return nil, err - } - switch cp.Protocol() { - case credential.ProtocolBasic: - user, pass := cp.Basic() - store.password = pass - store.user = user - default: - return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} - } + cp, err := credential.Parse(opt.Credential) + if err != nil { + return nil, err + } + switch cp.Protocol() { + case credential.ProtocolBasic: + user, pass := cp.Basic() + store.password = pass + store.user = user + default: + return nil, services.PairUnsupportedError{Pair: ps.WithCredential(opt.Credential)} } err = store.connect() From b4891b78a8f06c86f122ea88ed5a049f822e49c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Mon, 9 Aug 2021 19:16:34 +0800 Subject: [PATCH 26/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 9894334..64f160b 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -6,6 +6,10 @@ jobs: integration_test: name: Integration Test runs-on: ${{ matrix.os }} + env: + STORAGE_FTP_INTEGRATION_TEST: on + STORAGE_FTP_CREDENTIAL: basic:user:password + STORAGE_FTP_ENDPOINT: 127.0.0.1:2121 strategy: matrix: @@ -42,11 +46,5 @@ jobs: - name: Build run: make build - - name: Prepare env - run: | - export STORAGE_FTP_INTEGRATION_TEST=on - export STORAGE_FTP_CREDENTIAL=basic:user:password - export STORAGE_FTP_ENDPOINT=tcp:127.0.0.1:2121 - - name: Integration Test run: make integration_test From 2ccfa6263367490ac8de2e0c3fce6085bfe94945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?N/P=E7=A1=85?= Date: Wed, 11 Aug 2021 18:13:19 +0800 Subject: [PATCH 27/27] Update integration-test.yml --- .github/workflows/integration-test.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/integration-test.yml b/.github/workflows/integration-test.yml index 64f160b..cec11c0 100644 --- a/.github/workflows/integration-test.yml +++ b/.github/workflows/integration-test.yml @@ -9,7 +9,7 @@ jobs: env: STORAGE_FTP_INTEGRATION_TEST: on STORAGE_FTP_CREDENTIAL: basic:user:password - STORAGE_FTP_ENDPOINT: 127.0.0.1:2121 + STORAGE_FTP_ENDPOINT: tcp:127.0.0.1:2121 strategy: matrix: