Merge branch 't/21559/change-src-bin-installation' into t/29950/build_sagelib_using_installed_sage_setup

Author: Matthias Koeppe

.gitattributes

@@ -0,0 +1,5 @@
+# Force LF normalization
+* text=auto eol=lf
+# except for Windows batch files
+*.{cmd,[cC][mM][dD]} text eol=crlf
+*.{bat,[bB][aA][tT]} text eol=crlf

.github/workflows/ci-cygwin-minimal.yml

@@ -11,7 +11,7 @@ env:
   MAKE: make -j8
   SAGE_NUM_THREADS: 3
   SAGE_CHECK: warn
-  SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
+  SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
   CYGWIN: winsymlinks:native
   CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url
   SAGE_FAT_BINARY: yes

.github/workflows/ci-cygwin-standard.yml

@@ -11,7 +11,7 @@ env:
   MAKE: make -j8
   SAGE_NUM_THREADS: 3
   SAGE_CHECK: warn
-  SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!python2,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
+  SAGE_CHECK_PACKAGES: "!cython,!r,!python3,!nose,!pathpy,!gap,!cysignals,!linbox,!git,!ppl"
   CYGWIN: winsymlinks:native
   CONFIGURE_ARGS: --enable-experimental-packages --enable-download-from-upstream-url
   SAGE_FAT_BINARY: yes

README.md

@@ -173,7 +173,7 @@ Guide](https://doc.sagemath.org/html/en/installation).
 
     - [Git] Alternatively, clone the Sage git repository:
 
-            $ git clone --branch master git://trac.sagemath.org/sage.git
+            $ git clone -c core.symlinks=true --branch master git://trac.sagemath.org/sage.git
 
       This will create the subdirectory `sage`.
 
@@ -183,12 +183,9 @@ Guide](https://doc.sagemath.org/html/en/installation).
 
       Therefore it is crucial that you unpack the source tree from the
       Cygwin (or WSL) `bash` using the Cygwin (or WSL) `tar` utility
-      and not using other Windows tools (including mingw).  Likewise,
-      when using `git`, it is crucial that you use the Cygwin (or WSL)
-      version of `git`, and that you configure it as follows first:
-
-            $ git config --global core.autocrlf false
-            $ git config --global core.symlinks true
+      and not using other Windows tools (including mingw). Likewise,
+      when using `git`, it is recommended (but not necessary) to use the Cygwin (or WSL)
+      version of `git`.
 
 3. `cd` into the source/build directory:
 
@@ -271,8 +268,8 @@ Guide](https://doc.sagemath.org/html/en/installation).
    guide](https://doc.sagemath.org/html/en/installation/source.html#environment-variables).
 
 11. Type `./configure`, followed by any options that you wish to use.
-   For example, to build a Python2-based Sage (which was the default
-   before Sage 9.0), use `./configure --with-python=2`.
+   For example, to build Sage with `gf2x` package supplied by Sage,
+   use `./configure --with-system-gf2x=no`.
 
    At the end of a successful `./configure` run, you may see messages
    recommending to install extra system packages using your package
@@ -422,7 +419,7 @@ through to another Makefile under `build/make/Makefile`.
 The latter `build/make/Makefile` *is* generated by an autoconf-generated
 `configure` script, using the template in `build/make/Makefile.in`.  This
 includes rules for building the Sage library itself (`make sagelib`), and for
-building and installing each of Sage's dependencies (e.g. `make python2`).
+building and installing each of Sage's dependencies (e.g. `make gf2x`).
 
 The `configure` script itself, if it is not already built, can be generated by
 running the `bootstrap` script (the latter requires _GNU autotools_ being installed).

VERSION.txt

@@ -1 +1 @@
-SageMath version 9.2.beta3, Release Date: 2020-07-04
+SageMath version 9.2.beta4, Release Date: 2020-07-08

build/bin/sage-site

@@ -158,6 +158,7 @@ if [ "$1" = "-docbuild" -o "$1" = "--docbuild" ]; then
     # tends to ask interactive questions if something goes wrong. These
     # cause the build to hang. If stdin is /dev/null, TeX just aborts.
     shift
+    export LANG=C # to ensure it is possible to scrape out non-EN locale warnings
     exec sage-python -m sage_setup.docbuild "$@" </dev/null
 fi
 

build/bin/write-dockerfile.sh

@@ -161,7 +161,6 @@ ADD Makefile VERSION.txt README.md bootstrap configure.ac sage ./
 ADD src/doc/bootstrap src/doc/bootstrap
 ADD m4 ./m4
 ADD build ./build
-ADD src/bin/sage-version.sh src/bin/sage-version.sh
 $RUN ./bootstrap
 
 FROM bootstrapped as configured

build/make/install

@@ -19,8 +19,6 @@ export SAGE_SHARE="$SAGE_LOCAL/share"
 export SAGE_PKGCONFIG="$SAGE_LOCAL/lib/pkgconfig"
 export SAGE_LOGS="$SAGE_ROOT/logs/pkgs"
 export SAGE_SPKG_INST="$SAGE_LOCAL/var/lib/sage/installed"
-. "$SAGE_SRC"/bin/sage-version.sh
-export SAGE_VERSION
 
 if [ -z "${SAGE_ORIG_PATH_SET}" ]; then
     SAGE_ORIG_PATH=$PATH && export SAGE_ORIG_PATH

build/pkgs/configure/checksums.ini

@@ -1,4 +1,4 @@
 tarball=configure-VERSION.tar.gz
-sha1=f4040175ebc1ebcae8eb41eaab02f84e5840692a
-md5=8b89552d880b19af9d7830e068084ae9
-cksum=582947775
+sha1=1130babcd6d704914c96e5e9b686d04cffaaf86a
+md5=37ec521d87e3c4c46ea05a04cc57ff0e
+cksum=2049642027

build/pkgs/configure/package-version.txt

@@ -1 +1 @@
-73f0829779110ea1262fafbceecdd56e108f1b7b
+3e65aa67cc253468cee514c6ee448b0b7a78da01

build/pkgs/database_stein_watkins_mini/spkg-install.in

@@ -0,0 +1 @@
+../database_stein_watkins/spkg-install.in

build/pkgs/database_stein_watkins_mini/spkg-install.in

@@ -1,4 +1,5 @@
 tarball=fplll-VERSION.tar.gz
-sha1=67b70f4dcbb835025abce879b9acb4500e2f1d2c
-md5=e72082af9084c5b2d427977c9b79a602
-cksum=65319984
+sha1=3ab5e269fdf73b6f933b24db656e5c55d83aa690
+md5=31f0f5d2045802d0beb8945dd0fa07f0
+cksum=2692708834
+upstream_url=https://github.com/fplll/fplll/releases/download/VERSION/fplll-VERSION.tar.gz

build/pkgs/fplll/checksums.ini

@@ -1 +1 @@
-5.3.2
+5.3.3

build/pkgs/fplll/package-version.txt

@@ -1,13 +1,9 @@
 SAGE_SPKG_CONFIGURE([freetype], [
-    AC_REQUIRE([SAGE_SPKG_CONFIGURE_LIBPNG])
-    AC_MSG_CHECKING([Installing libpng? ])
-    if test x$sage_spkg_install_libpng = xyes; then
-      AC_MSG_RESULT([yes; install freetype as well])
-      sage_spkg_install_freetype=yes
-    else
-      AC_MSG_RESULT([no])
-      PKG_CHECK_MODULES([FREETYPE], [freetype2 >= 2.4], [], [sage_spkg_install_freetype=yes])
-    fi
+    SAGE_SPKG_DEPCHECK([libpng], [
+      dnl freetype versions are libtool's ones, cf trac #30014
+      PKG_CHECK_MODULES([FREETYPE], [freetype2 >= 16.1], [], [sage_spkg_install_freetype=yes])
+    ])
+], [], [], [
     if test x$sage_spkg_install_freetype = xyes; then
       AC_SUBST(SAGE_FREETYPE_PREFIX, ['$SAGE_LOCAL'])
     else

build/pkgs/freetype/spkg-configure.m4

@@ -1 +1 @@
-9.2.beta3
+9.2.beta4

build/pkgs/sagelib/package-version.txt

@@ -0,0 +1 @@
+tox

build/pkgs/tox/requirements.txt

@@ -0,0 +1 @@
+optional

build/pkgs/tox/type

@@ -14,6 +14,7 @@
 #                  https://www.gnu.org/licenses/
 # ****************************************************************************
 
+
 import os
 import logging
 log = logging.getLogger()
@@ -22,6 +23,9 @@
 from sage_bootstrap.tarball import Tarball
 from sage_bootstrap.updater import ChecksumUpdater, PackageUpdater
 from sage_bootstrap.creator import PackageCreator
+from sage_bootstrap.pypi import PyPiVersion, PyPiNotFound
+from sage_bootstrap.fileserver import FileServer
+from sage_bootstrap.expand_class import PackageClass
 
 
 class Application(object):
@@ -39,7 +43,7 @@ def config(self):
         from sage_bootstrap.config import Configuration
         print(Configuration())
 
-    def list(self):
+    def list_cls(self, package_class):
         """
         Print a list of all available packages
 
@@ -52,8 +56,9 @@ def list(self):
         zn_poly
         """
         log.debug('Listing packages')
-        for pkg in Package.all():
-            print(pkg.name)
+        pc = PackageClass(package_class)
+        for pkg_name in pc.names:
+            print(pkg_name)
 
     def name(self, tarball_filename):
         """
@@ -113,6 +118,32 @@ def update(self, package_name, new_version, url=None):
             update.download_upstream(url)
         update.fix_checksum()
 
+    def update_latest(self, package_name):
+        """
+        Update a package to the latest version. This modifies the Sage sources. 
+        """
+        try:
+            pypi = PyPiVersion(package_name)
+        except PyPiNotFound:
+            log.debug('%s is not a pypi package', package_name)
+            return
+        else:
+            pypi.update()
+
+    def update_latest_all(self):
+        log.debug('Attempting to update all packages')
+        exclude = [
+            'atlas', 'flint', 'bzip2', 'ecm', 'freetype', 'gap', 'glpk', 'graphs',
+            'iconv', 'patch', 'r', 'configure', 'bliss', 'readline', 'decorator',
+            'igraph', 'rw', 'planarity', 'gambit', 
+        ]
+        pc = PackageClass(':standard:')
+        for package_name in pc.names:
+            if package_name in exclude:
+                log.debug('skipping %s because of pypi name collision', package_name)
+                continue
+            self.update_latest(package_name)
+
     def download(self, package_name, allow_upstream=False):
         """
         Download a package
@@ -126,6 +157,31 @@ def download(self, package_name, allow_upstream=False):
         package.tarball.download(allow_upstream=allow_upstream)
         print(package.tarball.upstream_fqn)
 
+    def download_cls(self, package_name_or_class):
+        pc = PackageClass(package_name_or_class)
+        pc.apply(self.download)
+
+    def upload(self, package_name):
+        """
+        Upload a package to the Sage mirror network
+
+        $ sage --package upload pari
+        Uploading /home/vbraun/Code/sage.git/upstream/pari-2.8-2044-g89b0f1e.tar.gz
+        """
+        package = Package(package_name)
+        if not os.path.exists(package.tarball.upstream_fqn):
+            log.debug('Skipping %s because there is no local tarbal', package_name)
+            return
+        log.info('Uploading %s', package.tarball.upstream_fqn)
+        fs = FileServer()
+        fs.upload(package)
+
+    def upload_cls(self, package_name_or_class):
+        pc = PackageClass(package_name_or_class)
+        pc.apply(self.upload)
+        fs = FileServer()
+        fs.publish()
+        
     def fix_all_checksums(self):
         """
         Fix the checksum of a package

build/sage_bootstrap/app.py

@@ -69,6 +69,13 @@
     autotools
     [...]
     zn_poly
+
+    $ sage --package list :standard: | sort
+    arb
+    atlas
+    backports_ssl_match_hostname
+    [...]
+    zn_poly
 """
 
 
@@ -115,6 +122,16 @@
 """
 
 
+epilog_update_latest = \
+"""
+Update a package to the latest version. This modifies the Sage sources. 
+    
+EXAMPLE:
+
+    $ sage --package update-latest ipython
+"""
+
+
 epilog_download = \
 """
 Download the tarball for a package and print the filename to stdout
@@ -127,6 +144,17 @@
 """
 
 
+epilog_upload = \
+"""
+Upload the tarball to the Sage mirror network (requires ssh key authentication)
+    
+EXAMPLE:
+
+    $ sage --package upload pari
+    Uploading /home/vbraun/Code/sage.git/upstream/pari-2.8-2044-g89b0f1e.tar.gz
+"""
+
+
 epilog_fix_checksum = \
 """
 Fix the checksum of a package
@@ -170,6 +198,10 @@ def make_parser():
         'list', epilog=epilog_list,
         formatter_class=argparse.RawDescriptionHelpFormatter,
         help='Print a list of all available packages')
+    parser_list.add_argument(
+        'package_class',
+        type=str, default=':all:', nargs='?',
+        help='Package class like :all: (default) or :standard:')
 
     parser_name = subparsers.add_parser(
         'name', epilog=epilog_name,
@@ -202,15 +234,29 @@ def make_parser():
     parser_update.add_argument(
         '--url', type=str, default=None, help='Download URL')
 
+    parser_update_latest = subparsers.add_parser(
+        'update-latest', epilog=epilog_update_latest,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help='Update a package to the latest version. This modifies the Sage sources.')
+    parser_update_latest.add_argument(
+        'package_name', type=str, help='Package name (:all: for all packages)')
+
     parser_download = subparsers.add_parser(
         'download', epilog=epilog_download,
         formatter_class=argparse.RawDescriptionHelpFormatter,
         help='Download tarball')
     parser_download.add_argument(
-        'package_name', type=str, help='Package name')
+        'package_name', type=str, help='Package name or :type:')
     parser_download.add_argument(
         '--allow-upstream', action="store_true",
         help='Whether to fall back to downloading from the upstream URL')
+
+    parser_upload = subparsers.add_parser(
+        'upload', epilog=epilog_upload,
+        formatter_class=argparse.RawDescriptionHelpFormatter,
+        help='Upload tarball to Sage mirrors')
+    parser_upload.add_argument(
+        'package_name', type=str, help='Package name or :type:')
 
     parser_fix_checksum = subparsers.add_parser(
         'fix-checksum', epilog=epilog_fix_checksum,
@@ -254,7 +300,7 @@ def run():
     if args.subcommand == 'config':
         app.config()
     elif args.subcommand == 'list':
-        app.list()
+        app.list_cls(args.package_class)
     elif args.subcommand == 'name':
         app.name(args.tarball_filename)
     elif args.subcommand == 'tarball':
@@ -263,10 +309,17 @@ def run():
         app.apropos(args.incorrect_name)
     elif args.subcommand == 'update':
         app.update(args.package_name, args.new_version, url=args.url)
+    elif args.subcommand == 'update-latest':
+        if args.package_name == ':all:':
+            app.update_latest_all()
+        else:
+            app.update_latest(args.package_name)
     elif args.subcommand == 'download':
-        app.download(args.package_name, args.allow_upstream)
+        app.download_cls(args.package_name, args.allow_upstream)
     elif args.subcommand == 'create':
         app.create(args.package_name, args.version, args.tarball, args.type, args.url)
+    elif args.subcommand == 'upload':
+        app.upload_cls(args.package_name)
     elif args.subcommand == 'fix-checksum':
         if args.package_name is None:
             app.fix_all_checksums()