Rebase

.circleci/config.yml

@@ -11,21 +11,22 @@ jobs:
     docker:
       - image: circleci/python:2.7
         environment:
-          JULIA_CPU_CORES: 6
-          JULIA_TEST_MAXRSS_MB: 800
           ARCH: i686
+          JULIA_CPU_THREADS: 4
+          JULIA_TEST_MAXRSS_MB: 800
     steps: &steps
       - run: | # install build dependencies
-          sudo apt-get install -y g++-4.8-multilib gfortran-4.8-multilib \
+          sudo apt-get install -y g++-6-multilib gfortran-6-multilib \
             time ccache bar &&
           for prog in gcc g++ gfortran; do
-            sudo ln -s /usr/bin/$prog-4.8 /usr/local/bin/$prog;
+            sudo ln -s /usr/bin/$prog-6 /usr/local/bin/$prog;
           done
       - checkout # circle ci code checkout step
 # (FIXME: need to unset url."ssh://[email protected]".insteadOf or libgit2 tests fail)
       - run: | # checkout merge commit, set versioning info and Make.user variables
           git config --global --unset url."ssh://[email protected]".insteadOf &&
           if [ -n "$CIRCLE_PULL_REQUEST" ]; then
+            git remote set-url origin [email protected]:JuliaLang/julia.git &&
             git fetch origin +refs/pull/$(basename $CIRCLE_PULL_REQUEST)/merge &&
             git checkout -qf FETCH_HEAD;
           fi &&
@@ -40,24 +41,29 @@ jobs:
             - ccache-{{ arch }}-{{ checksum "/tmp/weeknumber" }}
       - run: | # compile julia deps
           contrib/download_cmake.sh &&
-          make -j8 -C deps || make
+          make -j3 -C deps || make
       - run: | # build julia, output ccache stats when done
-          make -j8 all &&
+          make -j3 release &&
           make prefix=/tmp/julia install &&
           ccache -s &&
           make build-stats
       - run: | # move source tree out of the way, run tests from install tree
+          COMMIT_RANGE=$(echo "${CIRCLE_COMPARE_URL}" | cut -d/ -f7) &&
+          [[ "$COMMIT_RANGE" != *"..."* ]] && COMMIT_RANGE="${COMMIT_RANGE}...${COMMIT_RANGE}";
+          FILES_CHANGED=$(git diff --name-only $COMMIT_RANGE -- || git ls-files) &&
           cd .. &&
           mv project julia-src &&
-          /tmp/julia/bin/julia -e 'versioninfo()' &&
+          /tmp/julia/bin/julia -e 'Base.require(Main, :InteractiveUtils).versioninfo()' &&
           /tmp/julia/bin/julia --sysimage-native-code=no -e 'true' &&
-          /tmp/julia/bin/julia-debug --sysimage-native-code=no -e 'true' &&
+          #/tmp/julia/bin/julia-debug --sysimage-native-code=no -e 'true' &&
           pushd /tmp/julia/share/julia/test &&
-          /tmp/julia/bin/julia --check-bounds=yes runtests.jl all --skip socket | bar -i 30 &&
-          /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg &&
+          if [ $(echo "$FILES_CHANGED" | grep -cv '^doc/') -gt 0 ]; then
+            /tmp/julia/bin/julia --check-bounds=yes runtests.jl all --skip Sockets | bar -i 30 &&
+            /tmp/julia/bin/julia --check-bounds=yes runtests.jl LibGit2/online Pkg/pkg download;
+          fi &&
           popd &&
           mkdir /tmp/embedding-test &&
-          make check -C /tmp/julia/share/doc/julia/examples/embedding \
+          make check -C /tmp/julia/share/julia/test/embedding \
             JULIA=/tmp/julia/bin/julia BIN=/tmp/embedding-test \
             "$(cd julia-src && make print-CC)" &&
           mv julia-src project
@@ -74,7 +80,7 @@ jobs:
     docker:
       - image: circleci/python:2.7
         environment:
-          JULIA_CPU_CORES: 6
-          JULIA_TEST_MAXRSS_MB: 800
           ARCH: x86_64
+          JULIA_CPU_THREADS: 4
+          JULIA_TEST_MAXRSS_MB: 800
     steps: *steps

.freebsdci.sh

@@ -0,0 +1,65 @@
+#!/bin/sh
+# FreeBSD CI Build Scripts
+# The flow of a FreeBSD CI (https://freebsdci.julialang.org) build:
+#
+# 1. `cleanup`
+# 2. `compile`
+# 3. `build-state`
+# 4. `runtests`
+# 5. `test-embedding`
+#
+# Detail of flow is controlled by the variable `factory`
+# here.
+# https://github.com/iblis17/julia-fbsd-buildbot/blob/master/master/master.cfg
+#
+# Usage: .freebsdci.sh <stage>
+
+set -xe
+
+build-state(){
+    gmake build-stats
+}
+
+cleanup(){
+    git clean -fdx
+}
+
+compile(){
+    export MALLOC_CONF='junk:false'
+    export VERBOSE=1
+    export FORCE_ASSERTIONS=1
+    export LLVM_ASSERTIONS=1
+    export USECCACHE=1
+
+    gmake check-whitespace
+    gmake release -j $MAKE_JOBS_NUMBER
+}
+
+runtests(){
+    export MALLOC_CONF='junk:false'
+    export VERBOSE=1
+    export FORCE_ASSERTIONS=1
+    export LLVM_ASSERTIONS=1
+    export JULIA_TEST_MAXRSS_MB=600
+    export JULIA_CPU_THREADS=$MAKE_JOBS_NUMBER
+
+    ./usr/bin/julia --check-bounds=yes test/runtests.jl all
+    ./usr/bin/julia --check-bounds=yes test/runtests.jl \
+        LibGit2/online Pkg/pkg download
+}
+
+test-embedding(){
+    export JULIA='../../julia'
+    export BIN='../../tmp'
+
+    mkdir -vp tmp
+    gmake -C test embedding
+}
+
+
+if [ -z $1 ]
+then
+    exit 1
+fi
+
+$1

.gitattributes

@@ -0,0 +1,2 @@
+# treat patches as files that should not be modified
+*.patch -text

.github/CODE_OF_CONDUCT.md

@@ -0,0 +1,23 @@
+Julia Community Standards
+=========================
+
+The Julia community is committed to maintaining a welcoming, civil and constructive environment. We expect the following standards to be observed and upheld by all participants in any community forum (mailing lists, GitHub, IRC, etc.).
+
+**Be respectful and inclusive.**
+Please do not use overtly sexual language or imagery, and do not attack anyone based on any aspect of personal identity, including gender, sexuality, religion, ethnicity, race, age or ability. Keep in mind that what you write in public forums is read by many people who don't know you personally, so please refrain from making prejudiced or sexual jokes and comments – even ones that you might consider acceptable in private. Ask yourself if a comment or statement might make someone feel unwelcomed or like an outsider.
+
+In particular, do not sexualize the term "Julia" or any other aspects of the project. While "Julia" is a female name in many parts of the world, the programming language is not a person and does not have a gender.
+
+**Give credit.**
+All participants in the Julia community are expected to respect copyright laws and ethical attribution standards. This applies to both code and written materials, such as documentation or blog posts. Materials that violate the law, are plagiaristic, or ethically dubious in some way will be removed from officially-maintained lists of resources.
+
+If you believe one of these standards has been violated, you can either file an issue on an appropriate repository or confidentially contact the [Julia Stewards](https://julialang.org/community/stewards/) at [[email protected]](mailto:[email protected]). Keep in mind that most mistakes are due to ignorance rather than malice.
+
+**Be concise.**
+Constructive criticism and suggestions are welcome, but high-traffic forums do not generally have the bandwidth for extensive discourse. Consider writing a blog post if you feel that you have enough to say on a particular subject.
+
+**Get involved.**
+The Julia community is built on a foundation of reciprocity and collaboration. Be aware that most community members contribute on a voluntary basis, so ideas and bug reports are ok, but demands are not. Pull requests are always welcomed – see the [guidelines for contributing](https://github.com/JuliaLang/julia/blob/master/CONTRIBUTING.md) to read about how to get started.
+
+**Any concerns?**
+If you have a conflict or concern that requires resolution, please contact the [Julia Community Stewards](https://julialang.org/community/stewards/).

.gitignore

@@ -4,6 +4,7 @@
 /dist-extras
 /julia
 /usr
+/oprofile_data
 /usr-staging
 /Make.user
 /julia-*
@@ -24,4 +25,5 @@
 *.jl.mem
 *.ji
 
+/perf*
 .DS_Store

.travis.yml

@@ -11,6 +11,7 @@ matrix:
           sources:
             - ubuntu-toolchain-r-test
           packages:
+            - ccache
             - libssl1.0.0
             - bar
             - time
@@ -31,6 +32,7 @@ matrix:
           sources:
             - ubuntu-toolchain-r-test
           packages:
+            - ccache
             - libssl1.0.0
             - bar
             - time
@@ -39,27 +41,14 @@ matrix:
     - os: osx
       env: ARCH="x86_64"
       osx_image: xcode8
-cache:
-  directories:
-    - $TRAVIS_BUILD_DIR/deps/srccache
-    - $TRAVIS_BUILD_DIR/deps/scratch
-    - $TRAVIS_BUILD_DIR/deps/usr-staging
+cache: ccache
 branches:
   only:
     - master
     - /^release-.*/
     - /^v\d+\.\d+\.\d+$/
 notifications:
     email: false
-    irc:
-        channels:
-            - "chat.freenode.net#julia-notifications"
-        on_success: change
-        on_failure: always
-    webhooks:
-        urls:
-          - http://status.julialang.org/put/travis
-          - http://julia.mit.edu:8000/travis-hook
 before_install:
     - make check-whitespace
     - if [ `uname` = "Linux" ]; then
@@ -72,40 +61,41 @@ before_install:
         ln -s /usr/bin/g++-5 $HOME/bin/x86_64-linux-gnu-g++;
         gcc --version;
         BAR="bar -i 30";
-        BUILDOPTS="-j5 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1";
+        BUILDOPTS="-j5 VERBOSE=1 FORCE_ASSERTIONS=1 LLVM_ASSERTIONS=1 USECCACHE=1";
         echo "override ARCH=$ARCH" >> Make.user;
         sudo sh -c "echo 0 > /proc/sys/net/ipv6/conf/lo/disable_ipv6";
-        export JULIA_CPU_CORES=4;
+        export JULIA_CPU_THREADS=4;
         export JULIA_TEST_MAXRSS_MB=1200;
         TESTSTORUN="all";
       elif [ `uname` = "Darwin" ]; then
         brew update;
-        brew install -v jq pv;
+        brew install -v jq pv ccache;
+        export PATH="$(brew --prefix ccache)/libexec:$PATH";
         BAR="pv -i 30";
         contrib/travis_fastfail.sh || exit 1;
-        brew tap staticfloat/julia;
+        brew tap staticfloat/julia > /dev/null;
         brew rm --force $(brew deps --HEAD julia);
-        brew install -v --only-dependencies --HEAD julia;
-        brew install -v staticfloat/juliadeps/libgfortran llvm39-julia;
-        BUILDOPTS="-j3 USECLANG=1 LLVM_CONFIG=$(brew --prefix llvm39-julia)/bin/llvm-config LLVM_SIZE=$(brew --prefix llvm39-julia)/bin/llvm-size";
+        brew install -v gcc gmp mpfr pcre2 staticfloat/julia/openblas-julia staticfloat/julia/suite-sparse-julia staticfloat/juliadeps/libgfortran;
+        BUILDOPTS="-j3 USECLANG=1 USECCACHE=1 BINARYBUILDER_TRIPLET=x86_64-apple-darwin14 BINARYBUILDER_LLVM_ASSERTS=1";
+        BUILDOPTS="$BUILDOPTS USE_BINARYBUILDER_LLVM=1 LLVM_CONFIG=$TRAVIS_BUILD_DIR/usr/tools/llvm-config LLVM_SIZE=$TRAVIS_BUILD_DIR/usr/tools/llvm-size";
         BUILDOPTS="$BUILDOPTS VERBOSE=1 USE_BLAS64=0 SUITESPARSE_INC=-I$(brew --prefix suite-sparse-julia)/include FORCE_ASSERTIONS=1";
         BUILDOPTS="$BUILDOPTS LIBBLAS=-lopenblas LIBBLASNAME=libopenblas LIBLAPACK=-lopenblas LIBLAPACKNAME=libopenblas";
-        for lib in LLVM SUITESPARSE ARPACK BLAS LAPACK GMP MPFR PCRE LIBUNWIND; do
+        for lib in SUITESPARSE BLAS LAPACK GMP MPFR PCRE LIBUNWIND; do
             BUILDOPTS="$BUILDOPTS USE_SYSTEM_$lib=1";
         done;
         export LDFLAGS="-L$(brew --prefix openblas-julia)/lib -L$(brew --prefix suite-sparse-julia)/lib";
         spawn_DYLD_FALLBACK_LIBRARY_PATH="/usr/local/lib:/lib:/usr/lib";
         spawn_DYLD_FALLBACK_LIBRARY_PATH+=":$(brew --prefix openblas-julia)/lib";
         spawn_DYLD_FALLBACK_LIBRARY_PATH+=":$(brew --prefix suite-sparse-julia)/lib";
-        spawn_DYLD_FALLBACK_LIBRARY_PATH+=":$(brew --prefix arpack-julia)/lib";
         export JULIA_MACOS_SPAWN="DYLD_FALLBACK_LIBRARY_PATH=\"$spawn_DYLD_FALLBACK_LIBRARY_PATH\" \$1";
         export BUILDOPTS="$BUILDOPTS spawn=\$(JULIA_MACOS_SPAWN)";
         make $BUILDOPTS -C contrib -f repackage_system_suitesparse4.make;
-        export JULIA_CPU_CORES=2;
+        export JULIA_CPU_THREADS=2;
         export JULIA_TEST_MAXRSS_MB=600;
         TESTSTORUN="all --skip linalg/triangular subarray"; fi # TODO: re enable these if possible without timing out
     - echo "override JULIA_CPU_TARGET=generic;native" >> Make.user
-    - git clone -q git://git.kitenet.net/moreutils
+    - wget http://http.debian.net/debian/pool/main/m/moreutils/moreutils_0.62.orig.tar.xz
+    - tar -xJvf moreutils_0.62.orig.tar.xz && mv moreutils-0.62 moreutils
 script:
     - echo BUILDOPTS=$BUILDOPTS
     - export BUILDOPTS
@@ -128,35 +118,41 @@ script:
           echo "-- end of deps build log -----------------------------------------------";
           false; }
     # compile / install Julia
+    - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia release | moreutils/ts -s "%.s"
     - make $BUILDOPTS NO_GIT=1 prefix=/tmp/julia install | moreutils/ts -s "%.s"
-    - if [ `uname` = "Darwin" ]; then
-        for name in suitesparseconfig spqr umfpack colamd cholmod amd suitesparse_wrapper; do
-            install -pm755 usr/lib/lib${name}*.dylib* /tmp/julia/lib/julia/;
-        done;
-      fi
     - make $BUILDOPTS NO_GIT=1 build-stats
     - du -sk /tmp/julia/*
     - ls -l /tmp/julia/lib
     - ls -l /tmp/julia/lib/julia
+    - FILES_CHANGED=$(git diff --name-only $TRAVIS_COMMIT_RANGE -- || git ls-files)
     - cd .. && mv julia julia2
     # run tests
-    - /tmp/julia/bin/julia --sysimage-native-code=no -e 'true' &&
-        /tmp/julia/bin/julia-debug --sysimage-native-code=no -e 'true'
-    - /tmp/julia/bin/julia -e 'versioninfo()'
+    - /tmp/julia/bin/julia --sysimage-native-code=no -e 'true'
+    # - /tmp/julia/bin/julia-debug --sysimage-native-code=no -e 'true'
+    - /tmp/julia/bin/julia -e 'Base.require(Main, :InteractiveUtils).versioninfo()'
     - pushd /tmp/julia/share/julia/test
-    - /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
-      /tmp/julia/bin/julia --check-bounds=yes runtests.jl libgit2-online download pkg
+    # skip tests if only files within the "doc" dir have changed
+    - if [ $(echo "$FILES_CHANGED" | grep -cv '^doc/') -gt 0 ]; then
+        /tmp/julia/bin/julia --check-bounds=yes runtests.jl $TESTSTORUN &&
+        /tmp/julia/bin/julia --check-bounds=yes runtests.jl LibGit2/online Pkg/pkg download; fi
     - popd
     # test that the embedding code works on our installation
     - mkdir /tmp/embedding-test &&
-        make check -C /tmp/julia/share/doc/julia/examples/embedding \
+        make check -C /tmp/julia/share/julia/test/embedding \
              JULIA="/tmp/julia/bin/julia" \
              BIN=/tmp/embedding-test \
              "$(cd julia2 && make print-CC)"
     # restore initial state and prepare for travis caching
     - mv julia2 julia &&
         rm -f julia/deps/scratch/libgit2-*/CMakeFiles/CMakeOutput.log
+    # run the LLVM tests on Linux
+    - if [ `uname` = "Linux" ]; then
+        pushd julia && make -C test/llvmpasses && popd; fi
+    # run the doctests on Linux 64-bit
+    - if [ `uname` = "Linux" ] && [ $ARCH = "x86_64" ]; then
+        pushd julia && make -C doc doctest=true && popd; fi
 # uncomment the following if failures are suspected to be due to the out-of-memory killer
 #    - dmesg
 after_success:
-    - cd julia && make -C doc deploy
+    - if [ `uname` = "Linux" ] && [ $ARCH = "x86_64" ]; then
+        cd julia && make -C doc deploy; fi

CONTRIBUTING.md

@@ -41,7 +41,7 @@ A useful bug report filed as a GitHub issue provides information about how to re
   - Try some simple debugging techniques to help isolate the problem.
     - Try running the code with the debug build of Julia with `make debug`, which produces the `usr/bin/julia-debug`.
     - Consider running `julia-debug` with a debugger such as `gdb` or `lldb`. Obtaining even a simple [backtrace](http://www.unknownroad.com/rtfm/gdbtut/gdbsegfault.html) is very useful.
-    - If Julia segfaults, try following [these debugging tips](https://docs.julialang.org/en/latest/devdocs/backtraces.html#Reporting-and-analyzing-crashes-(segfaults)-1) to help track down the specific origin of the bug.
+    - If Julia segfaults, try following [these debugging tips](https://docs.julialang.org/en/latest/devdocs/backtraces#Reporting-and-analyzing-crashes-(segfaults)-1) to help track down the specific origin of the bug.
 
 2. If the problem is caused by a Julia package rather than core Julia, file a bug report with the relevant package author rather than here.
 
@@ -84,7 +84,7 @@ There are never enough tests. Track [code coverage at Coveralls](https://coveral
 
 2. Browse through the source files and find some untested functionality (highlighted in red) that you think you might be able to write a test for.
 
-3. Write a test that exercises this functionality---you can add your test to one of the existing files, or start a new one, whichever seems most appropriate to you. If you're adding a new test file, make sure you include it in the list of tests in `test/choosetests.jl`. https://docs.julialang.org/en/latest/stdlib/test may be helpful in explaining how the testing infrastructure works.
+3. Write a test that exercises this functionality---you can add your test to one of the existing files, or start a new one, whichever seems most appropriate to you. If you're adding a new test file, make sure you include it in the list of tests in `test/choosetests.jl`. https://docs.julialang.org/en/latest/stdlib/Test/ may be helpful in explaining how the testing infrastructure works.
 
 4. Run `make test-all` to rebuild Julia and run your new test(s). If you had to fix a bug or add functionality in `base`, this will ensure that your test passes and that you have not introduced extraneous whitespace.
 
@@ -178,7 +178,7 @@ Examples written within docstrings can be used as testcases known as "doctests"
     "DOCSTRING TEST"
     ```
 
-A doctest needs to match an interactive REPL including the `julia>` prompt. To run doctests you need to run `make -C doc check` from the root directory. It is recommended to add the header `# Examples` above the doctests.
+A doctest needs to match an interactive REPL including the `julia>` prompt. To run doctests you need to run `make -C doc doctest=true` from the root directory. It is recommended to add the header `# Examples` above the doctests.
 
 #### News-worthy changes
 
@@ -235,7 +235,7 @@ Here is the standard procedure:
 2. Start a Julia REPL session. Then issue the following commands:
 
 ```julia
-using Revise    # if you aren't launching it in your .juliarc.jl
+using Revise    # if you aren't launching it in your `.julia/config/startup.jl`
 Revise.track(Base)
 ```