diff --git a/.JuliaFormatter.toml b/.JuliaFormatter.toml new file mode 100644 index 00000000..c7439503 --- /dev/null +++ b/.JuliaFormatter.toml @@ -0,0 +1 @@ +style = "blue" \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 00000000..7f81a55a --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,17 @@ +// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.166.0/containers/julia +// See https://github.com/julia-vscode/julia-devcontainer/blob/master/Dockerfile for image contents +{ + "name": "Julia (Community)", + "image": "ghcr.io/julia-vscode/julia-devcontainer", + "extensions": [ + "julialang.language-julia", + "eamodio.gitlens", + "visualstudioexptteam.vscodeintellicode", + "ms-vscode.vscode-js-profile-flame", + "vscode-icons-team.vscode-icons", + "yzhang.markdown-all-in-one" + ], + "postCreateCommand": "/julia-devcontainer-scripts/postcreate.jl", + "remoteUser": "vscode" +} diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 00000000..f58a1763 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,9 @@ +repos: + - repo: https://github.com/crate-ci/typos + rev: v1.16.23 + hooks: + - id: typos + - repo: https://github.com/qiaojunfeng/pre-commit-julia-format + rev: v0.1.1 # use the most recent version + hooks: + - id: julia-format # formatter for Julia code diff --git a/.typos.toml b/.typos.toml new file mode 100644 index 00000000..e3858ade --- /dev/null +++ b/.typos.toml @@ -0,0 +1,31 @@ +[default] +extend-ignore-re = [ + "textcite\\{.*\\}", # citation keys + "\\b[0-9A-Za-z+/]{91}(=|==)?\\b", # base64 strings + "[0-9a-fA-F]{7,}", # git commit hashes + "\\b[0-9A-Za-z+/]{33,}(=|==)?\\b", # SHA/tpub/adresses etc strings +] + +[default.extend-words] +# code stuff +lik = "lik" +quation = "quation" +# surnames +Yau = "Yau" + +[files] +extend-exclude = [ + "datasets/*", + "_assets/*", + "_libs/*", + "images/*", + "pages/images/*", +] + +[type.bib] +check-file = false +extend-glob = ["*.bib"] + +[type.gitignore] +check-file = false +extend-glob = [".gitignore"] diff --git a/404.html b/404.html new file mode 100644 index 00000000..424ab8d5 --- /dev/null +++ b/404.html @@ -0,0 +1 @@ + 404: File not found
Bayesian Stats
Julia Logo
Code on GitHub

404: File not found

The requested file was not found.

Please click here to go to the home page.

Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language.
\ No newline at end of file diff --git a/CITATION.cff b/CITATION.cff new file mode 100644 index 00000000..6800c541 --- /dev/null +++ b/CITATION.cff @@ -0,0 +1,19 @@ +# YAML 1.2 +--- +authors: + - + family-names: Storopoli + given-names: Jose + orcid: "https://orcid.org/0000-0002-0559-5176" +cff-version: "1.2.0" +date-released: 2021-06-01 +identifiers: + - + type: url + value: "https://storopoli.io/Bayesian-Julia" +license: "CC-BY-SA-4.0" +message: "If you use this software, please cite it using these metadata." +repository-code: "https://github.com/storopoli/Bayesian-Julia" +title: "Bayesian Statistics with Julia and Turing" +version: 1.0.0 +... diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..33bec29d --- /dev/null +++ b/LICENSE @@ -0,0 +1,427 @@ +Attribution-ShareAlike 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution-ShareAlike 4.0 International Public +License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution-ShareAlike 4.0 International Public License ("Public +License"). To the extent this Public License may be interpreted as a +contract, You are granted the Licensed Rights in consideration of Your +acceptance of these terms and conditions, and the Licensor grants You +such rights in consideration of benefits the Licensor receives from +making the Licensed Material available under these terms and +conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. BY-SA Compatible License means a license listed at + creativecommons.org/compatiblelicenses, approved by Creative + Commons as essentially the equivalent of this Public License. + + d. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + e. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + f. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + g. License Elements means the license attributes listed in the name + of a Creative Commons Public License. The License Elements of this + Public License are Attribution and ShareAlike. + + h. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + i. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + j. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + k. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + l. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + m. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. Additional offer from the Licensor -- Adapted Material. + Every recipient of Adapted Material from You + automatically receives an offer from the Licensor to + exercise the Licensed Rights in the Adapted Material + under the conditions of the Adapter's License You apply. + + c. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + b. ShareAlike. + + In addition to the conditions in Section 3(a), if You Share + Adapted Material You produce, the following conditions also apply. + + 1. The Adapter's License You apply must be a Creative Commons + license with the same License Elements, this version or + later, or a BY-SA Compatible License. + + 2. You must include the text of, or the URI or hyperlink to, the + Adapter's License You apply. You may satisfy this condition + in any reasonable manner based on the medium, means, and + context in which You Share Adapted Material. + + 3. You may not offer or impose any additional or different terms + or conditions on, or apply any Effective Technological + Measures to, Adapted Material that restrict exercise of the + rights granted under the Adapter's License You apply. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material, + + including for purposes of Section 3(b); and + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. \ No newline at end of file diff --git a/Manifest.toml b/Manifest.toml new file mode 100644 index 00000000..21df450c --- /dev/null +++ b/Manifest.toml @@ -0,0 +1,2965 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.10.0" +manifest_format = "2.0" +project_hash = "1124fe7994759bb5d0f9768ab2257f95642b7a0f" + +[[deps.ADTypes]] +git-tree-sha1 = "41c37aa88889c171f1300ceac1313c06e891d245" +uuid = "47edcb42-4c32-4615-8424-f2b9edc5f35b" +version = "0.2.6" + +[[deps.AbstractFFTs]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef" +uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c" +version = "1.5.0" +weakdeps = ["ChainRulesCore", "Test"] + + [deps.AbstractFFTs.extensions] + AbstractFFTsChainRulesCoreExt = "ChainRulesCore" + AbstractFFTsTestExt = "Test" + +[[deps.AbstractLattices]] +git-tree-sha1 = "222ee9e50b98f51b5d78feb93dd928880df35f06" +uuid = "398f06c4-4d28-53ec-89ca-5b2656b7603d" +version = "0.3.0" + +[[deps.AbstractMCMC]] +deps = ["BangBang", "ConsoleProgressMonitor", "Distributed", "FillArrays", "LogDensityProblems", "Logging", "LoggingExtras", "ProgressLogging", "Random", "StatsBase", "TerminalLoggers", "Transducers"] +git-tree-sha1 = "63ae0647e8db221d63256820d1e346216c65ac66" +uuid = "80f14c24-f653-4e6a-9b94-39d6b0f70001" +version = "5.0.0" + +[[deps.AbstractPPL]] +deps = ["AbstractMCMC", "DensityInterface", "Random", "Setfield", "SparseArrays"] +git-tree-sha1 = "917ad8da4becae82028aba80b7e25197f0c76dd1" +uuid = "7a57a42e-76ec-4ea3-a279-07e840d6d9cf" +version = "0.7.0" + +[[deps.AbstractTrees]] +git-tree-sha1 = "faa260e4cb5aba097a73fab382dd4b5819d8ec8c" +uuid = "1520ce14-60c1-5f80-bbc7-55ef81b5835c" +version = "0.4.4" + +[[deps.Accessors]] +deps = ["CompositionsBase", "ConstructionBase", "Dates", "InverseFunctions", "LinearAlgebra", "MacroTools", "Test"] +git-tree-sha1 = "cb96992f1bec110ad211b7e410e57ddf7944c16f" +uuid = "7d9f7c33-5ae7-4f3b-8dc6-eff91059b697" +version = "0.1.35" + + [deps.Accessors.extensions] + AccessorsAxisKeysExt = "AxisKeys" + AccessorsIntervalSetsExt = "IntervalSets" + AccessorsStaticArraysExt = "StaticArrays" + AccessorsStructArraysExt = "StructArrays" + + [deps.Accessors.weakdeps] + AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5" + IntervalSets = "8197267c-284f-5f27-9208-e0e47529a953" + Requires = "ae029012-a4dd-5104-9daa-d747884805df" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + +[[deps.Adapt]] +deps = ["LinearAlgebra", "Requires"] +git-tree-sha1 = "cde29ddf7e5726c9fb511f340244ea3481267608" +uuid = "79e6a3ab-5dfb-504d-930d-738a2a938a0e" +version = "3.7.2" +weakdeps = ["StaticArrays"] + + [deps.Adapt.extensions] + AdaptStaticArraysExt = "StaticArrays" + +[[deps.AdvancedHMC]] +deps = ["AbstractMCMC", "ArgCheck", "DocStringExtensions", "InplaceOps", "LinearAlgebra", "LogDensityProblems", "LogDensityProblemsAD", "ProgressMeter", "Random", "Requires", "Setfield", "SimpleUnPack", "Statistics", "StatsBase", "StatsFuns"] +git-tree-sha1 = "dfa0e3508fc3df81d28624b328f3b937c1df8bc2" +uuid = "0bf59076-c3b1-5ca4-86bd-e02cd72cde3d" +version = "0.6.1" + + [deps.AdvancedHMC.extensions] + AdvancedHMCCUDAExt = "CUDA" + AdvancedHMCMCMCChainsExt = "MCMCChains" + AdvancedHMCOrdinaryDiffEqExt = "OrdinaryDiffEq" + + [deps.AdvancedHMC.weakdeps] + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" + OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" + +[[deps.AdvancedMH]] +deps = ["AbstractMCMC", "Distributions", "FillArrays", "LinearAlgebra", "LogDensityProblems", "Random", "Requires"] +git-tree-sha1 = "1cc336be36fef7df68473a7d0d60ebba25958b9e" +uuid = "5b7e9947-ddc0-4b3f-9b55-0d8042f74170" +version = "0.8.0" +weakdeps = ["DiffResults", "ForwardDiff", "MCMCChains", "StructArrays"] + + [deps.AdvancedMH.extensions] + AdvancedMHForwardDiffExt = ["DiffResults", "ForwardDiff"] + AdvancedMHMCMCChainsExt = "MCMCChains" + AdvancedMHStructArraysExt = "StructArrays" + +[[deps.AdvancedPS]] +deps = ["AbstractMCMC", "Distributions", "Random", "Random123", "Requires", "StatsFuns"] +git-tree-sha1 = "672f7ce648e06f93fceefde463c5855d77b6915a" +uuid = "576499cb-2369-40b2-a588-c64705576edc" +version = "0.5.4" +weakdeps = ["Libtask"] + + [deps.AdvancedPS.extensions] + AdvancedPSLibtaskExt = "Libtask" + +[[deps.AdvancedVI]] +deps = ["Bijectors", "Distributions", "DistributionsAD", "DocStringExtensions", "ForwardDiff", "LinearAlgebra", "ProgressMeter", "Random", "Requires", "StatsBase", "StatsFuns", "Tracker"] +git-tree-sha1 = "1f919a9c59cf3dfc68b64c22c453a2e356fca473" +uuid = "b5ca4192-6429-45e5-a2d9-87aec30a685c" +version = "0.2.4" + +[[deps.AlgebraOfGraphics]] +deps = ["Colors", "Dates", "Dictionaries", "FileIO", "GLM", "GeoInterface", "GeometryBasics", "GridLayoutBase", "KernelDensity", "Loess", "Makie", "PlotUtils", "PooledArrays", "PrecompileTools", "RelocatableFolders", "StatsBase", "StructArrays", "Tables"] +git-tree-sha1 = "3fbdee81b0cdc2b106b681dd2b9d4bdc60ca35a2" +uuid = "cbdf2221-f076-402e-a563-3d30da359d67" +version = "0.6.18" + +[[deps.Animations]] +deps = ["Colors"] +git-tree-sha1 = "e81c509d2c8e49592413bfb0bb3b08150056c79d" +uuid = "27a7e980-b3e6-11e9-2bcd-0b925532e340" +version = "0.4.1" + +[[deps.ArgCheck]] +git-tree-sha1 = "a3a402a35a2f7e0b87828ccabbd5ebfbebe356b4" +uuid = "dce04be8-c92d-5529-be00-80e4d2c0e197" +version = "2.3.0" + +[[deps.ArgTools]] +uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f" +version = "1.1.1" + +[[deps.ArnoldiMethod]] +deps = ["LinearAlgebra", "Random", "StaticArrays"] +git-tree-sha1 = "62e51b39331de8911e4a7ff6f5aaf38a5f4cc0ae" +uuid = "ec485272-7323-5ecc-a04f-4719b315124d" +version = "0.2.0" + +[[deps.ArrayInterface]] +deps = ["Adapt", "LinearAlgebra", "Requires", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "bbec08a37f8722786d87bedf84eae19c020c4efa" +uuid = "4fba245c-0d91-5ea0-9b3e-6abc04ee57a9" +version = "7.7.0" + + [deps.ArrayInterface.extensions] + ArrayInterfaceBandedMatricesExt = "BandedMatrices" + ArrayInterfaceBlockBandedMatricesExt = "BlockBandedMatrices" + ArrayInterfaceCUDAExt = "CUDA" + ArrayInterfaceGPUArraysCoreExt = "GPUArraysCore" + ArrayInterfaceStaticArraysCoreExt = "StaticArraysCore" + ArrayInterfaceTrackerExt = "Tracker" + + [deps.ArrayInterface.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" + BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + GPUArraysCore = "46192b85-c4d5-4398-a991-12ede77f4527" + StaticArraysCore = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.ArrayLayouts]] +deps = ["FillArrays", "LinearAlgebra"] +git-tree-sha1 = "a45ec4acc9d905f94b47243cff666820bb107789" +uuid = "4c555306-a7a7-4459-81d9-ec55ddd5c99a" +version = "1.5.2" +weakdeps = ["SparseArrays"] + + [deps.ArrayLayouts.extensions] + ArrayLayoutsSparseArraysExt = "SparseArrays" + +[[deps.Artifacts]] +uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33" + +[[deps.Atomix]] +deps = ["UnsafeAtomics"] +git-tree-sha1 = "c06a868224ecba914baa6942988e2f2aade419be" +uuid = "a9b6321e-bd34-4604-b9c9-b65b8de01458" +version = "0.1.0" + +[[deps.Automa]] +deps = ["PrecompileTools", "TranscodingStreams"] +git-tree-sha1 = "588e0d680ad1d7201d4c6a804dcb1cd9cba79fbb" +uuid = "67c07d97-cdcb-5c2c-af73-a7f9c32a568b" +version = "1.0.3" + +[[deps.AxisAlgorithms]] +deps = ["LinearAlgebra", "Random", "SparseArrays", "WoodburyMatrices"] +git-tree-sha1 = "01b8ccb13d68535d73d2b0c23e39bd23155fb712" +uuid = "13072b0f-2c55-5437-9ae7-d433b7a33950" +version = "1.1.0" + +[[deps.AxisArrays]] +deps = ["Dates", "IntervalSets", "IterTools", "RangeArrays"] +git-tree-sha1 = "16351be62963a67ac4083f748fdb3cca58bfd52f" +uuid = "39de3d68-74b9-583c-8d2d-e117c070f3a9" +version = "0.4.7" + +[[deps.BandedMatrices]] +deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "PrecompileTools"] +git-tree-sha1 = "27baf04c642465b4289179f29bb7127f0673d4f1" +uuid = "aae01518-5342-5314-be14-df237901396f" +version = "1.4.0" +weakdeps = ["SparseArrays"] + + [deps.BandedMatrices.extensions] + BandedMatricesSparseArraysExt = "SparseArrays" + +[[deps.BangBang]] +deps = ["Compat", "ConstructionBase", "InitialValues", "LinearAlgebra", "Requires", "Setfield", "Tables"] +git-tree-sha1 = "7aa7ad1682f3d5754e3491bb59b8103cae28e3a3" +uuid = "198e06fe-97b7-11e9-32a5-e1d131e6ad66" +version = "0.3.40" + + [deps.BangBang.extensions] + BangBangChainRulesCoreExt = "ChainRulesCore" + BangBangDataFramesExt = "DataFrames" + BangBangStaticArraysExt = "StaticArrays" + BangBangStructArraysExt = "StructArrays" + BangBangTypedTablesExt = "TypedTables" + + [deps.BangBang.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + StructArrays = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" + TypedTables = "9d95f2ec-7b3d-5a63-8d20-e2491e220bb9" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" + +[[deps.Baselet]] +git-tree-sha1 = "aebf55e6d7795e02ca500a689d326ac979aaf89e" +uuid = "9718e550-a3fa-408a-8086-8db961cd8217" +version = "0.1.1" + +[[deps.BenchmarkTools]] +deps = ["JSON", "Logging", "Printf", "Profile", "Statistics", "UUIDs"] +git-tree-sha1 = "f1f03a9fa24271160ed7e73051fba3c1a759b53f" +uuid = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +version = "1.4.0" + +[[deps.Bijectors]] +deps = ["ArgCheck", "ChainRules", "ChainRulesCore", "ChangesOfVariables", "Compat", "Distributions", "Functors", "InverseFunctions", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "MappedArrays", "Random", "Reexport", "Requires", "Roots", "SparseArrays", "Statistics"] +git-tree-sha1 = "199dc2c4151db557549a0ad8888ce1a60337ff42" +uuid = "76274a88-744f-5084-9051-94815aaf08c4" +version = "0.13.8" + + [deps.Bijectors.extensions] + BijectorsDistributionsADExt = "DistributionsAD" + BijectorsForwardDiffExt = "ForwardDiff" + BijectorsLazyArraysExt = "LazyArrays" + BijectorsReverseDiffExt = "ReverseDiff" + BijectorsTrackerExt = "Tracker" + BijectorsZygoteExt = "Zygote" + + [deps.Bijectors.weakdeps] + DistributionsAD = "ced4e74d-a319-5a8a-b0ac-84af2272839c" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.BitFlags]] +git-tree-sha1 = "2dc09997850d68179b69dafb58ae806167a32b1b" +uuid = "d1d4a3ce-64b1-5f1a-9ba4-7e7e69966f35" +version = "0.1.8" + +[[deps.BitTwiddlingConvenienceFunctions]] +deps = ["Static"] +git-tree-sha1 = "0c5f81f47bbbcf4aea7b2959135713459170798b" +uuid = "62783981-4cbd-42fc-bca8-16325de8dc4b" +version = "0.1.5" + +[[deps.BoundaryValueDiffEq]] +deps = ["ADTypes", "Adapt", "ArrayInterface", "BandedMatrices", "ConcreteStructs", "DiffEqBase", "FastAlmostBandedMatrices", "ForwardDiff", "LinearAlgebra", "LinearSolve", "NonlinearSolve", "PreallocationTools", "PrecompileTools", "Preferences", "RecursiveArrayTools", "Reexport", "SciMLBase", "Setfield", "SparseArrays", "SparseDiffTools", "Tricks", "TruncatedStacktraces", "UnPack"] +git-tree-sha1 = "dd234c9a030350d5ff4c45761d6cad0cfb358cb9" +uuid = "764a87c0-6b3e-53db-9096-fe964310641d" +version = "5.6.0" + + [deps.BoundaryValueDiffEq.extensions] + BoundaryValueDiffEqODEInterfaceExt = "ODEInterface" + BoundaryValueDiffEqOrdinaryDiffEqExt = "OrdinaryDiffEq" + + [deps.BoundaryValueDiffEq.weakdeps] + ODEInterface = "54ca160b-1b9f-5127-a996-1867f4bc2a2c" + OrdinaryDiffEq = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" + +[[deps.Bzip2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9e2a6b69137e6969bab0152632dcb3bc108c8bdd" +uuid = "6e34b625-4abd-537c-b88f-471c36dfa7a0" +version = "1.0.8+1" + +[[deps.CEnum]] +git-tree-sha1 = "389ad5c84de1ae7cf0e28e381131c98ea87d54fc" +uuid = "fa961155-64e5-5f13-b03f-caf6b980ea82" +version = "0.5.0" + +[[deps.CPUSummary]] +deps = ["CpuId", "IfElse", "PrecompileTools", "Static"] +git-tree-sha1 = "601f7e7b3d36f18790e2caf83a882d88e9b71ff1" +uuid = "2a0fbf3d-bb9c-48f3-b0a9-814d99fd7ab9" +version = "0.2.4" + +[[deps.CRC32c]] +uuid = "8bf52ea8-c179-5cab-976a-9e18b702a9bc" + +[[deps.CRlibm]] +deps = ["CRlibm_jll"] +git-tree-sha1 = "32abd86e3c2025db5172aa182b982debed519834" +uuid = "96374032-68de-5a5b-8d9e-752f78720389" +version = "1.0.1" + +[[deps.CRlibm_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e329286945d0cfc04456972ea732551869af1cfc" +uuid = "4e9b3aee-d8a1-5a3d-ad8b-7d824db253f0" +version = "1.0.1+0" + +[[deps.CSV]] +deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"] +git-tree-sha1 = "679e69c611fff422038e9e21e270c4197d49d918" +uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +version = "0.10.12" + +[[deps.Cairo]] +deps = ["Cairo_jll", "Colors", "Glib_jll", "Graphics", "Libdl", "Pango_jll"] +git-tree-sha1 = "d0b3f8b4ad16cb0a2988c6788646a5e6a17b6b1b" +uuid = "159f3aea-2a34-519c-b102-8c37f9878175" +version = "1.0.5" + +[[deps.CairoMakie]] +deps = ["CRC32c", "Cairo", "Colors", "FFTW", "FileIO", "FreeType", "GeometryBasics", "LinearAlgebra", "Makie", "PrecompileTools"] +git-tree-sha1 = "8af8e491c0809610cdec28c9c44b1f1dce01046d" +uuid = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +version = "0.11.6" + +[[deps.Cairo_jll]] +deps = ["Artifacts", "Bzip2_jll", "CompilerSupportLibraries_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "JLLWrappers", "LZO_jll", "Libdl", "Pixman_jll", "Pkg", "Xorg_libXext_jll", "Xorg_libXrender_jll", "Zlib_jll", "libpng_jll"] +git-tree-sha1 = "4b859a208b2397a7a623a03449e4636bdb17bcf2" +uuid = "83423d85-b0ee-5818-9007-b63ccbeb887a" +version = "1.16.1+1" + +[[deps.Calculus]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "f641eb0a4f00c343bbc32346e1217b86f3ce9dad" +uuid = "49dc2e85-a5d0-5ad3-a950-438e2897f1b9" +version = "0.5.1" + +[[deps.CategoricalArrays]] +deps = ["DataAPI", "Future", "Missings", "Printf", "Requires", "Statistics", "Unicode"] +git-tree-sha1 = "1568b28f91293458345dabba6a5ea3f183250a61" +uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597" +version = "0.10.8" + + [deps.CategoricalArrays.extensions] + CategoricalArraysJSONExt = "JSON" + CategoricalArraysRecipesBaseExt = "RecipesBase" + CategoricalArraysSentinelArraysExt = "SentinelArrays" + CategoricalArraysStructTypesExt = "StructTypes" + + [deps.CategoricalArrays.weakdeps] + JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" + RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" + SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c" + StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4" + +[[deps.Chain]] +git-tree-sha1 = "8c4920235f6c561e401dfe569beb8b924adad003" +uuid = "8be319e6-bccf-4806-a6f7-6fae938471bc" +version = "0.5.0" + +[[deps.ChainRules]] +deps = ["Adapt", "ChainRulesCore", "Compat", "Distributed", "GPUArraysCore", "IrrationalConstants", "LinearAlgebra", "Random", "RealDot", "SparseArrays", "SparseInverseSubset", "Statistics", "StructArrays", "SuiteSparse"] +git-tree-sha1 = "b308c249f38637cc5022c079fe5e19701a160395" +uuid = "082447d4-558c-5d27-93f4-14fc19e9eca2" +version = "1.60.0" + +[[deps.ChainRulesCore]] +deps = ["Compat", "LinearAlgebra"] +git-tree-sha1 = "0d12ee16b3f62e4e33c3277773730a5b21a74152" +uuid = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" +version = "1.20.0" +weakdeps = ["SparseArrays"] + + [deps.ChainRulesCore.extensions] + ChainRulesCoreSparseArraysExt = "SparseArrays" + +[[deps.ChangesOfVariables]] +deps = ["LinearAlgebra", "Test"] +git-tree-sha1 = "2fba81a302a7be671aefe194f0525ef231104e7f" +uuid = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0" +version = "0.1.8" +weakdeps = ["InverseFunctions"] + + [deps.ChangesOfVariables.extensions] + ChangesOfVariablesInverseFunctionsExt = "InverseFunctions" + +[[deps.CloseOpenIntervals]] +deps = ["Static", "StaticArrayInterface"] +git-tree-sha1 = "70232f82ffaab9dc52585e0dd043b5e0c6b714f1" +uuid = "fb6a15b2-703c-40df-9091-08a04967cfa9" +version = "0.1.12" + +[[deps.CodecBzip2]] +deps = ["Bzip2_jll", "Libdl", "TranscodingStreams"] +git-tree-sha1 = "c0ae2a86b162fb5d7acc65269b469ff5b8a73594" +uuid = "523fee87-0ab8-5b00-afb7-3ecf72e48cfd" +version = "0.8.1" + +[[deps.CodecZlib]] +deps = ["TranscodingStreams", "Zlib_jll"] +git-tree-sha1 = "cd67fc487743b2f0fd4380d4cbd3a24660d0eec8" +uuid = "944b1d66-785c-5afd-91f1-9de20f533193" +version = "0.7.3" + +[[deps.ColorBrewer]] +deps = ["Colors", "JSON", "Test"] +git-tree-sha1 = "61c5334f33d91e570e1d0c3eb5465835242582c4" +uuid = "a2cac450-b92f-5266-8821-25eda20663c8" +version = "0.4.0" + +[[deps.ColorSchemes]] +deps = ["ColorTypes", "ColorVectorSpace", "Colors", "FixedPointNumbers", "PrecompileTools", "Random"] +git-tree-sha1 = "67c1f244b991cad9b0aa4b7540fb758c2488b129" +uuid = "35d6a980-a343-548e-a6ea-1d62b119f2f4" +version = "3.24.0" + +[[deps.ColorTypes]] +deps = ["FixedPointNumbers", "Random"] +git-tree-sha1 = "eb7f0f8307f71fac7c606984ea5fb2817275d6e4" +uuid = "3da002f7-5984-5a60-b8a6-cbb66c0b333f" +version = "0.11.4" + +[[deps.ColorVectorSpace]] +deps = ["ColorTypes", "FixedPointNumbers", "LinearAlgebra", "Requires", "Statistics", "TensorCore"] +git-tree-sha1 = "a1f44953f2382ebb937d60dafbe2deea4bd23249" +uuid = "c3611d14-8923-5661-9e6a-0046d554d3a4" +version = "0.10.0" +weakdeps = ["SpecialFunctions"] + + [deps.ColorVectorSpace.extensions] + SpecialFunctionsExt = "SpecialFunctions" + +[[deps.Colors]] +deps = ["ColorTypes", "FixedPointNumbers", "Reexport"] +git-tree-sha1 = "fc08e5930ee9a4e03f84bfb5211cb54e7769758a" +uuid = "5ae59095-9a9b-59fe-a467-6f913c188581" +version = "0.12.10" + +[[deps.Combinatorics]] +git-tree-sha1 = "08c8b6831dc00bfea825826be0bc8336fc369860" +uuid = "861a8166-3701-5b0c-9a16-15d98fcdc6aa" +version = "1.0.2" + +[[deps.CommonSolve]] +git-tree-sha1 = "0eee5eb66b1cf62cd6ad1b460238e60e4b09400c" +uuid = "38540f10-b2f7-11e9-35d8-d573e4eb0ff2" +version = "0.2.4" + +[[deps.CommonSubexpressions]] +deps = ["MacroTools", "Test"] +git-tree-sha1 = "7b8a93dba8af7e3b42fecabf646260105ac373f7" +uuid = "bbf7d656-a473-5ed7-a52c-81e309532950" +version = "0.3.0" + +[[deps.Compat]] +deps = ["TOML", "UUIDs"] +git-tree-sha1 = "75bd5b6fc5089df449b5d35fa501c846c9b6549b" +uuid = "34da2185-b29b-5c13-b0c7-acf172513d20" +version = "4.12.0" +weakdeps = ["Dates", "LinearAlgebra"] + + [deps.Compat.extensions] + CompatLinearAlgebraExt = "LinearAlgebra" + +[[deps.CompilerSupportLibraries_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae" +version = "1.0.5+1" + +[[deps.CompositionsBase]] +git-tree-sha1 = "802bb88cd69dfd1509f6670416bd4434015693ad" +uuid = "a33af91c-f02d-484b-be07-31d278c5ca2b" +version = "0.1.2" +weakdeps = ["InverseFunctions"] + + [deps.CompositionsBase.extensions] + CompositionsBaseInverseFunctionsExt = "InverseFunctions" + +[[deps.ConcreteStructs]] +git-tree-sha1 = "f749037478283d372048690eb3b5f92a79432b34" +uuid = "2569d6c7-a4a2-43d3-a901-331e8e4be471" +version = "0.2.3" + +[[deps.ConcurrentUtilities]] +deps = ["Serialization", "Sockets"] +git-tree-sha1 = "8cfa272e8bdedfa88b6aefbbca7c19f1befac519" +uuid = "f0e56b4a-5159-44fe-b623-3e5288b988bb" +version = "2.3.0" + +[[deps.ConsoleProgressMonitor]] +deps = ["Logging", "ProgressMeter"] +git-tree-sha1 = "3ab7b2136722890b9af903859afcf457fa3059e8" +uuid = "88cd18e8-d9cc-4ea6-8889-5259c0d15c8b" +version = "0.1.2" + +[[deps.ConstructionBase]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "c53fc348ca4d40d7b371e71fd52251839080cbc9" +uuid = "187b0558-2788-49d3-abe0-74a17ed4e7c9" +version = "1.5.4" +weakdeps = ["IntervalSets", "StaticArrays"] + + [deps.ConstructionBase.extensions] + ConstructionBaseIntervalSetsExt = "IntervalSets" + ConstructionBaseStaticArraysExt = "StaticArrays" + +[[deps.Contour]] +git-tree-sha1 = "d05d9e7b7aedff4e5b51a029dced05cfb6125781" +uuid = "d38c429a-6771-53c6-b99e-75d170b6e991" +version = "0.6.2" + +[[deps.CpuId]] +deps = ["Markdown"] +git-tree-sha1 = "fcbb72b032692610bfbdb15018ac16a36cf2e406" +uuid = "adafc99b-e345-5852-983c-f28acb93d879" +version = "0.3.1" + +[[deps.Crayons]] +git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15" +uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f" +version = "4.1.1" + +[[deps.DataAPI]] +git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe" +uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a" +version = "1.16.0" + +[[deps.DataFrames]] +deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "REPL", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"] +git-tree-sha1 = "04c738083f29f86e62c8afc341f0967d8717bdb8" +uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +version = "1.6.1" + +[[deps.DataStructures]] +deps = ["Compat", "InteractiveUtils", "OrderedCollections"] +git-tree-sha1 = "ac67408d9ddf207de5cfa9a97e114352430f01ed" +uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8" +version = "0.18.16" + +[[deps.DataValueInterfaces]] +git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6" +uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464" +version = "1.0.0" + +[[deps.Dates]] +deps = ["Printf"] +uuid = "ade2ca70-3891-5945-98fb-dc099432e06a" + +[[deps.DefineSingletons]] +git-tree-sha1 = "0fba8b706d0178b4dc7fd44a96a92382c9065c2c" +uuid = "244e2a9f-e319-4986-a169-4d1fe445cd52" +version = "0.1.2" + +[[deps.DelaunayTriangulation]] +deps = ["DataStructures", "EnumX", "ExactPredicates", "Random", "SimpleGraphs"] +git-tree-sha1 = "26eb8e2331b55735c3d305d949aabd7363f07ba7" +uuid = "927a84f5-c5f4-47a5-9785-b46e178433df" +version = "0.8.11" + +[[deps.DelayDiffEq]] +deps = ["ArrayInterface", "DataStructures", "DiffEqBase", "LinearAlgebra", "Logging", "OrdinaryDiffEq", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SimpleNonlinearSolve", "SimpleUnPack"] +git-tree-sha1 = "6725c56e3e3d563e37d8fd5e6c5eb66ac19321fd" +uuid = "bcd4f6db-9728-5f36-b5f7-82caef46ccdb" +version = "5.46.0" + +[[deps.DelimitedFiles]] +deps = ["Mmap"] +git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae" +uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab" +version = "1.9.1" + +[[deps.DensityInterface]] +deps = ["InverseFunctions", "Test"] +git-tree-sha1 = "80c3e8639e3353e5d2912fb3a1916b8455e2494b" +uuid = "b429d917-457f-4dbc-8f4c-0cc954292b1d" +version = "0.4.0" + +[[deps.Dictionaries]] +deps = ["Indexing", "Random", "Serialization"] +git-tree-sha1 = "bd57fc07ff32179367a230b22c539e4776e452e2" +uuid = "85a47980-9c8c-11e8-2b9f-f7ca1fa99fb4" +version = "0.3.28" + +[[deps.DiffEqBase]] +deps = ["ArrayInterface", "DataStructures", "DocStringExtensions", "EnumX", "EnzymeCore", "FastBroadcast", "ForwardDiff", "FunctionWrappers", "FunctionWrappersWrappers", "LinearAlgebra", "Logging", "Markdown", "MuladdMacro", "Parameters", "PreallocationTools", "PrecompileTools", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Static", "StaticArraysCore", "Statistics", "Tricks", "TruncatedStacktraces"] +git-tree-sha1 = "6af33c2eb7478db06bcf5c810e6f3dda53aac2ac" +uuid = "2b5f629d-d688-5b77-993f-72d75c75574e" +version = "6.146.0" + + [deps.DiffEqBase.extensions] + DiffEqBaseChainRulesCoreExt = "ChainRulesCore" + DiffEqBaseDistributionsExt = "Distributions" + DiffEqBaseEnzymeExt = ["ChainRulesCore", "Enzyme"] + DiffEqBaseGeneralizedGeneratedExt = "GeneralizedGenerated" + DiffEqBaseMPIExt = "MPI" + DiffEqBaseMeasurementsExt = "Measurements" + DiffEqBaseMonteCarloMeasurementsExt = "MonteCarloMeasurements" + DiffEqBaseReverseDiffExt = "ReverseDiff" + DiffEqBaseTrackerExt = "Tracker" + DiffEqBaseUnitfulExt = "Unitful" + + [deps.DiffEqBase.weakdeps] + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + GeneralizedGenerated = "6b9d7cbe-bcb9-11e9-073f-15a7a543e2eb" + MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195" + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.DiffEqCallbacks]] +deps = ["DataStructures", "DiffEqBase", "ForwardDiff", "Functors", "LinearAlgebra", "Markdown", "NLsolve", "Parameters", "RecipesBase", "RecursiveArrayTools", "SciMLBase", "StaticArraysCore"] +git-tree-sha1 = "cf334da651a6e42c50e1477d6ab978f1b8be3057" +uuid = "459566f4-90b8-5000-8ac3-15dfb0a30def" +version = "2.36.1" +weakdeps = ["OrdinaryDiffEq", "Sundials"] + +[[deps.DiffEqNoiseProcess]] +deps = ["DiffEqBase", "Distributions", "GPUArraysCore", "LinearAlgebra", "Markdown", "Optim", "PoissonRandom", "QuadGK", "Random", "Random123", "RandomNumbers", "RecipesBase", "RecursiveArrayTools", "Requires", "ResettableStacks", "SciMLBase", "StaticArraysCore", "Statistics"] +git-tree-sha1 = "319377c927a4aa1f491228b2ac23f3554a3497c6" +uuid = "77a26b50-5914-5dd7-bc55-306e6241c503" +version = "5.20.0" + + [deps.DiffEqNoiseProcess.extensions] + DiffEqNoiseProcessReverseDiffExt = "ReverseDiff" + + [deps.DiffEqNoiseProcess.weakdeps] + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + +[[deps.DiffResults]] +deps = ["StaticArraysCore"] +git-tree-sha1 = "782dd5f4561f5d267313f23853baaaa4c52ea621" +uuid = "163ba53b-c6d8-5494-b064-1a9d43ac40c5" +version = "1.1.0" + +[[deps.DiffRules]] +deps = ["IrrationalConstants", "LogExpFunctions", "NaNMath", "Random", "SpecialFunctions"] +git-tree-sha1 = "23163d55f885173722d1e4cf0f6110cdbaf7e272" +uuid = "b552c78f-8df3-52c6-915a-8e097449b14b" +version = "1.15.1" + +[[deps.DifferentialEquations]] +deps = ["BoundaryValueDiffEq", "DelayDiffEq", "DiffEqBase", "DiffEqCallbacks", "DiffEqNoiseProcess", "JumpProcesses", "LinearAlgebra", "LinearSolve", "NonlinearSolve", "OrdinaryDiffEq", "Random", "RecursiveArrayTools", "Reexport", "SciMLBase", "SteadyStateDiffEq", "StochasticDiffEq", "Sundials"] +git-tree-sha1 = "8864b6a953eeba7890d23258aca468d90ca73fd6" +uuid = "0c46a032-eb83-5123-abaf-570d42b7fbaa" +version = "7.12.0" + +[[deps.Distances]] +deps = ["LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "66c4c81f259586e8f002eacebc177e1fb06363b0" +uuid = "b4f34e82-e78d-54a5-968a-f98e89d6e8f7" +version = "0.10.11" +weakdeps = ["ChainRulesCore", "SparseArrays"] + + [deps.Distances.extensions] + DistancesChainRulesCoreExt = "ChainRulesCore" + DistancesSparseArraysExt = "SparseArrays" + +[[deps.Distributed]] +deps = ["Random", "Serialization", "Sockets"] +uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b" + +[[deps.Distributions]] +deps = ["FillArrays", "LinearAlgebra", "PDMats", "Printf", "QuadGK", "Random", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "7c302d7a5fec5214eb8a5a4c466dcf7a51fcf169" +uuid = "31c24e10-a181-5473-b8eb-7969acd0382f" +version = "0.25.107" +weakdeps = ["ChainRulesCore", "DensityInterface", "Test"] + + [deps.Distributions.extensions] + DistributionsChainRulesCoreExt = "ChainRulesCore" + DistributionsDensityInterfaceExt = "DensityInterface" + DistributionsTestExt = "Test" + +[[deps.DistributionsAD]] +deps = ["Adapt", "ChainRules", "ChainRulesCore", "Compat", "Distributions", "FillArrays", "LinearAlgebra", "PDMats", "Random", "Requires", "SpecialFunctions", "StaticArrays", "StatsFuns", "ZygoteRules"] +git-tree-sha1 = "d61f08c7bd15c5ab215fd7a2eb61c1ae15d8ff5e" +uuid = "ced4e74d-a319-5a8a-b0ac-84af2272839c" +version = "0.6.53" + + [deps.DistributionsAD.extensions] + DistributionsADForwardDiffExt = "ForwardDiff" + DistributionsADLazyArraysExt = "LazyArrays" + DistributionsADReverseDiffExt = "ReverseDiff" + DistributionsADTrackerExt = "Tracker" + + [deps.DistributionsAD.weakdeps] + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + +[[deps.DocStringExtensions]] +deps = ["LibGit2"] +git-tree-sha1 = "2fb1e02f2b635d0845df5d7c167fec4dd739b00d" +uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae" +version = "0.9.3" + +[[deps.Downloads]] +deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"] +uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +version = "1.6.0" + +[[deps.DualNumbers]] +deps = ["Calculus", "NaNMath", "SpecialFunctions"] +git-tree-sha1 = "5837a837389fccf076445fce071c8ddaea35a566" +uuid = "fa6b7ba4-c1ee-5f82-b5fc-ecf0adba8f74" +version = "0.6.8" + +[[deps.DynamicPPL]] +deps = ["AbstractMCMC", "AbstractPPL", "BangBang", "Bijectors", "Compat", "ConstructionBase", "Distributions", "DocStringExtensions", "LinearAlgebra", "LogDensityProblems", "MacroTools", "OrderedCollections", "Random", "Requires", "Setfield", "Test"] +git-tree-sha1 = "e1435190e3bc4870dcd1beac77624caa85016600" +uuid = "366bfd00-2699-11ea-058f-f148b4cae6d8" +version = "0.24.5" +weakdeps = ["ChainRulesCore", "EnzymeCore", "MCMCChains", "ZygoteRules"] + + [deps.DynamicPPL.extensions] + DynamicPPLChainRulesCoreExt = ["ChainRulesCore"] + DynamicPPLEnzymeCoreExt = ["EnzymeCore"] + DynamicPPLMCMCChainsExt = ["MCMCChains"] + DynamicPPLZygoteRulesExt = ["ZygoteRules"] + +[[deps.EarCut_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e3290f2d49e661fbd94046d7e3726ffcb2d41053" +uuid = "5ae413db-bbd1-5e63-b57d-d24a61df00f5" +version = "2.2.4+0" + +[[deps.EllipticalSliceSampling]] +deps = ["AbstractMCMC", "ArrayInterface", "Distributions", "Random", "Statistics"] +git-tree-sha1 = "e611b7fdfbfb5b18d5e98776c30daede41b44542" +uuid = "cad2338a-1db2-11e9-3401-43bc07c9ede2" +version = "2.0.0" + +[[deps.EnumX]] +git-tree-sha1 = "bdb1942cd4c45e3c678fd11569d5cccd80976237" +uuid = "4e289a0a-7415-4d19-859d-a7e5c4648b56" +version = "1.0.4" + +[[deps.EnzymeCore]] +git-tree-sha1 = "59c44d8fbc651c0395d8a6eda64b05ce316f58b4" +uuid = "f151be2c-9106-41f4-ab19-57ee4f262869" +version = "0.6.5" +weakdeps = ["Adapt"] + + [deps.EnzymeCore.extensions] + AdaptExt = "Adapt" + +[[deps.ExactPredicates]] +deps = ["IntervalArithmetic", "Random", "StaticArrays"] +git-tree-sha1 = "e8b8c949551f417e040f16e5c431b6e83e306e54" +uuid = "429591f6-91af-11e9-00e2-59fbe8cec110" +version = "2.2.7" + +[[deps.ExceptionUnwrapping]] +deps = ["Test"] +git-tree-sha1 = "dcb08a0d93ec0b1cdc4af184b26b591e9695423a" +uuid = "460bff9d-24e4-43bc-9d9f-a8973cb893f4" +version = "0.1.10" + +[[deps.Expat_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "4558ab818dcceaab612d1bb8c19cee87eda2b83c" +uuid = "2e619515-83b5-522b-bb60-26c02a35a201" +version = "2.5.0+0" + +[[deps.ExponentialUtilities]] +deps = ["Adapt", "ArrayInterface", "GPUArraysCore", "GenericSchur", "LinearAlgebra", "PrecompileTools", "Printf", "SparseArrays", "libblastrampoline_jll"] +git-tree-sha1 = "602e4585bcbd5a25bc06f514724593d13ff9e862" +uuid = "d4d017d3-3776-5f7e-afef-a10c40355c18" +version = "1.25.0" + +[[deps.ExprTools]] +git-tree-sha1 = "27415f162e6028e81c72b82ef756bf321213b6ec" +uuid = "e2ba6199-217a-4e67-a87a-7c52f15ade04" +version = "0.1.10" + +[[deps.Extents]] +git-tree-sha1 = "2140cd04483da90b2da7f99b2add0750504fc39c" +uuid = "411431e0-e8b7-467b-b5e0-f676ba4f2910" +version = "0.1.2" + +[[deps.FFMPEG_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "JLLWrappers", "LAME_jll", "Libdl", "Ogg_jll", "OpenSSL_jll", "Opus_jll", "PCRE2_jll", "Zlib_jll", "libaom_jll", "libass_jll", "libfdk_aac_jll", "libvorbis_jll", "x264_jll", "x265_jll"] +git-tree-sha1 = "466d45dc38e15794ec7d5d63ec03d776a9aff36e" +uuid = "b22a6f82-2f65-5046-a5b2-351ab43fb4e5" +version = "4.4.4+1" + +[[deps.FFTW]] +deps = ["AbstractFFTs", "FFTW_jll", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"] +git-tree-sha1 = "4820348781ae578893311153d69049a93d05f39d" +uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341" +version = "1.8.0" + +[[deps.FFTW_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c6033cc3892d0ef5bb9cd29b7f2f0331ea5184ea" +uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a" +version = "3.3.10+0" + +[[deps.FastAlmostBandedMatrices]] +deps = ["ArrayInterface", "ArrayLayouts", "BandedMatrices", "ConcreteStructs", "LazyArrays", "LinearAlgebra", "MatrixFactorizations", "PrecompileTools", "Reexport"] +git-tree-sha1 = "178316d87f883f0702e79d9c83a8049484c9f619" +uuid = "9d29842c-ecb8-4973-b1e9-a27b1157504e" +version = "0.1.0" + +[[deps.FastBroadcast]] +deps = ["ArrayInterface", "LinearAlgebra", "Polyester", "Static", "StaticArrayInterface", "StrideArraysCore"] +git-tree-sha1 = "a6e756a880fc419c8b41592010aebe6a5ce09136" +uuid = "7034ab61-46d4-4ed7-9d0f-46aef9175898" +version = "0.2.8" + +[[deps.FastClosures]] +git-tree-sha1 = "acebe244d53ee1b461970f8910c235b259e772ef" +uuid = "9aa1b823-49e4-5ca5-8b0f-3971ec8bab6a" +version = "0.3.2" + +[[deps.FastLapackInterface]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "d576a29bf8bcabf4b1deb9abe88a3d7f78306ab5" +uuid = "29a986be-02c6-4525-aec4-84b980013641" +version = "2.0.1" + +[[deps.FileIO]] +deps = ["Pkg", "Requires", "UUIDs"] +git-tree-sha1 = "c5c28c245101bd59154f649e19b038d15901b5dc" +uuid = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" +version = "1.16.2" + +[[deps.FilePaths]] +deps = ["FilePathsBase", "MacroTools", "Reexport", "Requires"] +git-tree-sha1 = "919d9412dbf53a2e6fe74af62a73ceed0bce0629" +uuid = "8fc22ac5-c921-52a6-82fd-178b2807b824" +version = "0.8.3" + +[[deps.FilePathsBase]] +deps = ["Compat", "Dates", "Mmap", "Printf", "Test", "UUIDs"] +git-tree-sha1 = "9f00e42f8d99fdde64d40c8ea5d14269a2e2c1aa" +uuid = "48062228-2e41-5def-b9a4-89aafe57970f" +version = "0.9.21" + +[[deps.FileWatching]] +uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee" + +[[deps.FillArrays]] +deps = ["LinearAlgebra", "Random"] +git-tree-sha1 = "5b93957f6dcd33fc343044af3d48c215be2562f1" +uuid = "1a297f60-69ca-5386-bcde-b61e274b549b" +version = "1.9.3" +weakdeps = ["PDMats", "SparseArrays", "Statistics"] + + [deps.FillArrays.extensions] + FillArraysPDMatsExt = "PDMats" + FillArraysSparseArraysExt = "SparseArrays" + FillArraysStatisticsExt = "Statistics" + +[[deps.FiniteDiff]] +deps = ["ArrayInterface", "LinearAlgebra", "Requires", "Setfield", "SparseArrays"] +git-tree-sha1 = "73d1214fec245096717847c62d389a5d2ac86504" +uuid = "6a86dc24-6348-571c-b903-95158fe2bd41" +version = "2.22.0" + + [deps.FiniteDiff.extensions] + FiniteDiffBandedMatricesExt = "BandedMatrices" + FiniteDiffBlockBandedMatricesExt = "BlockBandedMatrices" + FiniteDiffStaticArraysExt = "StaticArrays" + + [deps.FiniteDiff.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" + BlockBandedMatrices = "ffab5731-97b5-5995-9138-79e8c1846df0" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.FixedPointNumbers]] +deps = ["Statistics"] +git-tree-sha1 = "335bfdceacc84c5cdf16aadc768aa5ddfc5383cc" +uuid = "53c48c17-4a7d-5ca2-90c5-79b7896eea93" +version = "0.8.4" + +[[deps.Fontconfig_jll]] +deps = ["Artifacts", "Bzip2_jll", "Expat_jll", "FreeType2_jll", "JLLWrappers", "Libdl", "Libuuid_jll", "Pkg", "Zlib_jll"] +git-tree-sha1 = "21efd19106a55620a188615da6d3d06cd7f6ee03" +uuid = "a3f928ae-7b40-5064-980b-68af3947d34b" +version = "2.13.93+0" + +[[deps.Formatting]] +deps = ["Printf"] +git-tree-sha1 = "8339d61043228fdd3eb658d86c926cb282ae72a8" +uuid = "59287772-0a20-5a39-b81b-1366585eb4c0" +version = "0.4.2" + +[[deps.ForwardDiff]] +deps = ["CommonSubexpressions", "DiffResults", "DiffRules", "LinearAlgebra", "LogExpFunctions", "NaNMath", "Preferences", "Printf", "Random", "SpecialFunctions"] +git-tree-sha1 = "cf0fe81336da9fb90944683b8c41984b08793dad" +uuid = "f6369f11-7733-5829-9624-2563aa707210" +version = "0.10.36" +weakdeps = ["StaticArrays"] + + [deps.ForwardDiff.extensions] + ForwardDiffStaticArraysExt = "StaticArrays" + +[[deps.Franklin]] +deps = ["Dates", "DelimitedFiles", "DocStringExtensions", "ExprTools", "FranklinTemplates", "HTTP", "Literate", "LiveServer", "Logging", "Markdown", "NodeJS", "OrderedCollections", "Pkg", "REPL", "Random", "TOML"] +git-tree-sha1 = "31e70717e0640d6576fe04d611a33df1c9c312d6" +uuid = "713c75ef-9fc9-4b05-94a9-213340da978e" +version = "0.10.95" + +[[deps.FranklinTemplates]] +deps = ["LiveServer"] +git-tree-sha1 = "c01813a615149ddb3b3d133f33de29d642fbe57b" +uuid = "3a985190-f512-4703-8d38-2a7944ed5916" +version = "0.10.2" + +[[deps.FreeType]] +deps = ["CEnum", "FreeType2_jll"] +git-tree-sha1 = "907369da0f8e80728ab49c1c7e09327bf0d6d999" +uuid = "b38be410-82b0-50bf-ab77-7b57e271db43" +version = "4.1.1" + +[[deps.FreeType2_jll]] +deps = ["Artifacts", "Bzip2_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "d8db6a5a2fe1381c1ea4ef2cab7c69c2de7f9ea0" +uuid = "d7e528f0-a631-5988-bf34-fe36492bcfd7" +version = "2.13.1+0" + +[[deps.FreeTypeAbstraction]] +deps = ["ColorVectorSpace", "Colors", "FreeType", "GeometryBasics"] +git-tree-sha1 = "055626e1a35f6771fe99060e835b72ca61a52621" +uuid = "663a7486-cb36-511b-a19d-713bb74d65c9" +version = "0.10.1" + +[[deps.FriBidi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "aa31987c2ba8704e23c6c8ba8a4f769d5d7e4f91" +uuid = "559328eb-81f9-559d-9380-de523a88c83c" +version = "1.0.10+0" + +[[deps.FunctionWrappers]] +git-tree-sha1 = "d62485945ce5ae9c0c48f124a84998d755bae00e" +uuid = "069b7b12-0de2-55c6-9aab-29f3d0a68a2e" +version = "1.1.3" + +[[deps.FunctionWrappersWrappers]] +deps = ["FunctionWrappers"] +git-tree-sha1 = "b104d487b34566608f8b4e1c39fb0b10aa279ff8" +uuid = "77dc65aa-8811-40c2-897b-53d922fa7daf" +version = "0.1.3" + +[[deps.Functors]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9a68d75d466ccc1218d0552a8e1631151c569545" +uuid = "d9f16b24-f501-4c13-a1f2-28368ffc5196" +version = "0.4.5" + +[[deps.Future]] +deps = ["Random"] +uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820" + +[[deps.GLM]] +deps = ["Distributions", "LinearAlgebra", "Printf", "Reexport", "SparseArrays", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns", "StatsModels"] +git-tree-sha1 = "273bd1cd30768a2fddfa3fd63bbc746ed7249e5f" +uuid = "38e38edf-8417-5370-95a0-9cbb8c7f171a" +version = "1.9.0" + +[[deps.GPUArraysCore]] +deps = ["Adapt"] +git-tree-sha1 = "2d6ca471a6c7b536127afccfa7564b5b39227fe0" +uuid = "46192b85-c4d5-4398-a991-12ede77f4527" +version = "0.1.5" + +[[deps.GenericSchur]] +deps = ["LinearAlgebra", "Printf"] +git-tree-sha1 = "fb69b2a645fa69ba5f474af09221b9308b160ce6" +uuid = "c145ed77-6b09-5dd9-b285-bf645a82121e" +version = "0.5.3" + +[[deps.GeoInterface]] +deps = ["Extents"] +git-tree-sha1 = "d4f85701f569584f2cff7ba67a137d03f0cfb7d0" +uuid = "cf35fbd7-0cd7-5166-be24-54bfbe79505f" +version = "1.3.3" + +[[deps.GeometryBasics]] +deps = ["EarCut_jll", "Extents", "GeoInterface", "IterTools", "LinearAlgebra", "StaticArrays", "StructArrays", "Tables"] +git-tree-sha1 = "5694b56ccf9d15addedc35e9a4ba9c317721b788" +uuid = "5c1252a2-5f33-56bf-86c9-59e7332b4326" +version = "0.4.10" + +[[deps.Gettext_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Libiconv_jll", "Pkg", "XML2_jll"] +git-tree-sha1 = "9b02998aba7bf074d14de89f9d37ca24a1a0b046" +uuid = "78b55507-aeef-58d4-861c-77aaff3498b1" +version = "0.21.0+0" + +[[deps.Glib_jll]] +deps = ["Artifacts", "Gettext_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Libiconv_jll", "Libmount_jll", "PCRE2_jll", "Zlib_jll"] +git-tree-sha1 = "e94c92c7bf4819685eb80186d51c43e71d4afa17" +uuid = "7746bdde-850d-59dc-9ae8-88ece973131d" +version = "2.76.5+0" + +[[deps.Graphics]] +deps = ["Colors", "LinearAlgebra", "NaNMath"] +git-tree-sha1 = "d61890399bc535850c4bf08e4e0d3a7ad0f21cbd" +uuid = "a2bd30eb-e257-5431-a919-1863eab51364" +version = "1.1.2" + +[[deps.Graphite2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "344bf40dcab1073aca04aa0df4fb092f920e4011" +uuid = "3b182d85-2403-5c21-9c21-1e1f0cc25472" +version = "1.3.14+0" + +[[deps.Graphs]] +deps = ["ArnoldiMethod", "Compat", "DataStructures", "Distributed", "Inflate", "LinearAlgebra", "Random", "SharedArrays", "SimpleTraits", "SparseArrays", "Statistics"] +git-tree-sha1 = "899050ace26649433ef1af25bc17a815b3db52b7" +uuid = "86223c79-3864-5bf0-83f7-82e725a168b6" +version = "1.9.0" + +[[deps.GridLayoutBase]] +deps = ["GeometryBasics", "InteractiveUtils", "Observables"] +git-tree-sha1 = "af13a277efd8a6e716d79ef635d5342ccb75be61" +uuid = "3955a311-db13-416c-9275-1d80ed98e5e9" +version = "0.10.0" + +[[deps.Grisu]] +git-tree-sha1 = "53bb909d1151e57e2484c3d1b53e19552b887fb2" +uuid = "42e2da0e-8278-4e71-bc24-59509adca0fe" +version = "1.0.2" + +[[deps.HTTP]] +deps = ["Base64", "CodecZlib", "ConcurrentUtilities", "Dates", "ExceptionUnwrapping", "Logging", "LoggingExtras", "MbedTLS", "NetworkOptions", "OpenSSL", "Random", "SimpleBufferStream", "Sockets", "URIs", "UUIDs"] +git-tree-sha1 = "abbbb9ec3afd783a7cbd82ef01dcd088ea051398" +uuid = "cd3eb016-35fb-5094-929b-558a96fad6f3" +version = "1.10.1" + +[[deps.HarfBuzz_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "Glib_jll", "Graphite2_jll", "JLLWrappers", "Libdl", "Libffi_jll", "Pkg"] +git-tree-sha1 = "129acf094d168394e80ee1dc4bc06ec835e510a3" +uuid = "2e76f6c2-a576-52d4-95c1-20adfe4de566" +version = "2.8.1+1" + +[[deps.HostCPUFeatures]] +deps = ["BitTwiddlingConvenienceFunctions", "IfElse", "Libdl", "Static"] +git-tree-sha1 = "eb8fed28f4994600e29beef49744639d985a04b2" +uuid = "3e5b6fbb-0976-4d2c-9146-d79de83f2fb0" +version = "0.1.16" + +[[deps.HypergeometricFunctions]] +deps = ["DualNumbers", "LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"] +git-tree-sha1 = "f218fe3736ddf977e0e772bc9a586b2383da2685" +uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a" +version = "0.3.23" + +[[deps.IOCapture]] +deps = ["Logging", "Random"] +git-tree-sha1 = "8b72179abc660bfab5e28472e019392b97d0985c" +uuid = "b5f81e59-6552-4d32-b1f0-c071b021bf89" +version = "0.2.4" + +[[deps.IfElse]] +git-tree-sha1 = "debdd00ffef04665ccbb3e150747a77560e8fad1" +uuid = "615f187c-cbe4-4ef1-ba3b-2fcf58d6d173" +version = "0.1.1" + +[[deps.ImageAxes]] +deps = ["AxisArrays", "ImageBase", "ImageCore", "Reexport", "SimpleTraits"] +git-tree-sha1 = "2e4520d67b0cef90865b3ef727594d2a58e0e1f8" +uuid = "2803e5a7-5153-5ecf-9a86-9b4c37f5f5ac" +version = "0.6.11" + +[[deps.ImageBase]] +deps = ["ImageCore", "Reexport"] +git-tree-sha1 = "eb49b82c172811fd2c86759fa0553a2221feb909" +uuid = "c817782e-172a-44cc-b673-b171935fbb9e" +version = "0.1.7" + +[[deps.ImageCore]] +deps = ["ColorVectorSpace", "Colors", "FixedPointNumbers", "MappedArrays", "MosaicViews", "OffsetArrays", "PaddedViews", "PrecompileTools", "Reexport"] +git-tree-sha1 = "b2a7eaa169c13f5bcae8131a83bc30eff8f71be0" +uuid = "a09fc81d-aa75-5fe9-8630-4744c3626534" +version = "0.10.2" + +[[deps.ImageIO]] +deps = ["FileIO", "IndirectArrays", "JpegTurbo", "LazyModules", "Netpbm", "OpenEXR", "PNGFiles", "QOI", "Sixel", "TiffImages", "UUIDs"] +git-tree-sha1 = "bca20b2f5d00c4fbc192c3212da8fa79f4688009" +uuid = "82e4d734-157c-48bb-816b-45c225c6df19" +version = "0.6.7" + +[[deps.ImageMetadata]] +deps = ["AxisArrays", "ImageAxes", "ImageBase", "ImageCore"] +git-tree-sha1 = "355e2b974f2e3212a75dfb60519de21361ad3cb7" +uuid = "bc367c6b-8a6b-528e-b4bd-a4b897500b49" +version = "0.9.9" + +[[deps.Imath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "3d09a9f60edf77f8a4d99f9e015e8fbf9989605d" +uuid = "905a6f67-0a94-5f89-b386-d35d92009cd1" +version = "3.1.7+0" + +[[deps.Indexing]] +git-tree-sha1 = "ce1566720fd6b19ff3411404d4b977acd4814f9f" +uuid = "313cdc1a-70c2-5d6a-ae34-0150d3930a38" +version = "1.1.1" + +[[deps.IndirectArrays]] +git-tree-sha1 = "012e604e1c7458645cb8b436f8fba789a51b257f" +uuid = "9b13fd28-a010-5f03-acff-a1bbcff69959" +version = "1.0.0" + +[[deps.Inflate]] +git-tree-sha1 = "ea8031dea4aff6bd41f1df8f2fdfb25b33626381" +uuid = "d25df0c9-e2be-5dd7-82c8-3ad0b3e990b9" +version = "0.1.4" + +[[deps.InitialValues]] +git-tree-sha1 = "4da0f88e9a39111c2fa3add390ab15f3a44f3ca3" +uuid = "22cec73e-a1b8-11e9-2c92-598750a2cf9c" +version = "0.3.1" + +[[deps.InlineStrings]] +deps = ["Parsers"] +git-tree-sha1 = "9cc2baf75c6d09f9da536ddf58eb2f29dedaf461" +uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48" +version = "1.4.0" + +[[deps.InplaceOps]] +deps = ["LinearAlgebra", "Test"] +git-tree-sha1 = "50b41d59e7164ab6fda65e71049fee9d890731ff" +uuid = "505f98c9-085e-5b2c-8e89-488be7bf1f34" +version = "0.3.0" + +[[deps.IntegerMathUtils]] +git-tree-sha1 = "b8ffb903da9f7b8cf695a8bead8e01814aa24b30" +uuid = "18e54dd8-cb9d-406c-a71d-865a43cbb235" +version = "0.1.2" + +[[deps.IntelOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "5fdf2fe6724d8caabf43b557b84ce53f3b7e2f6b" +uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0" +version = "2024.0.2+0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" + +[[deps.Interpolations]] +deps = ["Adapt", "AxisAlgorithms", "ChainRulesCore", "LinearAlgebra", "OffsetArrays", "Random", "Ratios", "Requires", "SharedArrays", "SparseArrays", "StaticArrays", "WoodburyMatrices"] +git-tree-sha1 = "88a101217d7cb38a7b481ccd50d21876e1d1b0e0" +uuid = "a98d9a8b-a2ab-59e6-89dd-64a1c18fca59" +version = "0.15.1" + + [deps.Interpolations.extensions] + InterpolationsUnitfulExt = "Unitful" + + [deps.Interpolations.weakdeps] + Unitful = "1986cc42-f94f-5a68-af5c-568840ba703d" + +[[deps.IntervalArithmetic]] +deps = ["CRlibm", "RoundingEmulator"] +git-tree-sha1 = "c274ec586ea58eb7b42afd0c5d67e50ff50229b5" +uuid = "d1acc4aa-44c8-5952-acd4-ba5d80a2a253" +version = "0.22.5" +weakdeps = ["DiffRules", "RecipesBase"] + + [deps.IntervalArithmetic.extensions] + IntervalArithmeticDiffRulesExt = "DiffRules" + IntervalArithmeticRecipesBaseExt = "RecipesBase" + +[[deps.IntervalSets]] +deps = ["Dates", "Random"] +git-tree-sha1 = "3d8866c029dd6b16e69e0d4a939c4dfcb98fac47" +uuid = "8197267c-284f-5f27-9208-e0e47529a953" +version = "0.7.8" +weakdeps = ["Statistics"] + + [deps.IntervalSets.extensions] + IntervalSetsStatisticsExt = "Statistics" + +[[deps.InverseFunctions]] +deps = ["Test"] +git-tree-sha1 = "68772f49f54b479fa88ace904f6127f0a3bb2e46" +uuid = "3587e190-3f89-42d0-90ee-14403ec27112" +version = "0.1.12" + +[[deps.InvertedIndices]] +git-tree-sha1 = "0dc7b50b8d436461be01300fd8cd45aa0274b038" +uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f" +version = "1.3.0" + +[[deps.IrrationalConstants]] +git-tree-sha1 = "630b497eafcc20001bba38a4651b327dcfc491d2" +uuid = "92d709cd-6900-40b7-9082-c6be49f344b6" +version = "0.2.2" + +[[deps.Isoband]] +deps = ["isoband_jll"] +git-tree-sha1 = "f9b6d97355599074dc867318950adaa6f9946137" +uuid = "f1662d9f-8043-43de-a69a-05efc1cc6ff4" +version = "0.1.1" + +[[deps.IterTools]] +git-tree-sha1 = "42d5f897009e7ff2cf88db414a389e5ed1bdd023" +uuid = "c8e1da08-722c-5040-9ed9-7db0dc04731e" +version = "1.10.0" + +[[deps.IteratorInterfaceExtensions]] +git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856" +uuid = "82899510-4779-5014-852e-03e436cf321d" +version = "1.0.0" + +[[deps.JLLWrappers]] +deps = ["Artifacts", "Preferences"] +git-tree-sha1 = "7e5d6779a1e09a36db2a7b6cff50942a0a7d0fca" +uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210" +version = "1.5.0" + +[[deps.JSON]] +deps = ["Dates", "Mmap", "Parsers", "Unicode"] +git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a" +uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6" +version = "0.21.4" + +[[deps.JpegTurbo]] +deps = ["CEnum", "FileIO", "ImageCore", "JpegTurbo_jll", "TOML"] +git-tree-sha1 = "fa6d0bcff8583bac20f1ffa708c3913ca605c611" +uuid = "b835a17e-a41a-41e7-81f0-2f016b05efe0" +version = "0.1.5" + +[[deps.JpegTurbo_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "60b1194df0a3298f460063de985eae7b01bc011a" +uuid = "aacddb02-875f-59d6-b918-886e6ef4fbf8" +version = "3.0.1+0" + +[[deps.JumpProcesses]] +deps = ["ArrayInterface", "DataStructures", "DiffEqBase", "DocStringExtensions", "FunctionWrappers", "Graphs", "LinearAlgebra", "Markdown", "PoissonRandom", "Random", "RandomNumbers", "RecursiveArrayTools", "Reexport", "SciMLBase", "StaticArrays", "UnPack"] +git-tree-sha1 = "c451feb97251965a9fe40bacd62551a72cc5902c" +uuid = "ccbc3e58-028d-4f4c-8cd5-9ae44345cda5" +version = "9.10.1" +weakdeps = ["FastBroadcast"] + + [deps.JumpProcesses.extensions] + JumpProcessFastBroadcastExt = "FastBroadcast" + +[[deps.KLU]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse_jll"] +git-tree-sha1 = "884c2968c2e8e7e6bf5956af88cb46aa745c854b" +uuid = "ef3ab10e-7fda-4108-b977-705223b18434" +version = "0.4.1" + +[[deps.KernelAbstractions]] +deps = ["Adapt", "Atomix", "InteractiveUtils", "LinearAlgebra", "MacroTools", "PrecompileTools", "Requires", "SparseArrays", "StaticArrays", "UUIDs", "UnsafeAtomics", "UnsafeAtomicsLLVM"] +git-tree-sha1 = "4e0cb2f5aad44dcfdc91088e85dee4ecb22c791c" +uuid = "63c18a36-062a-441e-b654-da1e3ab1ce7c" +version = "0.9.16" +weakdeps = ["EnzymeCore"] + + [deps.KernelAbstractions.extensions] + EnzymeExt = "EnzymeCore" + +[[deps.KernelDensity]] +deps = ["Distributions", "DocStringExtensions", "FFTW", "Interpolations", "StatsBase"] +git-tree-sha1 = "fee018a29b60733876eb557804b5b109dd3dd8a7" +uuid = "5ab0869b-81aa-558d-bb23-cbf5423bbe9b" +version = "0.6.8" + +[[deps.Krylov]] +deps = ["LinearAlgebra", "Printf", "SparseArrays"] +git-tree-sha1 = "8a6837ec02fe5fb3def1abc907bb802ef11a0729" +uuid = "ba0b0d4f-ebba-5204-a429-3ac8c609bfb7" +version = "0.9.5" + +[[deps.LAME_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "f6250b16881adf048549549fba48b1161acdac8c" +uuid = "c1c5ebd0-6772-5130-a774-d5fcae4a789d" +version = "3.100.1+0" + +[[deps.LLVM]] +deps = ["CEnum", "LLVMExtra_jll", "Libdl", "Preferences", "Printf", "Requires", "Unicode"] +git-tree-sha1 = "cb4619f7353fc62a1a22ffa3d7ed9791cfb47ad8" +uuid = "929cbde3-209d-540e-8aea-75f648917ca0" +version = "6.4.2" + + [deps.LLVM.extensions] + BFloat16sExt = "BFloat16s" + + [deps.LLVM.weakdeps] + BFloat16s = "ab4f0b2a-ad5b-11e8-123f-65d77653426b" + +[[deps.LLVMExtra_jll]] +deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl", "TOML"] +git-tree-sha1 = "98eaee04d96d973e79c25d49167668c5c8fb50e2" +uuid = "dad2f222-ce93-54a1-a47d-0025e8a3acab" +version = "0.0.27+1" + +[[deps.LLVMOpenMP_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "d986ce2d884d49126836ea94ed5bfb0f12679713" +uuid = "1d63c593-3942-5779-bab2-d838dc0a180e" +version = "15.0.7+0" + +[[deps.LRUCache]] +git-tree-sha1 = "b3cc6698599b10e652832c2f23db3cab99d51b59" +uuid = "8ac3fa9e-de4c-5943-b1dc-09c6b5f20637" +version = "1.6.1" +weakdeps = ["Serialization"] + + [deps.LRUCache.extensions] + SerializationExt = ["Serialization"] + +[[deps.LZO_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "e5b909bcf985c5e2605737d2ce278ed791b89be6" +uuid = "dd4b983a-f0e5-5f8d-a1b7-129d4a5fb1ac" +version = "2.10.1+0" + +[[deps.LaTeXStrings]] +git-tree-sha1 = "50901ebc375ed41dbf8058da26f9de442febbbec" +uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" +version = "1.3.1" + +[[deps.LayoutPointers]] +deps = ["ArrayInterface", "LinearAlgebra", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface"] +git-tree-sha1 = "62edfee3211981241b57ff1cedf4d74d79519277" +uuid = "10f19ff3-798f-405d-979b-55457f8fc047" +version = "0.1.15" + +[[deps.Lazy]] +deps = ["MacroTools"] +git-tree-sha1 = "1370f8202dac30758f3c345f9909b97f53d87d3f" +uuid = "50d2b5c4-7a5e-59d5-8109-a42b560f39c0" +version = "0.15.1" + +[[deps.LazyArrays]] +deps = ["ArrayLayouts", "FillArrays", "LinearAlgebra", "MacroTools", "MatrixFactorizations", "SparseArrays"] +git-tree-sha1 = "9cfca23ab83b0dfac93cb1a1ef3331ab9fe596a5" +uuid = "5078a376-72f3-5289-bfd5-ec5146d43c02" +version = "1.8.3" +weakdeps = ["StaticArrays"] + + [deps.LazyArrays.extensions] + LazyArraysStaticArraysExt = "StaticArrays" + +[[deps.LazyArtifacts]] +deps = ["Artifacts", "Pkg"] +uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3" + +[[deps.LazyModules]] +git-tree-sha1 = "a560dd966b386ac9ae60bdd3a3d3a326062d3c3e" +uuid = "8cdb02fc-e678-4876-92c5-9defec4f444e" +version = "0.3.1" + +[[deps.LeftChildRightSiblingTrees]] +deps = ["AbstractTrees"] +git-tree-sha1 = "fb6803dafae4a5d62ea5cab204b1e657d9737e7f" +uuid = "1d6d02ad-be62-4b6b-8a6d-2f90e265016e" +version = "0.2.0" + +[[deps.LevyArea]] +deps = ["LinearAlgebra", "Random", "SpecialFunctions"] +git-tree-sha1 = "56513a09b8e0ae6485f34401ea9e2f31357958ec" +uuid = "2d8b4e74-eb68-11e8-0fb9-d5eb67b50637" +version = "1.0.0" + +[[deps.LibCURL]] +deps = ["LibCURL_jll", "MozillaCACerts_jll"] +uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21" +version = "0.6.4" + +[[deps.LibCURL_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"] +uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0" +version = "8.4.0+0" + +[[deps.LibGit2]] +deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"] +uuid = "76f85450-5226-5b5a-8eaa-529ad045b433" + +[[deps.LibGit2_jll]] +deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"] +uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5" +version = "1.6.4+0" + +[[deps.LibSSH2_jll]] +deps = ["Artifacts", "Libdl", "MbedTLS_jll"] +uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8" +version = "1.11.0+1" + +[[deps.Libdl]] +uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb" + +[[deps.Libffi_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "0b4a5d71f3e5200a7dff793393e09dfc2d874290" +uuid = "e9f186c6-92d2-5b65-8a66-fee21dc1b490" +version = "3.2.2+1" + +[[deps.Libgcrypt_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgpg_error_jll", "Pkg"] +git-tree-sha1 = "64613c82a59c120435c067c2b809fc61cf5166ae" +uuid = "d4300ac3-e22c-5743-9152-c294e39db1e4" +version = "1.8.7+0" + +[[deps.Libgpg_error_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "c333716e46366857753e273ce6a69ee0945a6db9" +uuid = "7add5ba3-2f88-524e-9cd5-f83b8a55f7b8" +version = "1.42.0+0" + +[[deps.Libiconv_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "f9557a255370125b405568f9767d6d195822a175" +uuid = "94ce4f54-9a6c-5748-9c1c-f9c7231a4531" +version = "1.17.0+0" + +[[deps.Libmount_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "9c30530bf0effd46e15e0fdcf2b8636e78cbbd73" +uuid = "4b2f31a3-9ecc-558c-b454-b3730dcb73e9" +version = "2.35.0+0" + +[[deps.Libtask]] +deps = ["FunctionWrappers", "LRUCache", "LinearAlgebra", "Statistics"] +git-tree-sha1 = "345a40c746404dd9cb1bbc368715856838ab96f2" +uuid = "6f1fad26-d15e-5dc8-ae53-837a1d7b8c9f" +version = "0.8.6" + +[[deps.Libuuid_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "7f3efec06033682db852f8b3bc3c1d2b0a0ab066" +uuid = "38a345b3-de98-5d2b-a5d3-14cd9215e700" +version = "2.36.0+0" + +[[deps.LightXML]] +deps = ["Libdl", "XML2_jll"] +git-tree-sha1 = "3a994404d3f6709610701c7dabfc03fed87a81f8" +uuid = "9c8b4983-aa76-5018-a973-4c85ecc9e179" +version = "0.9.1" + +[[deps.LineSearches]] +deps = ["LinearAlgebra", "NLSolversBase", "NaNMath", "Parameters", "Printf"] +git-tree-sha1 = "7bbea35cec17305fc70a0e5b4641477dc0789d9d" +uuid = "d3d80556-e9d4-5f37-9878-2ab0fcc64255" +version = "7.2.0" + +[[deps.LinearAlgebra]] +deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"] +uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e" + +[[deps.LinearAlgebraX]] +deps = ["LinearAlgebra", "Mods", "Primes", "SimplePolynomials"] +git-tree-sha1 = "d76cec8007ec123c2b681269d40f94b053473fcf" +uuid = "9b3f67b0-2d00-526e-9884-9e4938f8fb88" +version = "0.2.7" + +[[deps.LinearSolve]] +deps = ["ArrayInterface", "ConcreteStructs", "DocStringExtensions", "EnumX", "FastLapackInterface", "GPUArraysCore", "InteractiveUtils", "KLU", "Krylov", "Libdl", "LinearAlgebra", "MKL_jll", "PrecompileTools", "Preferences", "RecursiveFactorization", "Reexport", "SciMLBase", "SciMLOperators", "Setfield", "SparseArrays", "Sparspak", "StaticArraysCore", "UnPack"] +git-tree-sha1 = "6f8e084deabe3189416c4e505b1c53e1b590cae8" +uuid = "7ed4a6bd-45f5-4d41-b270-4a48e9bafcae" +version = "2.22.1" + + [deps.LinearSolve.extensions] + LinearSolveBandedMatricesExt = "BandedMatrices" + LinearSolveBlockDiagonalsExt = "BlockDiagonals" + LinearSolveCUDAExt = "CUDA" + LinearSolveEnzymeExt = ["Enzyme", "EnzymeCore"] + LinearSolveFastAlmostBandedMatricesExt = ["FastAlmostBandedMatrices"] + LinearSolveHYPREExt = "HYPRE" + LinearSolveIterativeSolversExt = "IterativeSolvers" + LinearSolveKernelAbstractionsExt = "KernelAbstractions" + LinearSolveKrylovKitExt = "KrylovKit" + LinearSolveMetalExt = "Metal" + LinearSolvePardisoExt = "Pardiso" + LinearSolveRecursiveArrayToolsExt = "RecursiveArrayTools" + + [deps.LinearSolve.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" + BlockDiagonals = "0a1fb500-61f7-11e9-3c65-f5ef3456f9f0" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + FastAlmostBandedMatrices = "9d29842c-ecb8-4973-b1e9-a27b1157504e" + HYPRE = "b5ffcf37-a2bd-41ab-a3da-4bd9bc8ad771" + IterativeSolvers = "42fd0dbc-a981-5370-80f2-aaf504508153" + KernelAbstractions = "63c18a36-062a-441e-b654-da1e3ab1ce7c" + KrylovKit = "0b1a1467-8014-51b9-945f-bf0ae24f4b77" + Metal = "dde4c033-4e86-420c-a63e-0dd931031962" + Pardiso = "46dd5b70-b6fb-5a00-ae2d-e8fea33afaf2" + RecursiveArrayTools = "731186ca-8d62-57ce-b412-fbd966d074cd" + +[[deps.Literate]] +deps = ["Base64", "IOCapture", "JSON", "REPL"] +git-tree-sha1 = "bad26f1ccd99c553886ec0725e99a509589dcd11" +uuid = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +version = "2.16.1" + +[[deps.LiveServer]] +deps = ["HTTP", "LoggingExtras", "MIMEs", "Pkg", "Sockets", "Test"] +git-tree-sha1 = "24d05efe53436b22a42bf2ae459f47c48b0c2603" +uuid = "16fef848-5104-11e9-1b77-fb7a48bbb589" +version = "1.2.7" + +[[deps.Loess]] +deps = ["Distances", "LinearAlgebra", "Statistics", "StatsAPI"] +git-tree-sha1 = "a113a8be4c6d0c64e217b472fb6e61c760eb4022" +uuid = "4345ca2d-374a-55d4-8d30-97f9976e7612" +version = "0.6.3" + +[[deps.LogDensityProblems]] +deps = ["ArgCheck", "DocStringExtensions", "Random"] +git-tree-sha1 = "f9a11237204bc137617194d79d813069838fcf61" +uuid = "6fdf6af0-433a-55f7-b3ed-c6c6e0b8df7c" +version = "2.1.1" + +[[deps.LogDensityProblemsAD]] +deps = ["DocStringExtensions", "LogDensityProblems", "Requires", "SimpleUnPack"] +git-tree-sha1 = "9c50732cd0f188766b6217ed6a2ebbdaf9890029" +uuid = "996a588d-648d-4e1f-a8f0-a84b347e47b1" +version = "1.7.0" + + [deps.LogDensityProblemsAD.extensions] + LogDensityProblemsADADTypesExt = "ADTypes" + LogDensityProblemsADEnzymeExt = "Enzyme" + LogDensityProblemsADFiniteDifferencesExt = "FiniteDifferences" + LogDensityProblemsADForwardDiffBenchmarkToolsExt = ["BenchmarkTools", "ForwardDiff"] + LogDensityProblemsADForwardDiffExt = "ForwardDiff" + LogDensityProblemsADReverseDiffExt = "ReverseDiff" + LogDensityProblemsADTrackerExt = "Tracker" + LogDensityProblemsADZygoteExt = "Zygote" + + [deps.LogDensityProblemsAD.weakdeps] + ADTypes = "47edcb42-4c32-4615-8424-f2b9edc5f35b" + BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + FiniteDifferences = "26cc04aa-876d-5657-8c51-4c34ba976000" + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.LogExpFunctions]] +deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"] +git-tree-sha1 = "7d6dd4e9212aebaeed356de34ccf262a3cd415aa" +uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688" +version = "0.3.26" +weakdeps = ["ChainRulesCore", "ChangesOfVariables", "InverseFunctions"] + + [deps.LogExpFunctions.extensions] + LogExpFunctionsChainRulesCoreExt = "ChainRulesCore" + LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables" + LogExpFunctionsInverseFunctionsExt = "InverseFunctions" + +[[deps.Logging]] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" + +[[deps.LoggingExtras]] +deps = ["Dates", "Logging"] +git-tree-sha1 = "c1dd6d7978c12545b4179fb6153b9250c96b0075" +uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36" +version = "1.0.3" + +[[deps.LoopVectorization]] +deps = ["ArrayInterface", "CPUSummary", "CloseOpenIntervals", "DocStringExtensions", "HostCPUFeatures", "IfElse", "LayoutPointers", "LinearAlgebra", "OffsetArrays", "PolyesterWeave", "PrecompileTools", "SIMDTypes", "SLEEFPirates", "Static", "StaticArrayInterface", "ThreadingUtilities", "UnPack", "VectorizationBase"] +git-tree-sha1 = "0f5648fbae0d015e3abe5867bca2b362f67a5894" +uuid = "bdcacae8-1622-11e9-2a5c-532679323890" +version = "0.12.166" +weakdeps = ["ChainRulesCore", "ForwardDiff", "SpecialFunctions"] + + [deps.LoopVectorization.extensions] + ForwardDiffExt = ["ChainRulesCore", "ForwardDiff"] + SpecialFunctionsExt = "SpecialFunctions" + +[[deps.MCMCChains]] +deps = ["AbstractMCMC", "AxisArrays", "Dates", "Distributions", "Formatting", "IteratorInterfaceExtensions", "KernelDensity", "LinearAlgebra", "MCMCDiagnosticTools", "MLJModelInterface", "NaturalSort", "OrderedCollections", "PrettyTables", "Random", "RecipesBase", "Statistics", "StatsBase", "StatsFuns", "TableTraits", "Tables"] +git-tree-sha1 = "3b1ae6bcb0a94ed7760e72cd3524794f613658d2" +uuid = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" +version = "6.0.4" + +[[deps.MCMCDiagnosticTools]] +deps = ["AbstractFFTs", "DataAPI", "DataStructures", "Distributions", "LinearAlgebra", "MLJModelInterface", "Random", "SpecialFunctions", "Statistics", "StatsBase", "StatsFuns", "Tables"] +git-tree-sha1 = "6ea46c36b86320593d2017da3c28c79165167ef4" +uuid = "be115224-59cd-429b-ad48-344e309966f0" +version = "0.3.8" + +[[deps.MIMEs]] +git-tree-sha1 = "65f28ad4b594aebe22157d6fac869786a255b7eb" +uuid = "6c6e2e6c-3030-632d-7369-2d6c69616d65" +version = "0.1.4" + +[[deps.MKL_jll]] +deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl"] +git-tree-sha1 = "72dc3cf284559eb8f53aa593fe62cb33f83ed0c0" +uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7" +version = "2024.0.0+0" + +[[deps.MLJModelInterface]] +deps = ["Random", "ScientificTypesBase", "StatisticalTraits"] +git-tree-sha1 = "14bd8088cf7cd1676aa83a57004f8d23d43cd81e" +uuid = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" +version = "1.9.5" + +[[deps.MacroTools]] +deps = ["Markdown", "Random"] +git-tree-sha1 = "2fa9ee3e63fd3a4f7a9a4f4744a52f4856de82df" +uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09" +version = "0.5.13" + +[[deps.Makie]] +deps = ["Animations", "Base64", "CRC32c", "ColorBrewer", "ColorSchemes", "ColorTypes", "Colors", "Contour", "DelaunayTriangulation", "Distributions", "DocStringExtensions", "Downloads", "FFMPEG_jll", "FileIO", "FilePaths", "FixedPointNumbers", "Formatting", "FreeType", "FreeTypeAbstraction", "GeometryBasics", "GridLayoutBase", "ImageIO", "InteractiveUtils", "IntervalSets", "Isoband", "KernelDensity", "LaTeXStrings", "LinearAlgebra", "MacroTools", "MakieCore", "Markdown", "MathTeXEngine", "Observables", "OffsetArrays", "Packing", "PlotUtils", "PolygonOps", "PrecompileTools", "Printf", "REPL", "Random", "RelocatableFolders", "Scratch", "ShaderAbstractions", "Showoff", "SignedDistanceFields", "SparseArrays", "StableHashTraits", "Statistics", "StatsBase", "StatsFuns", "StructArrays", "TriplotBase", "UnicodeFun"] +git-tree-sha1 = "5458e49833d178c071dd6ab6938ac5320f37bc98" +uuid = "ee78f7c6-11fb-53f2-987a-cfe4a2b5a57a" +version = "0.20.5" + +[[deps.MakieCore]] +deps = ["Observables", "REPL"] +git-tree-sha1 = "248b7a4be0f92b497f7a331aed02c1e9a878f46b" +uuid = "20f20a25-4f0e-4fdf-b5d1-57303727442b" +version = "0.7.3" + +[[deps.ManualMemory]] +git-tree-sha1 = "bcaef4fc7a0cfe2cba636d84cda54b5e4e4ca3cd" +uuid = "d125e4d3-2237-4719-b19c-fa641b8a4667" +version = "0.1.8" + +[[deps.MappedArrays]] +git-tree-sha1 = "2dab0221fe2b0f2cb6754eaa743cc266339f527e" +uuid = "dbb5928d-eab1-5f90-85c2-b9b0edb7c900" +version = "0.4.2" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" + +[[deps.MathOptInterface]] +deps = ["BenchmarkTools", "CodecBzip2", "CodecZlib", "DataStructures", "ForwardDiff", "JSON", "LinearAlgebra", "MutableArithmetics", "NaNMath", "OrderedCollections", "PrecompileTools", "Printf", "SparseArrays", "SpecialFunctions", "Test", "Unicode"] +git-tree-sha1 = "8b40681684df46785a0012d352982e22ac3be59e" +uuid = "b8f27783-ece8-5eb3-8dc8-9495eed66fee" +version = "1.25.2" + +[[deps.MathTeXEngine]] +deps = ["AbstractTrees", "Automa", "DataStructures", "FreeTypeAbstraction", "GeometryBasics", "LaTeXStrings", "REPL", "RelocatableFolders", "UnicodeFun"] +git-tree-sha1 = "96ca8a313eb6437db5ffe946c457a401bbb8ce1d" +uuid = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" +version = "0.5.7" + +[[deps.MatrixFactorizations]] +deps = ["ArrayLayouts", "LinearAlgebra", "Printf", "Random"] +git-tree-sha1 = "78f6e33434939b0ac9ba1df81e6d005ee85a7396" +uuid = "a3b82374-2e81-5b9e-98ce-41277c0e4c87" +version = "2.1.0" + +[[deps.MaybeInplace]] +deps = ["ArrayInterface", "LinearAlgebra", "MacroTools", "SparseArrays"] +git-tree-sha1 = "a85c6a98c9e5a2a7046bc1bb89f28a3241e1de4d" +uuid = "bb5d69b7-63fc-4a16-80bd-7e42200c7bdb" +version = "0.1.1" + +[[deps.MbedTLS]] +deps = ["Dates", "MbedTLS_jll", "MozillaCACerts_jll", "NetworkOptions", "Random", "Sockets"] +git-tree-sha1 = "c067a280ddc25f196b5e7df3877c6b226d390aaf" +uuid = "739be429-bea8-5141-9913-cc70e7f3736d" +version = "1.1.9" + +[[deps.MbedTLS_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1" +version = "2.28.2+1" + +[[deps.MicroCollections]] +deps = ["BangBang", "InitialValues", "Setfield"] +git-tree-sha1 = "629afd7d10dbc6935ec59b32daeb33bc4460a42e" +uuid = "128add7d-3638-4c79-886c-908ea0c25c34" +version = "0.1.4" + +[[deps.Missings]] +deps = ["DataAPI"] +git-tree-sha1 = "f66bdc5de519e8f8ae43bdc598782d35a25b1272" +uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28" +version = "1.1.0" + +[[deps.Mmap]] +uuid = "a63ad114-7e13-5084-954f-fe012c677804" + +[[deps.Mods]] +git-tree-sha1 = "924f962b524a71eef7a21dae1e6853817f9b658f" +uuid = "7475f97c-0381-53b1-977b-4c60186c8d62" +version = "2.2.4" + +[[deps.MosaicViews]] +deps = ["MappedArrays", "OffsetArrays", "PaddedViews", "StackViews"] +git-tree-sha1 = "7b86a5d4d70a9f5cdf2dacb3cbe6d251d1a61dbe" +uuid = "e94cdb99-869f-56ef-bcf0-1ae2bcbe0389" +version = "0.3.4" + +[[deps.MozillaCACerts_jll]] +uuid = "14a3606d-f60d-562e-9121-12d972cd8159" +version = "2023.1.10" + +[[deps.MuladdMacro]] +git-tree-sha1 = "cac9cc5499c25554cba55cd3c30543cff5ca4fab" +uuid = "46d2c3a1-f734-5fdb-9937-b9b9aeba4221" +version = "0.2.4" + +[[deps.Multisets]] +git-tree-sha1 = "8d852646862c96e226367ad10c8af56099b4047e" +uuid = "3b2b4ff1-bcff-5658-a3ee-dbcf1ce5ac09" +version = "0.4.4" + +[[deps.MutableArithmetics]] +deps = ["LinearAlgebra", "SparseArrays", "Test"] +git-tree-sha1 = "806eea990fb41f9b36f1253e5697aa645bf6a9f8" +uuid = "d8a4904e-b15c-11e9-3269-09a3773c0cb0" +version = "1.4.0" + +[[deps.NLSolversBase]] +deps = ["DiffResults", "Distributed", "FiniteDiff", "ForwardDiff"] +git-tree-sha1 = "a0b464d183da839699f4c79e7606d9d186ec172c" +uuid = "d41bc354-129a-5804-8e4c-c37616107c6c" +version = "7.8.3" + +[[deps.NLsolve]] +deps = ["Distances", "LineSearches", "LinearAlgebra", "NLSolversBase", "Printf", "Reexport"] +git-tree-sha1 = "019f12e9a1a7880459d0173c182e6a99365d7ac1" +uuid = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" +version = "4.5.1" + +[[deps.NNlib]] +deps = ["Adapt", "Atomix", "ChainRulesCore", "GPUArraysCore", "KernelAbstractions", "LinearAlgebra", "Pkg", "Random", "Requires", "Statistics"] +git-tree-sha1 = "900a11b3a2b02e36b25cb55a80777d4a4670f0f6" +uuid = "872c559c-99b0-510c-b3b7-b6c96a88d5cd" +version = "0.9.10" + + [deps.NNlib.extensions] + NNlibAMDGPUExt = "AMDGPU" + NNlibCUDACUDNNExt = ["CUDA", "cuDNN"] + NNlibCUDAExt = "CUDA" + NNlibEnzymeCoreExt = "EnzymeCore" + + [deps.NNlib.weakdeps] + AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e" + CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba" + EnzymeCore = "f151be2c-9106-41f4-ab19-57ee4f262869" + cuDNN = "02a925ec-e4fe-4b08-9a7e-0d78e3d38ccd" + +[[deps.NaNMath]] +deps = ["OpenLibm_jll"] +git-tree-sha1 = "0877504529a3e5c3343c6f8b4c0381e57e4387e4" +uuid = "77ba4419-2d1f-58cd-9bb1-8ffee604a2e3" +version = "1.0.2" + +[[deps.NamedArrays]] +deps = ["Combinatorics", "DataStructures", "DelimitedFiles", "InvertedIndices", "LinearAlgebra", "Random", "Requires", "SparseArrays", "Statistics"] +git-tree-sha1 = "6d42eca6c3a27dc79172d6d947ead136d88751bb" +uuid = "86f7a689-2022-50b4-a561-43c23ac3c673" +version = "0.10.0" + +[[deps.NaturalSort]] +git-tree-sha1 = "eda490d06b9f7c00752ee81cfa451efe55521e21" +uuid = "c020b1a1-e9b0-503a-9c33-f039bfc54a85" +version = "1.0.0" + +[[deps.Netpbm]] +deps = ["FileIO", "ImageCore", "ImageMetadata"] +git-tree-sha1 = "d92b107dbb887293622df7697a2223f9f8176fcd" +uuid = "f09324ee-3d7c-5217-9330-fc30815ba969" +version = "1.1.1" + +[[deps.NetworkOptions]] +uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908" +version = "1.2.0" + +[[deps.NodeJS]] +deps = ["Pkg"] +git-tree-sha1 = "bf1f49fd62754064bc42490a8ddc2aa3694a8e7a" +uuid = "2bd173c7-0d6d-553b-b6af-13a54713934c" +version = "2.0.0" + +[[deps.NonlinearSolve]] +deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "FastBroadcast", "FastClosures", "FiniteDiff", "ForwardDiff", "LazyArrays", "LineSearches", "LinearAlgebra", "LinearSolve", "MaybeInplace", "PrecompileTools", "Preferences", "Printf", "RecursiveArrayTools", "Reexport", "SciMLBase", "SimpleNonlinearSolve", "SparseArrays", "SparseDiffTools", "StaticArraysCore", "TimerOutputs"] +git-tree-sha1 = "78bdd3a4a62865cf43c53d63783b0cbfddcdbbe6" +uuid = "8913a72c-1f9b-4ce2-8d82-65094dcecaec" +version = "3.5.0" + + [deps.NonlinearSolve.extensions] + NonlinearSolveBandedMatricesExt = "BandedMatrices" + NonlinearSolveFastLevenbergMarquardtExt = "FastLevenbergMarquardt" + NonlinearSolveFixedPointAccelerationExt = "FixedPointAcceleration" + NonlinearSolveLeastSquaresOptimExt = "LeastSquaresOptim" + NonlinearSolveMINPACKExt = "MINPACK" + NonlinearSolveNLsolveExt = "NLsolve" + NonlinearSolveSIAMFANLEquationsExt = "SIAMFANLEquations" + NonlinearSolveSpeedMappingExt = "SpeedMapping" + NonlinearSolveSymbolicsExt = "Symbolics" + NonlinearSolveZygoteExt = "Zygote" + + [deps.NonlinearSolve.weakdeps] + BandedMatrices = "aae01518-5342-5314-be14-df237901396f" + FastLevenbergMarquardt = "7a0df574-e128-4d35-8cbd-3d84502bf7ce" + FixedPointAcceleration = "817d07cb-a79a-5c30-9a31-890123675176" + LeastSquaresOptim = "0fc2ff8b-aaa3-5acd-a817-1944a5e08891" + MINPACK = "4854310b-de5a-5eb6-a2a5-c1dee2bd17f9" + NLsolve = "2774e3e8-f4cf-5e23-947b-6d7e65073b56" + SIAMFANLEquations = "084e46ad-d928-497d-ad5e-07fa361a48c4" + SpeedMapping = "f1835b91-879b-4a3f-a438-e4baacf14412" + Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.Observables]] +git-tree-sha1 = "7438a59546cf62428fc9d1bc94729146d37a7225" +uuid = "510215fc-4207-5dde-b226-833fc4488ee2" +version = "0.5.5" + +[[deps.OffsetArrays]] +git-tree-sha1 = "6a731f2b5c03157418a20c12195eb4b74c8f8621" +uuid = "6fe1bfb0-de20-5000-8ca7-80f57d26f881" +version = "1.13.0" +weakdeps = ["Adapt"] + + [deps.OffsetArrays.extensions] + OffsetArraysAdaptExt = "Adapt" + +[[deps.Ogg_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "887579a3eb005446d514ab7aeac5d1d027658b8f" +uuid = "e7412a2a-1a6e-54c0-be00-318e2571c051" +version = "1.3.5+1" + +[[deps.OpenBLAS_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"] +uuid = "4536629a-c528-5b80-bd46-f80d51c5b363" +version = "0.3.23+2" + +[[deps.OpenEXR]] +deps = ["Colors", "FileIO", "OpenEXR_jll"] +git-tree-sha1 = "327f53360fdb54df7ecd01e96ef1983536d1e633" +uuid = "52e1d378-f018-4a11-a4be-720524705ac7" +version = "0.3.2" + +[[deps.OpenEXR_jll]] +deps = ["Artifacts", "Imath_jll", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "a4ca623df1ae99d09bc9868b008262d0c0ac1e4f" +uuid = "18a262bb-aa17-5467-a713-aee519bc75cb" +version = "3.1.4+0" + +[[deps.OpenLibm_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "05823500-19ac-5b8b-9628-191a04bc5112" +version = "0.8.1+2" + +[[deps.OpenSSL]] +deps = ["BitFlags", "Dates", "MozillaCACerts_jll", "OpenSSL_jll", "Sockets"] +git-tree-sha1 = "51901a49222b09e3743c65b8847687ae5fc78eb2" +uuid = "4d8831e6-92b7-49fb-bdf8-b643e874388c" +version = "1.4.1" + +[[deps.OpenSSL_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "cc6e1927ac521b659af340e0ca45828a3ffc748f" +uuid = "458c3c95-2e84-50aa-8efc-19380b2a3a95" +version = "3.0.12+0" + +[[deps.OpenSpecFun_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "13652491f6856acfd2db29360e1bbcd4565d04f1" +uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e" +version = "0.5.5+0" + +[[deps.Optim]] +deps = ["Compat", "FillArrays", "ForwardDiff", "LineSearches", "LinearAlgebra", "MathOptInterface", "NLSolversBase", "NaNMath", "Parameters", "PositiveFactorizations", "Printf", "SparseArrays", "StatsBase"] +git-tree-sha1 = "931b2ecf1b43de06dddc342c6af03f2b11434958" +uuid = "429524aa-4258-5aef-a3af-852621145aeb" +version = "1.9.0" + +[[deps.Optimisers]] +deps = ["ChainRulesCore", "Functors", "LinearAlgebra", "Random", "Statistics"] +git-tree-sha1 = "34205b1204cc83c43cd9cfe53ffbd3b310f6e8c5" +uuid = "3bd65402-5787-11e9-1adc-39752487f4e2" +version = "0.3.1" + +[[deps.Opus_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51a08fb14ec28da2ec7a927c4337e4332c2a4720" +uuid = "91d4177d-7536-5919-b921-800302f37372" +version = "1.3.2+0" + +[[deps.OrderedCollections]] +git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5" +uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d" +version = "1.6.3" + +[[deps.OrdinaryDiffEq]] +deps = ["ADTypes", "Adapt", "ArrayInterface", "DataStructures", "DiffEqBase", "DocStringExtensions", "ExponentialUtilities", "FastBroadcast", "FastClosures", "FillArrays", "FiniteDiff", "ForwardDiff", "FunctionWrappersWrappers", "IfElse", "InteractiveUtils", "LineSearches", "LinearAlgebra", "LinearSolve", "Logging", "MacroTools", "MuladdMacro", "NonlinearSolve", "Polyester", "PreallocationTools", "PrecompileTools", "Preferences", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "SimpleNonlinearSolve", "SimpleUnPack", "SparseArrays", "SparseDiffTools", "StaticArrayInterface", "StaticArrays", "TruncatedStacktraces"] +git-tree-sha1 = "7c6738f21fba2ccd07b7eaa9d23b437a8a97f1a1" +uuid = "1dea7af3-3e70-54e6-95c3-0bf5283fa5ed" +version = "6.69.0" + +[[deps.PCRE2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "efcefdf7-47ab-520b-bdef-62a2eaa19f15" +version = "10.42.0+1" + +[[deps.PDMats]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "949347156c25054de2db3b166c52ac4728cbad65" +uuid = "90014a1f-27ba-587c-ab20-58faa44d9150" +version = "0.11.31" + +[[deps.PNGFiles]] +deps = ["Base64", "CEnum", "ImageCore", "IndirectArrays", "OffsetArrays", "libpng_jll"] +git-tree-sha1 = "67186a2bc9a90f9f85ff3cc8277868961fb57cbd" +uuid = "f57f5aa1-a3ce-4bc8-8ab9-96f992907883" +version = "0.4.3" + +[[deps.PackageExtensionCompat]] +git-tree-sha1 = "fb28e33b8a95c4cee25ce296c817d89cc2e53518" +uuid = "65ce6f38-6b18-4e1d-a461-8949797d7930" +version = "1.0.2" +weakdeps = ["Requires", "TOML"] + +[[deps.Packing]] +deps = ["GeometryBasics"] +git-tree-sha1 = "ec3edfe723df33528e085e632414499f26650501" +uuid = "19eb6ba3-879d-56ad-ad62-d5c202156566" +version = "0.5.0" + +[[deps.PaddedViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "0fac6313486baae819364c52b4f483450a9d793f" +uuid = "5432bcbf-9aad-5242-b902-cca2824c8663" +version = "0.5.12" + +[[deps.Pango_jll]] +deps = ["Artifacts", "Cairo_jll", "Fontconfig_jll", "FreeType2_jll", "FriBidi_jll", "Glib_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl"] +git-tree-sha1 = "4745216e94f71cb768d58330b059c9b76f32cb66" +uuid = "36c8627f-9965-5494-a995-c6b170f724f3" +version = "1.50.14+0" + +[[deps.Parameters]] +deps = ["OrderedCollections", "UnPack"] +git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe" +uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a" +version = "0.12.3" + +[[deps.Parsers]] +deps = ["Dates", "PrecompileTools", "UUIDs"] +git-tree-sha1 = "8489905bcdbcfac64d1daa51ca07c0d8f0283821" +uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0" +version = "2.8.1" + +[[deps.Permutations]] +deps = ["Combinatorics", "LinearAlgebra", "Random"] +git-tree-sha1 = "eb3f9df2457819bf0a9019bd93cc451697a0751e" +uuid = "2ae35dd2-176d-5d53-8349-f30d82d94d4f" +version = "0.4.20" + +[[deps.PikaParser]] +deps = ["DocStringExtensions"] +git-tree-sha1 = "d6ff87de27ff3082131f31a714d25ab6d0a88abf" +uuid = "3bbf5609-3e7b-44cd-8549-7c69f321e792" +version = "0.6.1" + +[[deps.Pixman_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "LLVMOpenMP_jll", "Libdl"] +git-tree-sha1 = "64779bc4c9784fee475689a1752ef4d5747c5e87" +uuid = "30392449-352a-5448-841d-b1acce4e97dc" +version = "0.42.2+0" + +[[deps.Pkg]] +deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"] +uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f" +version = "1.10.0" + +[[deps.PkgVersion]] +deps = ["Pkg"] +git-tree-sha1 = "f9501cc0430a26bc3d156ae1b5b0c1b47af4d6da" +uuid = "eebad327-c553-4316-9ea0-9fa01ccd7688" +version = "0.3.3" + +[[deps.PlotUtils]] +deps = ["ColorSchemes", "Colors", "Dates", "PrecompileTools", "Printf", "Random", "Reexport", "Statistics"] +git-tree-sha1 = "862942baf5663da528f66d24996eb6da85218e76" +uuid = "995b91a9-d308-5afd-9ec6-746e21dbc043" +version = "1.4.0" + +[[deps.PoissonRandom]] +deps = ["Random"] +git-tree-sha1 = "a0f1159c33f846aa77c3f30ebbc69795e5327152" +uuid = "e409e4f3-bfea-5376-8464-e040bb5c01ab" +version = "0.4.4" + +[[deps.Polyester]] +deps = ["ArrayInterface", "BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "ManualMemory", "PolyesterWeave", "Requires", "Static", "StaticArrayInterface", "StrideArraysCore", "ThreadingUtilities"] +git-tree-sha1 = "fca25670784a1ae44546bcb17288218310af2778" +uuid = "f517fe37-dbe3-4b94-8317-1923a5111588" +version = "0.7.9" + +[[deps.PolyesterWeave]] +deps = ["BitTwiddlingConvenienceFunctions", "CPUSummary", "IfElse", "Static", "ThreadingUtilities"] +git-tree-sha1 = "240d7170f5ffdb285f9427b92333c3463bf65bf6" +uuid = "1d0040c9-8b98-4ee7-8388-3f51789ca0ad" +version = "0.2.1" + +[[deps.PolygonOps]] +git-tree-sha1 = "77b3d3605fc1cd0b42d95eba87dfcd2bf67d5ff6" +uuid = "647866c9-e3ac-4575-94e7-e3d426903924" +version = "0.1.2" + +[[deps.Polynomials]] +deps = ["LinearAlgebra", "RecipesBase", "Setfield", "SparseArrays"] +git-tree-sha1 = "a9c7a523d5ed375be3983db190f6a5874ae9286d" +uuid = "f27b6e38-b328-58d1-80ce-0feddd5e7a45" +version = "4.0.6" +weakdeps = ["ChainRulesCore", "FFTW", "MakieCore", "MutableArithmetics"] + + [deps.Polynomials.extensions] + PolynomialsChainRulesCoreExt = "ChainRulesCore" + PolynomialsFFTWExt = "FFTW" + PolynomialsMakieCoreExt = "MakieCore" + PolynomialsMutableArithmeticsExt = "MutableArithmetics" + +[[deps.PooledArrays]] +deps = ["DataAPI", "Future"] +git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3" +uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720" +version = "1.4.3" + +[[deps.PositiveFactorizations]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "17275485f373e6673f7e7f97051f703ed5b15b20" +uuid = "85a6dd25-e78a-55b7-8502-1745935b8125" +version = "0.2.4" + +[[deps.PreallocationTools]] +deps = ["Adapt", "ArrayInterface", "ForwardDiff"] +git-tree-sha1 = "64bb68f76f789f5fe5930a80af310f19cdafeaed" +uuid = "d236fae5-4411-538c-8e31-a6e3d9e00b46" +version = "0.4.17" + + [deps.PreallocationTools.extensions] + PreallocationToolsReverseDiffExt = "ReverseDiff" + + [deps.PreallocationTools.weakdeps] + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + +[[deps.PrecompileTools]] +deps = ["Preferences"] +git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f" +uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a" +version = "1.2.0" + +[[deps.Preferences]] +deps = ["TOML"] +git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e" +uuid = "21216c6a-2e73-6563-6e65-726566657250" +version = "1.4.1" + +[[deps.PrettyTables]] +deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "Reexport", "StringManipulation", "Tables"] +git-tree-sha1 = "88b895d13d53b5577fd53379d913b9ab9ac82660" +uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d" +version = "2.3.1" + +[[deps.Primes]] +deps = ["IntegerMathUtils"] +git-tree-sha1 = "1d05623b5952aed1307bf8b43bec8b8d1ef94b6e" +uuid = "27ebfcd6-29c5-5fa9-bf4b-fb8fc14df3ae" +version = "0.5.5" + +[[deps.Printf]] +deps = ["Unicode"] +uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7" + +[[deps.Profile]] +deps = ["Printf"] +uuid = "9abbd945-dff8-562f-b5e8-e1ebf5ef1b79" + +[[deps.ProgressLogging]] +deps = ["Logging", "SHA", "UUIDs"] +git-tree-sha1 = "80d919dee55b9c50e8d9e2da5eeafff3fe58b539" +uuid = "33c8b6b6-d38a-422a-b730-caa89a2f386c" +version = "0.1.4" + +[[deps.ProgressMeter]] +deps = ["Distributed", "Printf"] +git-tree-sha1 = "00099623ffee15972c16111bcf84c58a0051257c" +uuid = "92933f4c-e287-5a05-a399-4b506db050ca" +version = "1.9.0" + +[[deps.QOI]] +deps = ["ColorTypes", "FileIO", "FixedPointNumbers"] +git-tree-sha1 = "18e8f4d1426e965c7b532ddd260599e1510d26ce" +uuid = "4b34888f-f399-49d4-9bb3-47ed5cae4e65" +version = "1.0.0" + +[[deps.QuadGK]] +deps = ["DataStructures", "LinearAlgebra"] +git-tree-sha1 = "9b23c31e76e333e6fb4c1595ae6afa74966a729e" +uuid = "1fd47b50-473d-5c70-9696-f719f8f3bcdc" +version = "2.9.4" + +[[deps.REPL]] +deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"] +uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" + +[[deps.Random123]] +deps = ["Random", "RandomNumbers"] +git-tree-sha1 = "c860e84651f58ce240dd79e5d9e055d55234c35a" +uuid = "74087812-796a-5b5d-8853-05524746bad3" +version = "1.6.2" + +[[deps.RandomNumbers]] +deps = ["Random", "Requires"] +git-tree-sha1 = "043da614cc7e95c703498a491e2c21f58a2b8111" +uuid = "e6cf234a-135c-5ec9-84dd-332b85af5143" +version = "1.5.3" + +[[deps.RangeArrays]] +git-tree-sha1 = "b9039e93773ddcfc828f12aadf7115b4b4d225f5" +uuid = "b3c3ace0-ae52-54e7-9d0b-2c1406fd6b9d" +version = "0.3.2" + +[[deps.Ratios]] +deps = ["Requires"] +git-tree-sha1 = "1342a47bf3260ee108163042310d26f2be5ec90b" +uuid = "c84ed2f1-dad5-54f0-aa8e-dbefe2724439" +version = "0.4.5" +weakdeps = ["FixedPointNumbers"] + + [deps.Ratios.extensions] + RatiosFixedPointNumbersExt = "FixedPointNumbers" + +[[deps.RealDot]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "9f0a1b71baaf7650f4fa8a1d168c7fb6ee41f0c9" +uuid = "c1ae055f-0cd5-4b69-90a6-9a35b1a98df9" +version = "0.1.0" + +[[deps.RecipesBase]] +deps = ["PrecompileTools"] +git-tree-sha1 = "5c3d09cc4f31f5fc6af001c250bf1278733100ff" +uuid = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" +version = "1.3.4" + +[[deps.RecursiveArrayTools]] +deps = ["Adapt", "ArrayInterface", "DocStringExtensions", "GPUArraysCore", "IteratorInterfaceExtensions", "LinearAlgebra", "RecipesBase", "SparseArrays", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables"] +git-tree-sha1 = "16f1bb9de02b8bce31a7b2495345532901214cae" +uuid = "731186ca-8d62-57ce-b412-fbd966d074cd" +version = "3.6.2" + + [deps.RecursiveArrayTools.extensions] + RecursiveArrayToolsFastBroadcastExt = "FastBroadcast" + RecursiveArrayToolsMeasurementsExt = "Measurements" + RecursiveArrayToolsMonteCarloMeasurementsExt = "MonteCarloMeasurements" + RecursiveArrayToolsReverseDiffExt = ["ReverseDiff", "Zygote"] + RecursiveArrayToolsTrackerExt = "Tracker" + RecursiveArrayToolsZygoteExt = "Zygote" + + [deps.RecursiveArrayTools.weakdeps] + FastBroadcast = "7034ab61-46d4-4ed7-9d0f-46aef9175898" + Measurements = "eff96d63-e80a-5855-80a2-b1b0885c5ab7" + MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca" + ReverseDiff = "37e2e3b7-166d-5795-8a7a-e32c996b4267" + Tracker = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.RecursiveFactorization]] +deps = ["LinearAlgebra", "LoopVectorization", "Polyester", "PrecompileTools", "StrideArraysCore", "TriangularSolve"] +git-tree-sha1 = "8bc86c78c7d8e2a5fe559e3721c0f9c9e303b2ed" +uuid = "f2c3362d-daeb-58d1-803e-2bc74f2840b4" +version = "0.2.21" + +[[deps.Reexport]] +git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b" +uuid = "189a3867-3050-52da-a836-e630ba90ab69" +version = "1.2.2" + +[[deps.RelocatableFolders]] +deps = ["SHA", "Scratch"] +git-tree-sha1 = "ffdaf70d81cf6ff22c2b6e733c900c3321cab864" +uuid = "05181044-ff0b-4ac5-8273-598c1e38db00" +version = "1.0.1" + +[[deps.Requires]] +deps = ["UUIDs"] +git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7" +uuid = "ae029012-a4dd-5104-9daa-d747884805df" +version = "1.3.0" + +[[deps.ResettableStacks]] +deps = ["StaticArrays"] +git-tree-sha1 = "256eeeec186fa7f26f2801732774ccf277f05db9" +uuid = "ae5879a3-cd67-5da8-be7f-38c6eb64a37b" +version = "1.1.1" + +[[deps.RingLists]] +deps = ["Random"] +git-tree-sha1 = "f39da63aa6d2d88e0c1bd20ed6a3ff9ea7171ada" +uuid = "286e9d63-9694-5540-9e3c-4e6708fa07b2" +version = "0.2.8" + +[[deps.Rmath]] +deps = ["Random", "Rmath_jll"] +git-tree-sha1 = "f65dcb5fa46aee0cf9ed6274ccbd597adc49aa7b" +uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa" +version = "0.7.1" + +[[deps.Rmath_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "6ed52fdd3382cf21947b15e8870ac0ddbff736da" +uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f" +version = "0.4.0+0" + +[[deps.Roots]] +deps = ["Accessors", "ChainRulesCore", "CommonSolve", "Printf"] +git-tree-sha1 = "39ebae5b76c8cd5629bec21adfca78b437dac1e6" +uuid = "f2b01f46-fcfa-551c-844a-d8ac1e96c665" +version = "2.1.1" + + [deps.Roots.extensions] + RootsForwardDiffExt = "ForwardDiff" + RootsIntervalRootFindingExt = "IntervalRootFinding" + RootsSymPyExt = "SymPy" + RootsSymPyPythonCallExt = "SymPyPythonCall" + + [deps.Roots.weakdeps] + ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" + IntervalRootFinding = "d2bf35a9-74e0-55ec-b149-d360ff49b807" + SymPy = "24249f21-da20-56a4-8eb1-6a02cf4ae2e6" + SymPyPythonCall = "bc8888f7-b21e-4b7c-a06a-5d9c9496438c" + +[[deps.RoundingEmulator]] +git-tree-sha1 = "40b9edad2e5287e05bd413a38f61a8ff55b9557b" +uuid = "5eaf0fd0-dfba-4ccb-bf02-d820a40db705" +version = "0.2.1" + +[[deps.RuntimeGeneratedFunctions]] +deps = ["ExprTools", "SHA", "Serialization"] +git-tree-sha1 = "6aacc5eefe8415f47b3e34214c1d79d2674a0ba2" +uuid = "7e49a35a-f44a-4d26-94aa-eba1b4ca6b47" +version = "0.5.12" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.SIMDTypes]] +git-tree-sha1 = "330289636fb8107c5f32088d2741e9fd7a061a5c" +uuid = "94e857df-77ce-4151-89e5-788b33177be4" +version = "0.1.0" + +[[deps.SLEEFPirates]] +deps = ["IfElse", "Static", "VectorizationBase"] +git-tree-sha1 = "3aac6d68c5e57449f5b9b865c9ba50ac2970c4cf" +uuid = "476501e8-09a2-5ece-8869-fb82de89a1fa" +version = "0.6.42" + +[[deps.SciMLBase]] +deps = ["ADTypes", "ArrayInterface", "CommonSolve", "ConstructionBase", "Distributed", "DocStringExtensions", "EnumX", "FillArrays", "FunctionWrappersWrappers", "IteratorInterfaceExtensions", "LinearAlgebra", "Logging", "Markdown", "PrecompileTools", "Preferences", "Printf", "RecipesBase", "RecursiveArrayTools", "Reexport", "RuntimeGeneratedFunctions", "SciMLOperators", "StaticArraysCore", "Statistics", "SymbolicIndexingInterface", "Tables", "TruncatedStacktraces"] +git-tree-sha1 = "d91985cfda7d730a885d7dbc89889e8184b72802" +uuid = "0bca4576-84f4-4d90-8ffe-ffa030f20462" +version = "2.21.0" + + [deps.SciMLBase.extensions] + SciMLBaseChainRulesCoreExt = "ChainRulesCore" + SciMLBasePartialFunctionsExt = "PartialFunctions" + SciMLBasePyCallExt = "PyCall" + SciMLBasePythonCallExt = "PythonCall" + SciMLBaseRCallExt = "RCall" + SciMLBaseZygoteExt = "Zygote" + + [deps.SciMLBase.weakdeps] + ChainRules = "082447d4-558c-5d27-93f4-14fc19e9eca2" + ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4" + PartialFunctions = "570af359-4316-4cb7-8c74-252c00c2016b" + PyCall = "438e738f-606a-5dbb-bf0a-cddfbfd45ab0" + PythonCall = "6099a3de-0909-46bc-b1f4-468b9a2dfc0d" + RCall = "6f49c342-dc21-5d91-9882-a32aef131414" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.SciMLOperators]] +deps = ["ArrayInterface", "DocStringExtensions", "Lazy", "LinearAlgebra", "Setfield", "SparseArrays", "StaticArraysCore", "Tricks"] +git-tree-sha1 = "51ae235ff058a64815e0a2c34b1db7578a06813d" +uuid = "c0aeaf25-5076-4817-a8d5-81caf7dfa961" +version = "0.3.7" + +[[deps.ScientificTypesBase]] +git-tree-sha1 = "a8e18eb383b5ecf1b5e6fc237eb39255044fd92b" +uuid = "30f210dd-8aff-4c5f-94ba-8e64358c1161" +version = "3.0.0" + +[[deps.Scratch]] +deps = ["Dates"] +git-tree-sha1 = "3bac05bc7e74a75fd9cba4295cde4045d9fe2386" +uuid = "6c6a2e73-6563-6170-7368-637461726353" +version = "1.2.1" + +[[deps.SentinelArrays]] +deps = ["Dates", "Random"] +git-tree-sha1 = "0e7508ff27ba32f26cd459474ca2ede1bc10991f" +uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c" +version = "1.4.1" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" + +[[deps.Setfield]] +deps = ["ConstructionBase", "Future", "MacroTools", "StaticArraysCore"] +git-tree-sha1 = "e2cc6d8c88613c05e1defb55170bf5ff211fbeac" +uuid = "efcf1570-3423-57d1-acb7-fd33fddbac46" +version = "1.1.1" + +[[deps.ShaderAbstractions]] +deps = ["ColorTypes", "FixedPointNumbers", "GeometryBasics", "LinearAlgebra", "Observables", "StaticArrays", "StructArrays", "Tables"] +git-tree-sha1 = "db0219befe4507878b1a90e07820fed3e62c289d" +uuid = "65257c39-d410-5151-9873-9b3e5be5013e" +version = "0.4.0" + +[[deps.SharedArrays]] +deps = ["Distributed", "Mmap", "Random", "Serialization"] +uuid = "1a1011a3-84de-559e-8e89-a11a2f7dc383" + +[[deps.ShiftedArrays]] +git-tree-sha1 = "503688b59397b3307443af35cd953a13e8005c16" +uuid = "1277b4bf-5013-50f5-be3d-901d8477a67a" +version = "2.0.0" + +[[deps.Showoff]] +deps = ["Dates", "Grisu"] +git-tree-sha1 = "91eddf657aca81df9ae6ceb20b959ae5653ad1de" +uuid = "992d4aef-0814-514b-bc4d-f2e9a6c4116f" +version = "1.0.3" + +[[deps.SignedDistanceFields]] +deps = ["Random", "Statistics", "Test"] +git-tree-sha1 = "d263a08ec505853a5ff1c1ebde2070419e3f28e9" +uuid = "73760f76-fbc4-59ce-8f25-708e95d2df96" +version = "0.4.0" + +[[deps.SimpleBufferStream]] +git-tree-sha1 = "874e8867b33a00e784c8a7e4b60afe9e037b74e1" +uuid = "777ac1f9-54b0-4bf8-805c-2214025038e7" +version = "1.1.0" + +[[deps.SimpleGraphs]] +deps = ["AbstractLattices", "Combinatorics", "DataStructures", "IterTools", "LightXML", "LinearAlgebra", "LinearAlgebraX", "Optim", "Primes", "Random", "RingLists", "SimplePartitions", "SimplePolynomials", "SimpleRandom", "SparseArrays", "Statistics"] +git-tree-sha1 = "f65caa24a622f985cc341de81d3f9744435d0d0f" +uuid = "55797a34-41de-5266-9ec1-32ac4eb504d3" +version = "0.8.6" + +[[deps.SimpleNonlinearSolve]] +deps = ["ADTypes", "ArrayInterface", "ConcreteStructs", "DiffEqBase", "FastClosures", "FiniteDiff", "ForwardDiff", "LinearAlgebra", "MaybeInplace", "PrecompileTools", "Reexport", "SciMLBase", "StaticArraysCore"] +git-tree-sha1 = "470c5f97af31fa35926b45eb01e53a46c8d7d35f" +uuid = "727e6d20-b764-4bd8-a329-72de5adea6c7" +version = "1.3.1" + + [deps.SimpleNonlinearSolve.extensions] + SimpleNonlinearSolvePolyesterForwardDiffExt = "PolyesterForwardDiff" + SimpleNonlinearSolveStaticArraysExt = "StaticArrays" + + [deps.SimpleNonlinearSolve.weakdeps] + PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b" + StaticArrays = "90137ffa-7385-5640-81b9-e52037218182" + +[[deps.SimplePartitions]] +deps = ["AbstractLattices", "DataStructures", "Permutations"] +git-tree-sha1 = "e9330391d04241eafdc358713b48396619c83bcb" +uuid = "ec83eff0-a5b5-5643-ae32-5cbf6eedec9d" +version = "0.3.1" + +[[deps.SimplePolynomials]] +deps = ["Mods", "Multisets", "Polynomials", "Primes"] +git-tree-sha1 = "7063828369cafa93f3187b3d0159f05582011405" +uuid = "cc47b68c-3164-5771-a705-2bc0097375a0" +version = "0.2.17" + +[[deps.SimpleRandom]] +deps = ["Distributions", "LinearAlgebra", "Random"] +git-tree-sha1 = "3a6fb395e37afab81aeea85bae48a4db5cd7244a" +uuid = "a6525b86-64cd-54fa-8f65-62fc48bdc0e8" +version = "0.3.1" + +[[deps.SimpleTraits]] +deps = ["InteractiveUtils", "MacroTools"] +git-tree-sha1 = "5d7e3f4e11935503d3ecaf7186eac40602e7d231" +uuid = "699a6c99-e7fa-54fc-8d76-47d257e15c1d" +version = "0.9.4" + +[[deps.SimpleUnPack]] +git-tree-sha1 = "58e6353e72cde29b90a69527e56df1b5c3d8c437" +uuid = "ce78b400-467f-4804-87d8-8f486da07d0a" +version = "1.1.0" + +[[deps.Sixel]] +deps = ["Dates", "FileIO", "ImageCore", "IndirectArrays", "OffsetArrays", "REPL", "libsixel_jll"] +git-tree-sha1 = "2da10356e31327c7096832eb9cd86307a50b1eb6" +uuid = "45858cf5-a6b0-47a3-bbea-62219f50df47" +version = "0.1.3" + +[[deps.Sockets]] +uuid = "6462fe0b-24de-5631-8697-dd941f90decc" + +[[deps.SortingAlgorithms]] +deps = ["DataStructures"] +git-tree-sha1 = "66e0a8e672a0bdfca2c3f5937efb8538b9ddc085" +uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c" +version = "1.2.1" + +[[deps.SparseArrays]] +deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"] +uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf" +version = "1.10.0" + +[[deps.SparseDiffTools]] +deps = ["ADTypes", "Adapt", "ArrayInterface", "Compat", "DataStructures", "FiniteDiff", "ForwardDiff", "Graphs", "LinearAlgebra", "PackageExtensionCompat", "Random", "Reexport", "SciMLOperators", "Setfield", "SparseArrays", "StaticArrayInterface", "StaticArrays", "Tricks", "UnPack", "VertexSafeGraphs"] +git-tree-sha1 = "3b38ae7a1cbe9b8b1344359599753957644b03d4" +uuid = "47a9eef4-7e08-11e9-0b38-333d64bd3804" +version = "2.16.0" + + [deps.SparseDiffTools.extensions] + SparseDiffToolsEnzymeExt = "Enzyme" + SparseDiffToolsPolyesterForwardDiffExt = "PolyesterForwardDiff" + SparseDiffToolsSymbolicsExt = "Symbolics" + SparseDiffToolsZygoteExt = "Zygote" + + [deps.SparseDiffTools.weakdeps] + Enzyme = "7da242da-08ed-463a-9acd-ee780be4f1d9" + PolyesterForwardDiff = "98d1487c-24ca-40b6-b7ab-df2af84e126b" + Symbolics = "0c5d862f-8b57-4792-8d23-62f2024744c7" + Zygote = "e88e6eb3-aa80-5325-afca-941959d7151f" + +[[deps.SparseInverseSubset]] +deps = ["LinearAlgebra", "SparseArrays", "SuiteSparse"] +git-tree-sha1 = "52962839426b75b3021296f7df242e40ecfc0852" +uuid = "dc90abb0-5640-4711-901d-7e5b23a2fada" +version = "0.1.2" + +[[deps.Sparspak]] +deps = ["Libdl", "LinearAlgebra", "Logging", "OffsetArrays", "Printf", "SparseArrays", "Test"] +git-tree-sha1 = "342cf4b449c299d8d1ceaf00b7a49f4fbc7940e7" +uuid = "e56a9233-b9d6-4f03-8d0f-1825330902ac" +version = "0.3.9" + +[[deps.SpecialFunctions]] +deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"] +git-tree-sha1 = "e2cfc4012a19088254b3950b85c3c1d8882d864d" +uuid = "276daf66-3868-5448-9aa4-cd146d93841b" +version = "2.3.1" +weakdeps = ["ChainRulesCore"] + + [deps.SpecialFunctions.extensions] + SpecialFunctionsChainRulesCoreExt = "ChainRulesCore" + +[[deps.SplittablesBase]] +deps = ["Setfield", "Test"] +git-tree-sha1 = "e08a62abc517eb79667d0a29dc08a3b589516bb5" +uuid = "171d559e-b47b-412a-8079-5efa626c420e" +version = "0.1.15" + +[[deps.StableHashTraits]] +deps = ["Compat", "PikaParser", "SHA", "Tables", "TupleTools"] +git-tree-sha1 = "662f56ffe22b3985f3be7474f0aecbaf214ecf0f" +uuid = "c5dd0088-6c3f-4803-b00e-f31a60c170fa" +version = "1.1.6" + +[[deps.StackViews]] +deps = ["OffsetArrays"] +git-tree-sha1 = "46e589465204cd0c08b4bd97385e4fa79a0c770c" +uuid = "cae243ae-269e-4f55-b966-ac2d0dc13c15" +version = "0.1.1" + +[[deps.Static]] +deps = ["IfElse"] +git-tree-sha1 = "f295e0a1da4ca425659c57441bcb59abb035a4bc" +uuid = "aedffcd0-7271-4cad-89d0-dc628f76c6d3" +version = "0.8.8" + +[[deps.StaticArrayInterface]] +deps = ["ArrayInterface", "Compat", "IfElse", "LinearAlgebra", "PrecompileTools", "Requires", "SparseArrays", "Static", "SuiteSparse"] +git-tree-sha1 = "5d66818a39bb04bf328e92bc933ec5b4ee88e436" +uuid = "0d7ed370-da01-4f52-bd93-41d350b8b718" +version = "1.5.0" +weakdeps = ["OffsetArrays", "StaticArrays"] + + [deps.StaticArrayInterface.extensions] + StaticArrayInterfaceOffsetArraysExt = "OffsetArrays" + StaticArrayInterfaceStaticArraysExt = "StaticArrays" + +[[deps.StaticArrays]] +deps = ["LinearAlgebra", "PrecompileTools", "Random", "StaticArraysCore"] +git-tree-sha1 = "f68dd04d131d9a8a8eb836173ee8f105c360b0c5" +uuid = "90137ffa-7385-5640-81b9-e52037218182" +version = "1.9.1" +weakdeps = ["ChainRulesCore", "Statistics"] + + [deps.StaticArrays.extensions] + StaticArraysChainRulesCoreExt = "ChainRulesCore" + StaticArraysStatisticsExt = "Statistics" + +[[deps.StaticArraysCore]] +git-tree-sha1 = "36b3d696ce6366023a0ea192b4cd442268995a0d" +uuid = "1e83bf80-4336-4d27-bf5d-d5a4f845583c" +version = "1.4.2" + +[[deps.StatisticalTraits]] +deps = ["ScientificTypesBase"] +git-tree-sha1 = "30b9236691858e13f167ce829490a68e1a597782" +uuid = "64bff920-2084-43da-a3e6-9bb72801c0c9" +version = "3.2.0" + +[[deps.Statistics]] +deps = ["LinearAlgebra", "SparseArrays"] +uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2" +version = "1.10.0" + +[[deps.StatsAPI]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1ff449ad350c9c4cbc756624d6f8a8c3ef56d3ed" +uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0" +version = "1.7.0" + +[[deps.StatsBase]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"] +git-tree-sha1 = "1d77abd07f617c4868c33d4f5b9e1dbb2643c9cf" +uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +version = "0.34.2" + +[[deps.StatsFuns]] +deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"] +git-tree-sha1 = "f625d686d5a88bcd2b15cd81f18f98186fdc0c9a" +uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +version = "1.3.0" +weakdeps = ["ChainRulesCore", "InverseFunctions"] + + [deps.StatsFuns.extensions] + StatsFunsChainRulesCoreExt = "ChainRulesCore" + StatsFunsInverseFunctionsExt = "InverseFunctions" + +[[deps.StatsModels]] +deps = ["DataAPI", "DataStructures", "LinearAlgebra", "Printf", "REPL", "ShiftedArrays", "SparseArrays", "StatsAPI", "StatsBase", "StatsFuns", "Tables"] +git-tree-sha1 = "5cf6c4583533ee38639f73b880f35fc85f2941e0" +uuid = "3eaba693-59b7-5ba5-a881-562e759f1c8d" +version = "0.7.3" + +[[deps.SteadyStateDiffEq]] +deps = ["ConcreteStructs", "DiffEqBase", "DiffEqCallbacks", "LinearAlgebra", "Reexport", "SciMLBase"] +git-tree-sha1 = "a735fd5053724cf4de31c81b4e2cc429db844be5" +uuid = "9672c7b4-1e72-59bd-8a11-6ac3964bc41f" +version = "2.0.1" + +[[deps.StochasticDiffEq]] +deps = ["Adapt", "ArrayInterface", "DataStructures", "DiffEqBase", "DiffEqNoiseProcess", "DocStringExtensions", "FiniteDiff", "ForwardDiff", "JumpProcesses", "LevyArea", "LinearAlgebra", "Logging", "MuladdMacro", "NLsolve", "OrdinaryDiffEq", "Random", "RandomNumbers", "RecursiveArrayTools", "Reexport", "SciMLBase", "SciMLOperators", "SparseArrays", "SparseDiffTools", "StaticArrays", "UnPack"] +git-tree-sha1 = "753219de57ac7aab0feb88871d3c51e0eb5e3b03" +uuid = "789caeaf-c7a9-5a7d-9973-96adeb23e2a0" +version = "6.64.0" + +[[deps.StrideArraysCore]] +deps = ["ArrayInterface", "CloseOpenIntervals", "IfElse", "LayoutPointers", "ManualMemory", "SIMDTypes", "Static", "StaticArrayInterface", "ThreadingUtilities"] +git-tree-sha1 = "d6415f66f3d89c615929af907fdc6a3e17af0d8c" +uuid = "7792a7ef-975c-4747-a70f-980b88e8d1da" +version = "0.5.2" + +[[deps.StringManipulation]] +deps = ["PrecompileTools"] +git-tree-sha1 = "a04cabe79c5f01f4d723cc6704070ada0b9d46d5" +uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e" +version = "0.3.4" + +[[deps.StructArrays]] +deps = ["Adapt", "ConstructionBase", "DataAPI", "GPUArraysCore", "StaticArraysCore", "Tables"] +git-tree-sha1 = "1b0b1205a56dc288b71b1961d48e351520702e24" +uuid = "09ab397b-f2b6-538f-b94a-2f83cf4a842a" +version = "0.6.17" + +[[deps.SuiteSparse]] +deps = ["Libdl", "LinearAlgebra", "Serialization", "SparseArrays"] +uuid = "4607b0f0-06f3-5cda-b6b1-a6196a1729e9" + +[[deps.SuiteSparse_jll]] +deps = ["Artifacts", "Libdl", "libblastrampoline_jll"] +uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c" +version = "7.2.1+1" + +[[deps.Sundials]] +deps = ["CEnum", "DataStructures", "DiffEqBase", "Libdl", "LinearAlgebra", "Logging", "PrecompileTools", "Reexport", "SciMLBase", "SparseArrays", "Sundials_jll"] +git-tree-sha1 = "c25a1ea9a5699895a44a60bf01b341778bcccac5" +uuid = "c3572dad-4567-51f8-b174-8c6c989267f4" +version = "4.23.2" + +[[deps.Sundials_jll]] +deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl", "SuiteSparse_jll", "libblastrampoline_jll"] +git-tree-sha1 = "ba4d38faeb62de7ef47155ed321dce40a549c305" +uuid = "fb77eaff-e24c-56d4-86b1-d163f2edb164" +version = "5.2.2+0" + +[[deps.SymbolicIndexingInterface]] +git-tree-sha1 = "34573fc920adfd457c5be704098d0168e4f20e54" +uuid = "2efcf032-c050-4f8e-a9bb-153293bab1f5" +version = "0.3.4" + +[[deps.TOML]] +deps = ["Dates"] +uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76" +version = "1.0.3" + +[[deps.TableTraits]] +deps = ["IteratorInterfaceExtensions"] +git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39" +uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c" +version = "1.0.1" + +[[deps.Tables]] +deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "LinearAlgebra", "OrderedCollections", "TableTraits"] +git-tree-sha1 = "cb76cf677714c095e535e3501ac7954732aeea2d" +uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c" +version = "1.11.1" + +[[deps.Tar]] +deps = ["ArgTools", "SHA"] +uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e" +version = "1.10.0" + +[[deps.TensorCore]] +deps = ["LinearAlgebra"] +git-tree-sha1 = "1feb45f88d133a655e001435632f019a9a1bcdb6" +uuid = "62fd8b95-f654-4bbd-a8a5-9c27f68ccd50" +version = "0.1.1" + +[[deps.TerminalLoggers]] +deps = ["LeftChildRightSiblingTrees", "Logging", "Markdown", "Printf", "ProgressLogging", "UUIDs"] +git-tree-sha1 = "f133fab380933d042f6796eda4e130272ba520ca" +uuid = "5d786b92-1e48-4d6f-9151-6b4477ca9bed" +version = "0.1.7" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[[deps.ThreadingUtilities]] +deps = ["ManualMemory"] +git-tree-sha1 = "eda08f7e9818eb53661b3deb74e3159460dfbc27" +uuid = "8290d209-cae3-49c0-8002-c8c24d57dab5" +version = "0.5.2" + +[[deps.TiffImages]] +deps = ["ColorTypes", "DataStructures", "DocStringExtensions", "FileIO", "FixedPointNumbers", "IndirectArrays", "Inflate", "Mmap", "OffsetArrays", "PkgVersion", "ProgressMeter", "UUIDs"] +git-tree-sha1 = "34cc045dd0aaa59b8bbe86c644679bc57f1d5bd0" +uuid = "731e570b-9d59-4bfa-96dc-6df516fadf69" +version = "0.6.8" + +[[deps.TimerOutputs]] +deps = ["ExprTools", "Printf"] +git-tree-sha1 = "f548a9e9c490030e545f72074a41edfd0e5bcdd7" +uuid = "a759f4b9-e2f1-59dc-863e-4aeb61b1ea8f" +version = "0.5.23" + +[[deps.Tracker]] +deps = ["Adapt", "DiffRules", "ForwardDiff", "Functors", "LinearAlgebra", "LogExpFunctions", "MacroTools", "NNlib", "NaNMath", "Optimisers", "Printf", "Random", "Requires", "SpecialFunctions", "Statistics"] +git-tree-sha1 = "5c942be30a85ac75d14e9e527b55504031e1bbd3" +uuid = "9f7883ad-71c0-57eb-9f7f-b5c9e6d3789c" +version = "0.2.31" +weakdeps = ["PDMats"] + + [deps.Tracker.extensions] + TrackerPDMatsExt = "PDMats" + +[[deps.TranscodingStreams]] +git-tree-sha1 = "1fbeaaca45801b4ba17c251dd8603ef24801dd84" +uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa" +version = "0.10.2" +weakdeps = ["Random", "Test"] + + [deps.TranscodingStreams.extensions] + TestExt = ["Test", "Random"] + +[[deps.Transducers]] +deps = ["Adapt", "ArgCheck", "BangBang", "Baselet", "CompositionsBase", "ConstructionBase", "DefineSingletons", "Distributed", "InitialValues", "Logging", "Markdown", "MicroCollections", "Requires", "Setfield", "SplittablesBase", "Tables"] +git-tree-sha1 = "3064e780dbb8a9296ebb3af8f440f787bb5332af" +uuid = "28d57a85-8fef-5791-bfe6-a80928e7c999" +version = "0.4.80" + + [deps.Transducers.extensions] + TransducersBlockArraysExt = "BlockArrays" + TransducersDataFramesExt = "DataFrames" + TransducersLazyArraysExt = "LazyArrays" + TransducersOnlineStatsBaseExt = "OnlineStatsBase" + TransducersReferenceablesExt = "Referenceables" + + [deps.Transducers.weakdeps] + BlockArrays = "8e7c35d0-a365-5155-bbbb-fb81a777f24e" + DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" + LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" + OnlineStatsBase = "925886fa-5bf2-5e8e-b522-a9147a512338" + Referenceables = "42d2dcc6-99eb-4e98-b66c-637b7d73030e" + +[[deps.TriangularSolve]] +deps = ["CloseOpenIntervals", "IfElse", "LayoutPointers", "LinearAlgebra", "LoopVectorization", "Polyester", "Static", "VectorizationBase"] +git-tree-sha1 = "fadebab77bf3ae041f77346dd1c290173da5a443" +uuid = "d5829a12-d9aa-46ab-831f-fb7c9ab06edf" +version = "0.1.20" + +[[deps.Tricks]] +git-tree-sha1 = "eae1bb484cd63b36999ee58be2de6c178105112f" +uuid = "410a4b4d-49e4-4fbc-ab6d-cb71b17b3775" +version = "0.1.8" + +[[deps.TriplotBase]] +git-tree-sha1 = "4d4ed7f294cda19382ff7de4c137d24d16adc89b" +uuid = "981d1d27-644d-49a2-9326-4793e63143c3" +version = "0.1.0" + +[[deps.TruncatedStacktraces]] +deps = ["InteractiveUtils", "MacroTools", "Preferences"] +git-tree-sha1 = "ea3e54c2bdde39062abf5a9758a23735558705e1" +uuid = "781d530d-4396-4725-bb49-402e4bee1e77" +version = "1.4.0" + +[[deps.TupleTools]] +git-tree-sha1 = "155515ed4c4236db30049ac1495e2969cc06be9d" +uuid = "9d95972d-f1c8-5527-a6e0-b4b365fa01f6" +version = "1.4.3" + +[[deps.Turing]] +deps = ["ADTypes", "AbstractMCMC", "AdvancedHMC", "AdvancedMH", "AdvancedPS", "AdvancedVI", "BangBang", "Bijectors", "DataStructures", "Distributions", "DistributionsAD", "DocStringExtensions", "DynamicPPL", "EllipticalSliceSampling", "ForwardDiff", "Libtask", "LinearAlgebra", "LogDensityProblems", "LogDensityProblemsAD", "MCMCChains", "NamedArrays", "Printf", "Random", "Reexport", "Requires", "SciMLBase", "Setfield", "SpecialFunctions", "Statistics", "StatsAPI", "StatsBase", "StatsFuns"] +git-tree-sha1 = "423987de297d186871c11fc18b5f0b3b337a8e57" +uuid = "fce5fe82-541a-59a6-adf8-730c64b5f9a0" +version = "0.30.3" + + [deps.Turing.extensions] + TuringDynamicHMCExt = "DynamicHMC" + TuringOptimExt = "Optim" + + [deps.Turing.weakdeps] + DynamicHMC = "bbc10e6e-7c05-544b-b16e-64fede858acb" + Optim = "429524aa-4258-5aef-a3af-852621145aeb" + +[[deps.URIs]] +git-tree-sha1 = "67db6cc7b3821e19ebe75791a9dd19c9b1188f2b" +uuid = "5c2747f8-b7ea-4ff2-ba2e-563bfd36b1d4" +version = "1.5.1" + +[[deps.UUIDs]] +deps = ["Random", "SHA"] +uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4" + +[[deps.UnPack]] +git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b" +uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed" +version = "1.0.2" + +[[deps.Unicode]] +uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5" + +[[deps.UnicodeFun]] +deps = ["REPL"] +git-tree-sha1 = "53915e50200959667e78a92a418594b428dffddf" +uuid = "1cfade01-22cf-5700-b092-accc4b62d6e1" +version = "0.4.1" + +[[deps.UnsafeAtomics]] +git-tree-sha1 = "6331ac3440856ea1988316b46045303bef658278" +uuid = "013be700-e6cd-48c3-b4a1-df204f14c38f" +version = "0.2.1" + +[[deps.UnsafeAtomicsLLVM]] +deps = ["LLVM", "UnsafeAtomics"] +git-tree-sha1 = "323e3d0acf5e78a56dfae7bd8928c989b4f3083e" +uuid = "d80eeb9a-aca5-4d75-85e5-170c8b632249" +version = "0.1.3" + +[[deps.VectorizationBase]] +deps = ["ArrayInterface", "CPUSummary", "HostCPUFeatures", "IfElse", "LayoutPointers", "Libdl", "LinearAlgebra", "SIMDTypes", "Static", "StaticArrayInterface"] +git-tree-sha1 = "7209df901e6ed7489fe9b7aa3e46fb788e15db85" +uuid = "3d5dd08c-fd9d-11e8-17fa-ed2836048c2f" +version = "0.21.65" + +[[deps.VertexSafeGraphs]] +deps = ["Graphs"] +git-tree-sha1 = "8351f8d73d7e880bfc042a8b6922684ebeafb35c" +uuid = "19fa3120-7c27-5ec5-8db8-b0b0aa330d6f" +version = "0.2.0" + +[[deps.WeakRefStrings]] +deps = ["DataAPI", "InlineStrings", "Parsers"] +git-tree-sha1 = "b1be2855ed9ed8eac54e5caff2afcdb442d52c23" +uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5" +version = "1.4.2" + +[[deps.WoodburyMatrices]] +deps = ["LinearAlgebra", "SparseArrays"] +git-tree-sha1 = "c1a7aa6219628fcd757dede0ca95e245c5cd9511" +uuid = "efce3f68-66dc-5838-9240-27a6d6f5f9b6" +version = "1.0.0" + +[[deps.WorkerUtilities]] +git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7" +uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60" +version = "1.6.1" + +[[deps.XML2_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libiconv_jll", "Zlib_jll"] +git-tree-sha1 = "801cbe47eae69adc50f36c3caec4758d2650741b" +uuid = "02c8fc9c-b97f-50b9-bbe4-9be30ff0a78a" +version = "2.12.2+0" + +[[deps.XSLT_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Libgcrypt_jll", "Libgpg_error_jll", "Libiconv_jll", "Pkg", "XML2_jll", "Zlib_jll"] +git-tree-sha1 = "91844873c4085240b95e795f692c4cec4d805f8a" +uuid = "aed1982a-8fda-507f-9586-7b0439959a61" +version = "1.1.34+0" + +[[deps.Xorg_libX11_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Xorg_libxcb_jll", "Xorg_xtrans_jll"] +git-tree-sha1 = "afead5aba5aa507ad5a3bf01f58f82c8d1403495" +uuid = "4f6342f7-b3d2-589e-9d20-edeb45f2b2bc" +version = "1.8.6+0" + +[[deps.Xorg_libXau_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "6035850dcc70518ca32f012e46015b9beeda49d8" +uuid = "0c0b7dd1-d40b-584c-a123-a41640f87eec" +version = "1.0.11+0" + +[[deps.Xorg_libXdmcp_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "34d526d318358a859d7de23da945578e8e8727b7" +uuid = "a3789734-cfe1-5b06-b2d0-1dd0d9d62d05" +version = "1.1.4+0" + +[[deps.Xorg_libXext_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "b7c0aa8c376b31e4852b360222848637f481f8c3" +uuid = "1082639a-0dae-5f34-9b06-72781eeb8cb3" +version = "1.3.4+4" + +[[deps.Xorg_libXrender_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg", "Xorg_libX11_jll"] +git-tree-sha1 = "19560f30fd49f4d4efbe7002a1037f8c43d43b96" +uuid = "ea2f1a96-1ddc-540d-b46f-429655e07cfa" +version = "0.9.10+4" + +[[deps.Xorg_libpthread_stubs_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "8fdda4c692503d44d04a0603d9ac0982054635f9" +uuid = "14d82f49-176c-5ed1-bb49-ad3f5cbd8c74" +version = "0.1.1+0" + +[[deps.Xorg_libxcb_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "XSLT_jll", "Xorg_libXau_jll", "Xorg_libXdmcp_jll", "Xorg_libpthread_stubs_jll"] +git-tree-sha1 = "b4bfde5d5b652e22b9c790ad00af08b6d042b97d" +uuid = "c7cfdc94-dc32-55de-ac96-5a1b8d977c5b" +version = "1.15.0+0" + +[[deps.Xorg_xtrans_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl"] +git-tree-sha1 = "e92a1a012a10506618f10b7047e478403a046c77" +uuid = "c5fb5394-a638-5e4d-96e5-b29de1b5cf10" +version = "1.5.0+0" + +[[deps.Zlib_jll]] +deps = ["Libdl"] +uuid = "83775a58-1f1d-513f-b197-d71354ab007a" +version = "1.2.13+1" + +[[deps.ZygoteRules]] +deps = ["ChainRulesCore", "MacroTools"] +git-tree-sha1 = "27798139afc0a2afa7b1824c206d5e87ea587a00" +uuid = "700de1a5-db45-46bc-99cf-38207098b444" +version = "0.2.5" + +[[deps.isoband_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "51b5eeb3f98367157a7a12a1fb0aa5328946c03c" +uuid = "9a68df92-36a6-505f-a73e-abb412b6bfb4" +version = "0.2.3+0" + +[[deps.libaom_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "3a2ea60308f0996d26f1e5354e10c24e9ef905d4" +uuid = "a4ae2306-e953-59d6-aa16-d00cac43593b" +version = "3.4.0+0" + +[[deps.libass_jll]] +deps = ["Artifacts", "Bzip2_jll", "FreeType2_jll", "FriBidi_jll", "HarfBuzz_jll", "JLLWrappers", "Libdl", "Pkg", "Zlib_jll"] +git-tree-sha1 = "5982a94fcba20f02f42ace44b9894ee2b140fe47" +uuid = "0ac62f75-1d6f-5e53-bd7c-93b484bb37c0" +version = "0.15.1+0" + +[[deps.libblastrampoline_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850b90-86db-534c-a0d3-1478176c7d93" +version = "5.8.0+1" + +[[deps.libfdk_aac_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "daacc84a041563f965be61859a36e17c4e4fcd55" +uuid = "f638f0a6-7fb0-5443-88ba-1cc74229b280" +version = "2.0.2+0" + +[[deps.libpng_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Zlib_jll"] +git-tree-sha1 = "93284c28274d9e75218a416c65ec49d0e0fcdf3d" +uuid = "b53b4c65-9356-5827-b1ea-8c7a1a84506f" +version = "1.6.40+0" + +[[deps.libsixel_jll]] +deps = ["Artifacts", "JLLWrappers", "JpegTurbo_jll", "Libdl", "Pkg", "libpng_jll"] +git-tree-sha1 = "d4f63314c8aa1e48cd22aa0c17ed76cd1ae48c3c" +uuid = "075b6546-f08a-558a-be8f-8157d0f608a5" +version = "1.10.3+0" + +[[deps.libvorbis_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Ogg_jll", "Pkg"] +git-tree-sha1 = "b910cb81ef3fe6e78bf6acee440bda86fd6ae00c" +uuid = "f27f6e37-5d2b-51aa-960f-b287f2bc3b7a" +version = "1.3.7+1" + +[[deps.nghttp2_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d" +version = "1.52.0+1" + +[[deps.p7zip_jll]] +deps = ["Artifacts", "Libdl"] +uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0" +version = "17.4.0+2" + +[[deps.x264_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "4fea590b89e6ec504593146bf8b988b2c00922b2" +uuid = "1270edf5-f2f9-52d2-97e9-ab00b5d0237a" +version = "2021.5.5+0" + +[[deps.x265_jll]] +deps = ["Artifacts", "JLLWrappers", "Libdl", "Pkg"] +git-tree-sha1 = "ee567a171cce03570d77ad3a43e90218e38937a9" +uuid = "dfaa095f-4041-5dcd-9319-2fabd8486b76" +version = "3.5.0+0" diff --git a/Project.toml b/Project.toml new file mode 100644 index 00000000..0162d579 --- /dev/null +++ b/Project.toml @@ -0,0 +1,52 @@ +[deps] +AlgebraOfGraphics = "cbdf2221-f076-402e-a563-3d30da359d67" +BenchmarkTools = "6e4b80f9-dd63-53aa-95a3-0cdb28fa8baf" +Bijectors = "76274a88-744f-5084-9051-94815aaf08c4" +CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b" +CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597" +Chain = "8be319e6-bccf-4806-a6f7-6fae938471bc" +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" +DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0" +DifferentialEquations = "0c46a032-eb83-5123-abaf-570d42b7fbaa" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" +Downloads = "f43a241f-c20a-4ad4-852c-f6b1247861c6" +FillArrays = "1a297f60-69ca-5386-bcde-b61e274b549b" +ForwardDiff = "f6369f11-7733-5829-9624-2563aa707210" +Franklin = "713c75ef-9fc9-4b05-94a9-213340da978e" +HTTP = "cd3eb016-35fb-5094-929b-558a96fad6f3" +LazyArrays = "5078a376-72f3-5289-bfd5-ec5146d43c02" +Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306" +MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d" +NodeJS = "2bd173c7-0d6d-553b-b6af-13a54713934c" +PDMats = "90014a1f-27ba-587c-ab20-58faa44d9150" +StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91" +StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c" +Turing = "fce5fe82-541a-59a6-adf8-730c64b5f9a0" + +[compat] +AlgebraOfGraphics = "0.6" +BenchmarkTools = "1" +Bijectors = "0.13" +CSV = "0.10" +CairoMakie = "0.11" +CategoricalArrays = "0.10" +Chain = "0.5" +Colors = "0.12" +DataFrames = "1" +DifferentialEquations = "7" +Distributions = "0.25" +Downloads = "1" +FillArrays = "1" +ForwardDiff = "0.10" +Franklin = "0.10" +HTTP = "1" +LazyArrays = "1" +Literate = "2" +MCMCChains = "6" +NodeJS = "2" +PDMats = "0.11" +StatsBase = "0.34" +StatsFuns = "1" +Turing = "0.30" +julia = "1" diff --git a/assets/favicon.ico b/assets/favicon.ico new file mode 100644 index 00000000..5f340eac Binary files /dev/null and b/assets/favicon.ico differ diff --git a/assets/favicon.png b/assets/favicon.png new file mode 100644 index 00000000..5fcfc959 Binary files /dev/null and b/assets/favicon.png differ diff --git a/assets/hamburger.svg b/assets/hamburger.svg new file mode 100644 index 00000000..233bf217 --- /dev/null +++ b/assets/hamburger.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/assets/index/code/output/packages.out b/assets/index/code/output/packages.out new file mode 100644 index 00000000..39a460be --- /dev/null +++ b/assets/index/code/output/packages.out @@ -0,0 +1,24 @@ +AlgebraOfGraphics 0.6.18 +BenchmarkTools 1.4.0 +Bijectors 0.13.8 +CSV 0.10.12 +CairoMakie 0.11.6 +CategoricalArrays 0.10.8 +Chain 0.5.0 +Colors 0.12.10 +DataFrames 1.6.1 +DifferentialEquations 7.12.0 +Distributions 0.25.107 +Downloads 1.6.0 +FillArrays 1.9.3 +ForwardDiff 0.10.36 +Franklin 0.10.95 +HTTP 1.10.1 +LazyArrays 1.8.3 +Literate 2.16.1 +MCMCChains 6.0.4 +NodeJS 2.0.0 +PDMats 0.11.31 +StatsBase 0.34.2 +StatsFuns 1.3.0 +Turing 0.30.3 diff --git a/assets/index/code/output/packages.res b/assets/index/code/output/packages.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/index/code/output/packages.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/index/code/output/version.out b/assets/index/code/output/version.out new file mode 100644 index 00000000..81c871de --- /dev/null +++ b/assets/index/code/output/version.out @@ -0,0 +1 @@ +1.10.0 diff --git a/assets/index/code/output/version.res b/assets/index/code/output/version.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/index/code/output/version.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/index/code/packages.jl b/assets/index/code/packages.jl new file mode 100644 index 00000000..4b3d0ca6 --- /dev/null +++ b/assets/index/code/packages.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +# hideall +import Pkg + +deps = [pair.second for pair in Pkg.dependencies()] +deps = filter(p -> p.is_direct_dep, deps) +deps = filter(p -> !isnothing(p.version), deps) +list = ["$(p.name) $(p.version)" for p in deps] +sort!(list) +println(join(list, '\n')) \ No newline at end of file diff --git a/assets/index/code/version.jl b/assets/index/code/version.jl new file mode 100644 index 00000000..9aa3a972 --- /dev/null +++ b/assets/index/code/version.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +# hideall +println(VERSION) \ No newline at end of file diff --git a/assets/literate/01_why_Julia.md b/assets/literate/01_why_Julia.md new file mode 100644 index 00000000..63fdf770 --- /dev/null +++ b/assets/literate/01_why_Julia.md @@ -0,0 +1,753 @@ + +# Why Julia? + +[Julia](https://www.julialang.org) (Bezanson, Edelman, Karpinski & Shah, 2017) is a relatively new language, +first released in 2012, aims to be both **high-level** and **fast**. +Julia is a fast dynamic-typed language that just-in-time (JIT) compiles into native code using LLVM. +It ["runs like C but reads like Python"](https://www.nature.com/articles/d41586-019-02310-3), +meaning that is *blazing* fast, easy to prototype and read/write code. +It is **multi-paradigm**, combining features of imperative, functional, and object-oriented programming. + +**Why was Julia created?** Definitely read this now impressively +[old post by Julia founders](https://julialang.org/blog/2012/02/why-we-created-julia/). +Here is a clarifying quote: + +> We want the speed of C with the dynamism of Ruby. We want a language that's homoiconic, with true macros like Lisp, but with obvious, +> familiar mathematical notation like Matlab. We want something as usable for general programming as Python, as easy for statistics as R, +> as natural for string processing as Perl, as powerful for linear algebra as Matlab, as good at gluing programs together as the shell. +> Something that is dirt simple to learn, yet keeps the most serious hackers happy. We want it interactive and we want it compiled. + +**Why this needs to be an extra language?** Why cannot Python (or R) be made that fast for instance? +See the official compact answer to this in the +[Julia manual FAQ](https://docs.julialang.org/en/v1/manual/faq/#Why-don't-you-compile-Matlab/Python/R/%E2%80%A6-code-to-Julia?): + +> The basic issue is that there is nothing special about Julia's compiler: +> we use a commonplace compiler (LLVM) with no "secret sauce" that other language developers don't know about. +> Julia's performance advantage derives almost entirely from its front-end: its language semantics allow a well-written Julia program +> to give more opportunities to the compiler to generate efficient code and memory layouts. +> If you tried to compile Matlab or Python code to Julia, +> our compiler would be limited by the semantics of Matlab or +> Python to producing code no better than that of existing compilers for those languages (and probably worse). +> +> Julia's advantage is that good performance is not limited to a small subset of "built-in" types and operations, +> and one can write high-level type-generic code that works on arbitrary user-defined types while remaining fast and memory-efficient. +> Types in languages like Python simply don't provide enough information to the compiler for similar capabilities, +> so as soon as you used those languages as a Julia front-end you would be stuck. + +These are the "official" answers from the Julia community. +Now let me share with you my opinion. +From my point-of-view Julia has **three main features that makes it a unique language to work with**, +specially in scientific computing: + +* **Speed** +* **Ease of Use** +* **Multiple Dispatch** + +Now let's dive into each one of those three features. + +## Speed + +Yes, Julia is **fast**. **Very fast!** It was made for speed from the drawing board. +It bypass any sort of intermediate representation and translate code into machine native code using LLVM compiler. +Comparing this with R, that uses either FORTRAN or C, or Python, that uses CPython; +and you'll clearly see that Julia has a major speed advantage over other languages that are common in data science and statistics. +Julia exposes the machine code to LLVM's compiler which in turn can optimize code as it wishes, +like a good compiler such as LLVM excels in. + +One notable example: NASA uses Julia to analyze the +"[Largest Batch of Earth-Sized Planets Ever Found](https://exoplanets.nasa.gov/news/1669/seven-rocky-trappist-1-planets-may-be-made-of-similar-stuff/)". +Also, you can find [benchmarks](https://julialang.org/benchmarks/) for a range of common code patterns, +such as function calls, string parsing, sorting, numerical loops, random number generation, recursion, +and array operations using Julia and also several other languages such as C, Rust, Go, JavaScript, R, Python, Fortran and Java. +The figure below was taken from [Julia's website](https://julialang.org/benchmarks/). +As you can see Julia is **indeed** fast: + +![Common Benchmarks](/pages/images/benchmarks.svg) + +\center{*Common Benchmarks*} \\ + +Let me demonstrate how fast Julia is. +Here is a simple "groupby" operation using random stuff to emulate common data analysis +"split-apply-combine" operations in three languages[^updatedversion] : + +* Julia: using [`DataFrames.jl`](https://dataframes.juliadata.org/stable/) - 0.4ms +* Python: using `Pandas` and `NumPy` - 1.76ms +* R: using `{dplyr}` - 3.22ms + +Here is Julia: + +```julia +using Random +using StatsBase +using DataFrames +using BenchmarkTools +using Chain +Random.seed!(123) + +n = 10_000 + +df = DataFrame( + x=sample(["A", "B", "C", "D"], n, replace=true), + y=rand(n), + z=randn(n), +) + +@btime @chain $df begin # passing `df` as reference so the compiler cannot optimize + groupby(:x) + combine(:y => median, :z => mean) +end +``` + +Here is Python: + +```python +import pandas as pd +import numpy as np + +n = 10000 + +df = pd.DataFrame({'x': np.random.choice(['A', 'B', 'C', 'D'], n, replace=True), + 'y': np.random.randn(n), + 'z': np.random.rand(n)}) + +%timeit df.groupby('x').agg({'y': 'median', 'z': 'mean'}) +``` + +Here is R: + +```r +library(dplyr) + +n <- 10e3 +df <- tibble( + x = sample(c("A", "B", "C", "D"), n, replace = TRUE), + y = runif(n), + z = rnorm(n) +) + +bench::mark( + df %>% + group_by(x) %>% + summarize( + median(y), + mean(z) + ) +) +``` + +So clearly **Julia is the winner here**, being **4x faster than Python** and almost **10x faster than R**. +Also note that `Pandas` (along with `NumPy`) and `{dplyr}` are all written in C or C++. +Additionally, I didn't let Julia cheat by allowing the compiler optimize for `df` by passing a reference `$df`. +So, I guess this is a fair comparison. + +## Ease of Use + +What is most striking is that Julia can be as fast as C (and faster than Java in some applications) +while **having a very simple and intelligible syntax**. +This feature along with its speed is what Julia creators denote as **"the two language problem"** that Julia addresses. +The **"two language problem" is a very typical situation in scientific computing** +where a researcher or computer scientist devises an algorithm or a solution that he or she +prototypes in an easy to code language (like Python) and, if it works, +he or she would code in a fast language that is not easy to code (C or FORTRAN). +Thus, we have two languages involved in the process of developing a new solution. +One which is easy to prototype but is not suited for implementation (mostly due to being slow). +And another one which is not so easy to code (and, consequently, not easy to prototype) +but suited for implementation (mostly because it is fast). +Julia comes to **eliminate such situations** by being the **same language** that you **prototype** (ease of use) +and **implement the solution** (speed). + +Also, Julia lets you use **unicode characters as variables or parameters**. +This means no more using `sigma` or `sigma_i`, and instead just use `σ` or `σᵢ` as you would in mathematical notation. +When you see code for an algorithm or for a mathematical equation you see a **one-to-one relation to code and math**. +This is a **powerful** feature. + +I think that the "two language problem" and the one-to-one code and math relation are best described by +one of the creators of Julia, Alan Edelman, in a **TED Talk** (see the video below): + +~~~ +
+~~~ + +I will try to exemplify what would be the "two language problem" by showing you how I would code a simple +[**Metropolis algorithm**](https://en.wikipedia.org/wiki/Metropolis%E2%80%93Hastings_algorithm) +for a **bivariate normal distribution**. I would mostly prototype it in a dynamically-typed language such as R or Python. +Then, deploy the algorithm using a fast but hard to code language such as C++. This is exactly what I'll do now. +The algorithm will be coded in **Julia**, **R**, **C++** and [**`Stan`**](https://mc-stan.org). +There are two caveats. +First, I am coding the **original 1950s Metropolis version**, not the **1970s Metropolis-Hastings**, +which implies **symmetrical proposal distributions** just for the sake of the example. +Second, the proposals are based on a **uniform distribution** on the current proposal values of the proposal values ± a certain `width`. + +Let's start with **Julia** which uses the [`Distributions.jl`](https://juliastats.org/Distributions.jl/stable/) +package for its probabilistic distributions along with `logpdf()` defined methods for all of the distributions. + +```julia +using Distributions +function metropolis(S::Int64, width::Float64, ρ::Float64; + μ_x::Float64=0.0, μ_y::Float64=0.0, + σ_x::Float64=1.0, σ_y::Float64=1.0) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]); + draws = Matrix{Float64}(undef, S, 2); + x = randn(); y = randn(); + accepted = 0::Int64; + for s in 1:S + x_ = rand(Uniform(x - width, x + width)); + y_ = rand(Uniform(y - width, y + width)); + r = exp(logpdf(binormal, [x_, y_]) - logpdf(binormal, [x, y])); + + if r > rand(Uniform()) + x = x_; + y = y_; + accepted += 1; + end + @inbounds draws[s, :] = [x y]; + end + println("Acceptance rate is $(accepted / S)") + return draws +end +``` + +Now let's go to the **R version** (from now on no more fancy names like `μ` or `σ` 😭). +Since this is a bivariate normal I am using the package [`{mnormt}`](https://cran.r-project.org/web/packages/mnormt/index.html) +which allows for very fast (FORTRAN code) computation of multivariate normal distributions' pdf and logpdf. + +```r +metropolis <- function(S, width, + mu_X = 0, mu_Y = 0, + sigma_X = 1, sigma_Y = 1, + rho) { + Sigma <- diag(2) + Sigma[1, 2] <- rho + Sigma[2, 1] <- rho + draws <- matrix(nrow = S, ncol = 2) + x <- rnorm(1) + y <- rnorm(1) + accepted <- 0 + for (s in 1:S) { + x_ <- runif(1, x - width, x + width) + y_ <- runif(1, y - width, y + width) + r <- exp(mnormt::dmnorm(c(x_, y_), mean = c(mu_X, mu_Y), varcov = Sigma, log = TRUE) - + mnormt::dmnorm(c(x, y), mean = c(mu_X, mu_Y), varcov = Sigma, log = TRUE)) + if (r > runif(1, 0, 1)) { + x <- x_ + y <- y_ + accepted <- accepted + 1 + } + draws[s, 1] <- x + draws[s, 2] <- y + } + print(paste0("Acceptance rate is ", accepted / S)) + return(draws) +} +``` + +Now **C++**. +Here I am using the [`Eigen`](https://eigen.tuxfamily.org/) library. +Note that, since C++ is a very powerful language to be used as "close to the metal" as possible, +I don't have any convenient predefined multivariate normal to use. +So I will have to create this from zero[^mvnimplem].\ +Ok, be **ready**! +This is a mouthful: + +```cpp +#include +#include +#include +#include + +using std::cout; +std::random_device rd{}; +std::mt19937 gen{rd()}; + +// Random Number Generator Stuff +double random_normal(double mean = 0, double std = 1) { + std::normal_distribution d{mean, std}; + return d(gen); +}; + +double random_unif(double min = 0, double max = 1) { + std::uniform_real_distribution d{min, max}; + return d(gen); +}; + +// Multivariate Normal +struct Mvn { + Mvn(const Eigen::VectorXd &mu, const Eigen::MatrixXd &s) + : mean(mu), sigma(s) {} + double pdf(const Eigen::VectorXd &x) const; + double lpdf(const Eigen::VectorXd &x) const; + Eigen::VectorXd mean; + Eigen::MatrixXd sigma; +}; + +double Mvn::pdf(const Eigen::VectorXd &x) const { + double n = x.rows(); + double sqrt2pi = std::sqrt(2 * M_PI); + double quadform = (x - mean).transpose() * sigma.inverse() * (x - mean); + double norm = std::pow(sqrt2pi, -n) * std::pow(sigma.determinant(), -0.5); + + return norm * exp(-0.5 * quadform); +} + +double Mvn::lpdf(const Eigen::VectorXd &x) const { + double n = x.rows(); + double sqrt2pi = std::sqrt(2 * M_PI); + double quadform = (x - mean).transpose() * sigma.inverse() * (x - mean); + double norm = std::pow(sqrt2pi, -n) * std::pow(sigma.determinant(), -0.5); + + return log(norm) + (-0.5 * quadform); +} + +Eigen::MatrixXd metropolis(int S, double width, double mu_X = 0, + double mu_Y = 0, double sigma_X = 1, + double sigma_Y = 1, double rho = 0.8) { + Eigen::MatrixXd sigma(2, 2); + sigma << sigma_X, rho, rho, sigma_Y; + Eigen::VectorXd mean(2); + mean << mu_X, mu_Y; + Mvn binormal(mean, sigma); + + Eigen::MatrixXd out(S, 2); + double x = random_normal(); + double y = random_normal(); + double accepted = 0; + for (size_t i = 0; i < S - 1; i++) { + double xmw = x - width; + double xpw = x + width; + double ymw = y - width; + double ypw = y + width; + + double x_ = random_unif(xmw, xpw); + double y_ = random_unif(ymw, ypw); + + double r = std::exp(binormal.lpdf(Eigen::Vector2d(x_, y_)) - + binormal.lpdf(Eigen::Vector2d(x, y))); + if (r > random_unif()) { + x = x_; + y = y_; + accepted++; + } + out(i, 0) = x; + out(i, 1) = y; + } + cout << "Acceptance rate is " << accepted / S << '\n'; + + return out; +} +``` + +note that the [PDF for a multivariate normal](https://en.wikipedia.org/wiki/Multivariate_normal_distribution) is: + +$$ \text{PDF}(\boldsymbol{\mu}, \boldsymbol{\Sigma}) = (2\pi)^{-{\frac{k}{2}}}\det({\boldsymbol{\Sigma}})^{-{\frac {1}{2}}}e^{-{\frac{1}{2}}(\mathbf{x}-{\boldsymbol{\mu}})^{T}{\boldsymbol{\Sigma }}^{-1}(\mathbf{x} -{\boldsymbol{\mu}})} \label{mvnpdf} , $$ + +where $\boldsymbol{\mu}$ is a vector of means, $k$ is the number of dimensions, $\boldsymbol{\Sigma}$ is a covariance matrix, +$\det$ is the determinant and $\mathbf{x}$ is a vector of values that the PDF is evaluated for. + +**SPOILER ALERT**: Julia will beat this C++ Eigen implementation by being almost 100x faster. +So I will try to *help* C++ beat Julia (😂) by making a bivariate normal class `BiNormal` +in order to avoid the expensive operation of inverting a covariance matrix and +computing determinants in every logpdf proposal evaluation. +Also since we are not doing linear algebra computations I've removed Eigen and used C++ STL's ``: + +```cpp +#define M_PI 3.14159265358979323846 /* pi */ + +// Bivariate Normal +struct BiNormal { + BiNormal(const std::vector &mu, const double &rho) + : mean(mu), rho(rho) {} + double pdf(const std::vector &x) const; + double lpdf(const std::vector &x) const; + std::vector mean; + double rho; +}; + +double BiNormal::pdf(const std::vector &x) const { + double x_ = x[0]; + double y_ = x[1]; + return std::exp(-((std::pow(x_, 2) - (2 * rho * x_ * y_) + std::pow(y_, 2)) / + (2 * (1 - std::pow(rho, 2))))) / + (2 * M_PI * std::sqrt(1 - std::pow(rho, 2))); +} + +double BiNormal::lpdf(const std::vector &x) const { + double x_ = x[0]; + double y_ = x[1]; + return (-((std::pow(x_, 2) - (2 * rho * x_ * y_) + std::pow(y_, 2))) / + (2 * (1 - std::pow(rho, 2)))) - + std::log(2) - std::log(M_PI) - log(std::sqrt(1 - std::pow(rho, 2))); +} +``` + +This means that I've simplified the PDF [^mathbinormal] from equation \eqref{mvnpdf} into: + +$$ \text{PDF}(x, y)= \frac{1}{2 \pi \sqrt{1 - \rho^2 } \sigma_X \sigma_Y} e^{-\frac{\frac{x^{2}}{\sigma_{X}^{2}}-2 \rho-\frac{x y}{\sigma_{X} \sigma_{Y}}+\frac{y^{2}}{\sigma_{Y}^{2}}}{2\left(1-\rho^{2}\right)}} \label{bvnpdf} .$$ + +Since $\sigma_{X} = \sigma_{Y} = 1$, equation \eqref{bvnpdf} boils down to: + +$$ \text{PDF}(x, y)=\frac{1}{2 \pi \sqrt{1 - \rho^2 }} e^{-\frac{x^{2} -2 \rho-x y+y^{2}}{2\left(1-\rho^{2}\right)}} .$$ + +no more determinants or matrix inversions. Easy-peasy for C++. + +Now let's go to the last, +but not least: [`Stan`](https://mc-stan.org) is a probabilistic language for specifying probabilistic models (does the same as `Turing.jl` does) +and comes also with a very fast C++-based MCMC sampler. +`Stan` is a personal favorite of mine and I have a [whole graduate course of Bayesian statistics using `Stan`](https://storopoli.io/Bayesian-Statistics/). +Here's the `Stan` implementation: + +```stan +functions { + real binormal_lpdf(real [] xy, real mu_X, real mu_Y, real sigma_X, real sigma_Y, real rho) { + real beta = rho * sigma_Y / sigma_X; real sigma = sigma_Y * sqrt(1 - square(rho)); + return normal_lpdf(xy[1] | mu_X, sigma_X) + + normal_lpdf(xy[2] | mu_Y + beta * (xy[1] - mu_X), sigma); + } + + matrix metropolis_rng(int S, real width, + real mu_X, real mu_Y, + real sigma_X, real sigma_Y, + real rho) { + matrix[S, 2] out; real x = normal_rng(0, 1); real y = normal_rng(0, 1); real accepted = 0; + for (s in 1:S) { + real xmw = x - width; real xpw = x + width; real ymw = y - width; real ypw = y + width; + real x_ = uniform_rng(xmw, xpw); real y_ = uniform_rng(ymw, ypw); + real r = exp(binormal_lpdf({x_, y_} | mu_X, mu_Y, sigma_X, sigma_Y, rho) - + binormal_lpdf({x , y } | mu_X, mu_Y, sigma_X, sigma_Y, rho)); + if (r > uniform_rng(0, 1)) { + x = x_; y = y_; accepted += 1; + } + out[s, 1] = x; out[s, 2] = y; + } + print("Acceptance rate is ", accepted / S); + return out; + } +} +``` + +Wow, that was lot... +Not let's go to the results. +I've benchmarked R and `Stan` code using `{bench}` and `{rstan}` packages, C++ using `catch2`, Julia using `BenchmarkTools.jl`. +For all benchmarks the parameters were: `S = 10_000` simulations, `width = 2.75` and `ρ = 0.8`. +From fastest to slowest: + +* `Stan` - 3.6ms +* Julia - 6.3ms +* C++ `BiNormal` - 17ms +* C++ `MvNormal` - 592ms +* R - 1.35s which means 1350ms + +**Conclusion**: a naïve Julia implementation beats C++ +(while also beating a C++ math-helped faster implementation using bivariate normal PDFs) and gets very close to `Stan`, +a highly specialized probabilistic language that compiles and runs on C++ with lots of contributors, +funding and development time invested. + +Despite being *blazing* fast, Julia also **codes very easily**.\ +You can write and read code without much effort. + +## Multiple Dispatch + +I think that this is the **real game changer of Julia language**: +The ability to define **function behavior** across many combinations of argument types via [**multiple dispatch**](https://en.wikipedia.org/wiki/Multiple_dispatch). +**Multiple dispatch** is a feature that allows a function or method to be **dynamically dispatched** based on the run-time (dynamic) type or, +in the more general case, some other attribute of more than one of its arguments. +This is a **generalization of single-dispatch polymorphism** where a function or method call is dynamically dispatched based on the derived type of +the object on which the method has been called. Multiple dispatch routes the dynamic dispatch to the +implementing function or method using the combined characteristics of one or more arguments. + +Most languages have single-dispatch polymorphism that rely on the first parameter of a method in order to +determine which method should be called. But what Julia differs is that **multiple parameters are taken into account**. +This enables multiple definitions of similar functions that have the same initial parameter. +I think that this is best explained by one of the creators of Julia, Stefan Karpinski, at JuliaCon 2019 (see the video below): + +~~~ +
+~~~ + +### Example: Dogs and Cats + +I will reproduce Karpinski's example. In the talk, Karpinski designs a structure of classes which are very common in object-oriented programming (OOP). +In Julia, we don't have classes but we have **structures** (`struct`) that are meant to be "structured data": +they define the kind of information that is embedded in the structure, +that is a set of fields (aka "properties" or "attributes" in other languages), +and then individual instances (or "objects") can be produced each with its own specific values for the fields defined by the structure. + +We create an abstract `type` called `Pet`. +Then, we proceed by creating two derived `struct` from `Pet` that has one field `name` (a `String`). +These derived `struct` are `Dog` and `Cat`. +We also define some methods for what happens in an "encounter" by defining a generic function `meets()` +and several specific methods of `meets()` that will be multiple dispatched by Julia in runtime +to define the action that one type `Pet` takes when it meets another `Pet`: + +````julia:ex1 +abstract type Pet end +struct Dog <: Pet + name::String +end +struct Cat <: Pet + name::String +end + +function encounter(a::Pet, b::Pet) + verb = meets(a, b) + return println("$(a.name) meets $(b.name) and $verb") +end + +meets(a::Dog, b::Dog) = "sniffs"; +meets(a::Dog, b::Cat) = "chases"; +meets(a::Cat, b::Dog) = "hisses"; +meets(a::Cat, b::Cat) = "slinks"; +```` + +Let's see what happens when we instantiate objects from `Dog` and `Cat` and +call `encounter` on them in Julia: + +````julia:ex2 +fido = Dog("Fido"); +rex = Dog("Rex"); +whiskers = Cat("Whiskers"); +spots = Cat("Spots"); + +encounter(fido, rex) +encounter(rex, whiskers) +encounter(spots, fido) +encounter(whiskers, spots) +```` + +It works as expected. +Now let's translate this to modern C++ as literally as possible. +Let's define a class `Pet` with a member variable `name` -- in C ++ we can do this. +Then we define a base function `meets()`, +a function `encounter() `for two objects of the type `Pet`, and finally, define derived classes `Dog `and `Cat` +overload `meets()` for them: + +```cpp +#include +#include + +using std::string; +using std::cout; + +class Pet { + public: + string name; +}; + +string meets(Pet a, Pet b) { return "FALLBACK"; } // If we use `return meets(a, b)` doesn't work + +void encounter(Pet a, Pet b) { + string verb = meets(a, b); + cout << a.name << " meets " + << b. name << " and " << verb << '\n'; +} + +class Cat : public Pet {}; +class Dog : public Pet {}; + +string meets(Dog a, Dog b) { return "sniffs"; } +string meets(Dog a, Cat b) { return "chases"; } +string meets(Cat a, Dog b) { return "hisses"; } +string meets(Cat a, Cat b) { return "slinks"; } +``` + +Now we add a `main()` function to the C++ script: + +```cpp +int main() { + Dog fido; fido.name = "Fido"; + Dog rex; rex.name = "Rex"; + Cat whiskers; whiskers.name = "Whiskers"; + Cat spots; spots.name = "Spots"; + + encounter(fido, rex); + encounter(rex, whiskers); + encounter(spots, fido); + encounter(whiskers, spots); + + return 0; +} +``` + +And this is what we get: + +```bash +g++ main.cpp && ./a.out + +Fido meets Rex and FALLBACK +Rex meets Whiskers and FALLBACK +Spots meets Fido and FALLBACK +Whiskers meets Spots and FALLBACK +``` + +Doesn't work... 🤷🏼 + +### Example: One-hot Vector + +Now let's change to another nice example of creating a [one-hot vector](https://en.wikipedia.org/wiki/One-hot). +One-hot vector is a vector of integers in which all indices are zero (0) except for one single index that is one (1). +In machine learning, one-hot encoding is a frequently used method to deal with categorical data. Because many machine +learning models need their input variables to be numeric, categorical variables need to be transformed in the pre-processing part. +The example below is heavily inspired by a [post from Vasily Pisarev](https://habr.com/ru/post/468609/)[^onehotpost]. + +How we would represent one-hot vectors in Julia? Simple: we create a new type `OneHotVector` in Julia using the `struct` keyword +and define two fields `len` and `ind`, which represents the `OneHotVector` length and which index is the entry 1 +(*i.e.* which index is "hot"). Then, we define new methods for the `Base` functions `size()` and `getindex()` for our newly defined +`OneHotVector`. + +````julia:ex3 +import Base: size, getindex + +struct OneHotVector <: AbstractVector{Int} + len::Int + ind::Int +end + +size(v::OneHotVector) = (v.len,) + +getindex(v::OneHotVector, i::Integer) = Int(i == v.ind) +```` + +Since `OneHotVector` is a `struct` derived from `AbstractVector` we can use all of the methods previously defined for +`AbstractVector` and it simply works right off the bat. Here we are constructing an `Array` with a list comprehension: + +````julia:ex4 +onehot = [OneHotVector(3, rand(1:3)) for _ in 1:4] +```` + +Now I define a new function `inner_sum()` that is basically a recursive dot product with a summation. +Here A -- this is something matrix-like (although I did not indicate the types, and you can guess something only by the name), +and `vs` is a vector of some vector-like elements. The function proceeds by taking the dot product of the "matrix" +with all vector-like elements of `vs` and returning the accumulated values. +This is all given generic definition without specifying any types. +Generic programming here consists in this very function call `inner()` in a loop. + +````julia:ex5 +using LinearAlgebra + +function inner_sum(A, vs) + t = zero(eltype(A)) + for v in vs + t += inner(v, A, v) # multiple dispatch! + end + return t +end + +inner(v, A, w) = dot(v, A * w) # very general definition +```` + +So, "look mom, it works": + +````julia:ex6 +A = rand(3, 3) +vs = [rand(3) for _ in 1:4] +inner_sum(A, vs) +```` + +Since `OneHotVector` is a subtype of `AbstractVector`: + +````julia:ex7 +supertype(OneHotVector) +```` + +We can use `inner_sum` and it will do what it is supposed to do: + +````julia:ex8 +inner_sum(A, onehot) +```` + +But this default implementation is **slow**: + +````julia:ex9 +using BenchmarkTools + +@btime inner_sum($A, $onehot); +```` + +We can greatly optimize this procedure. Now let's redefine matrix multiplication by `OneHotVector` +with a simple column selection. We do this by defining a new method of the `*` function (multiplier function) +of `Base` Julia. Additionally we also create a new optimized method of `inner()` for dealing with `OneHotVector`: + +````julia:ex10 +import Base: * + +*(A::AbstractMatrix, v::OneHotVector) = A[:, v.ind] +inner(v::OneHotVector, A, w::OneHotVector) = A[v.ind, w.ind] +```` + +That's it! Simple, huh? Now let's benchmark: + +````julia:ex11 +@btime inner_sum($A, $onehot); +```` + +**Huge gains** of speed! 🚀 + +## Julia: the right approach + +Here are some more thoughts on why I believe Julia is the right approach to scientific computation. + +Below is a very opinionated image that divides the scientific computing languages that we've spoken so far in a 2x2 +diagram with two axes: *Slow-Fast* and *Easy-Hard*. I've put C++ and FORTRAN in the hard and fast quadrant. R and Python goes into +the easy and slow quadrant. Julia is the only language in the easy and fast quadrant. I don't know any language that would want +to be hard and slow, so this quadrant is empty. + +![Scientific Computing Language Comparisons](/pages/images/language_comparisons.svg) + +\center{*Scientific Computing Language Comparisons*} \\ + +What I want to say with this image is that if you want to **code fast and easy** use Julia. + +One other thing to note that I find quite astonishing is that Julia packages are all written in Julia. This does not happen in other scientific +computing languages. For example, the whole `{tidyverse}` ecosystem of R packages are based on C++. `NumPy` and `SciPy` are a mix +of FORTRAN and C. `Scikit-Learn` is also coded in C. + +See the figure below where I compare the GitHub's "Languages" stack bar of +[`PyTorch`](https://github.com/pytorch/pytorch), [`TensorFlow`](https://github.com/tensorflow/tensorflow) and +[`Flux.jl`](https://github.com/FluxML/Flux.jl)(Julia's Deep Learning package). This figure I would call *"Python my a**!"* 😂: + +![Python my ass](/pages/images/ML_code_breakdown.svg) + +\center{*Python my a**!*} \\ + +\note{On the other hand, language *interoperability* is extremely useful: +we want to exploit existing high-quality code in other languages from Julia (and vice versa)! +Julia community have worked hard on this, from the built-in intrinsic Julia `ccall` function (to call C and Fortran libraries) +to [JuliaInterop](https://github.com/JuliaInterop)[^interop] packages that connect Julia to Python, R, Matlab, C++, and more.} + +Another example comes from a Julia podcast that unfortunately I cannot recollect either what podcast was nor who was being interviewed. +While being asked about how he joined the Julia bandwagon, he replied something in the likes: + +> *"Well, I was doing some crazy calculation using a library that was a wrapper to an algorithm +> coded in FORTRAN and I was trying to get help with a bug. I opened an issue and after 2 weeks of no reply +> I've dived into the FORTRAN code (despite having no knowledge of FORTRAN). There I saw a comment from the original author +> describing exactly the same bug that I was experiencing and saying that he would fix this in the future. The comment +> was dated from 1992. At the same time a colleague of mine suggested that I could try to code the algorithm in some +> new language called Julia. +> I thought 'me?! code an algorithm?!'. So, I coded the algorithm in Julia and it was faster than the FORTRAN implementation +> and also without the evil bug. One thing to note that it was really easy to code the algorithm in Julia."* + +Having stuff in different language and wrappers can hinder further research as you can see from this example. + +As you saw from the [Karpinski's talk](https://youtu.be/kc9HwsxE1OY) above, **multiple dispatch empower users to define their +own types (if necessary)** and also allows them to **extend functions and types from other users** to their own special use. +This results in an ecosystem that stimulates code sharing and code reuse in scientific computing that is +unmatched. For instance, if I plug a differential equation from [`DifferentialEquations.jl`](https://diffeq.sciml.ai/) into +a [`Turing.jl`](https://turing.ml/) model I get a Bayesian stochastic differential equation model, *e.g.* **Bayesian SIR model +for infectious disease**. If I plug a [`Flux.jl`](https://fluxml.ai/) neural network into a [`Turing.jl`](https://turing.ml/) +model I get a **Bayesian neural network**! When I saw this type of code sharing I was blown away (and I still am). + +\note{This is the **true power** of a scientific computing language like Julia. It brings so much **power** and **flexibility** to the +user and allows different ways of **sharing**, **contributing**, **extending**, **mixing** and **implementing** code and science. +I hope this short dive into Julia has somehow sent you **towards** Julia.} + +## Footnotes + +[^updatedversion]: please note that I've used updated versions for all languages and packages as of April, 2021. `DataFrames.jl` version 1.0.1, `Pandas` version 1.2.4, `NumPy` version 1.20.2, `{dplyr}` version 1.0.5. We did not cover R's `{data.table}` here. Further benchmarking information is available for example here: [Tabular data benchmarking](https://h2oai.github.io/db-benchmark/) +[^mvnimplem]: which of course I did not. The `Mvn` class is inspired by [Iason Sarantopoulos' implementation](http://blog.sarantop.com/notes/mvn). +[^mathbinormal]: you can find all the math [here](http://www.athenasc.com/Bivariate-Normal.pdf). +[^onehotpost]: the post in Russian, I've "Google Translated" it to English. +[^interop]: Julia has a lot of interoperability between languages. Check out: [`PyCall.jl`](https://github.com/JuliaPy/PyCall.jl) and [`JuliaPy`](https://github.com/JuliaPy) for Python; [`RCall.jl`](https://juliainterop.github.io/RCall.jl/stable/) for Java; [`Cxx.jl`](https://juliainterop.github.io/Cxx.jl/stable/) and [`CxxWrap.jl`](https://github.com/JuliaInterop/CxxWrap.jl) for C++; [`Clang.jl`](https://github.com/JuliaInterop/Clang.jl) for libclang and C; [`ObjectiveC.jl`](https://github.com/JuliaInterop/ObjectiveC.jl) for Objective-C; [`JavaCall.jl`](https://juliainterop.github.io/JavaCall.jl/) for Java; [`MATLAB.jl`](https://github.com/JuliaInterop/MATLAB.jl) for MATLAB; [`MathLink.jl`](https://github.com/JuliaInterop/MathLink.jl) for Mathematica/Wolfram Engine; [`OctCall.jl`](https://github.com/JuliaInterop/OctCall.jl) for GNU Octave; and [`ZMQ.jl`](https://juliainterop.github.io/ZMQ.jl/stable/) for ZeroMQ. + +## References + +Bezanson, J., Edelman, A., Karpinski, S., & Shah, V. B. (2017). Julia: A fresh approach to numerical computing. SIAM Review, 59(1), 65–98. + diff --git a/assets/literate/01_why_Julia_script.jl b/assets/literate/01_why_Julia_script.jl new file mode 100644 index 00000000..abb5361d --- /dev/null +++ b/assets/literate/01_why_Julia_script.jl @@ -0,0 +1,73 @@ +# This file was generated, do not modify it. + +abstract type Pet end +struct Dog <: Pet + name::String +end +struct Cat <: Pet + name::String +end + +function encounter(a::Pet, b::Pet) + verb = meets(a, b) + return println("$(a.name) meets $(b.name) and $verb") +end + +meets(a::Dog, b::Dog) = "sniffs"; +meets(a::Dog, b::Cat) = "chases"; +meets(a::Cat, b::Dog) = "hisses"; +meets(a::Cat, b::Cat) = "slinks"; + +fido = Dog("Fido"); +rex = Dog("Rex"); +whiskers = Cat("Whiskers"); +spots = Cat("Spots"); + +encounter(fido, rex) +encounter(rex, whiskers) +encounter(spots, fido) +encounter(whiskers, spots) + +import Base: size, getindex + +struct OneHotVector <: AbstractVector{Int} + len::Int + ind::Int +end + +size(v::OneHotVector) = (v.len,) + +getindex(v::OneHotVector, i::Integer) = Int(i == v.ind) + +onehot = [OneHotVector(3, rand(1:3)) for _ in 1:4] + +using LinearAlgebra + +function inner_sum(A, vs) + t = zero(eltype(A)) + for v in vs + t += inner(v, A, v) # multiple dispatch! + end + return t +end + +inner(v, A, w) = dot(v, A * w) # very general definition + +A = rand(3, 3) +vs = [rand(3) for _ in 1:4] +inner_sum(A, vs) + +supertype(OneHotVector) + +inner_sum(A, onehot) + +using BenchmarkTools + +@btime inner_sum($A, $onehot); + +import Base: * + +*(A::AbstractMatrix, v::OneHotVector) = A[:, v.ind] +inner(v::OneHotVector, A, w::OneHotVector) = A[v.ind, w.ind] + +@btime inner_sum($A, $onehot); diff --git a/assets/literate/02_bayes_stats.md b/assets/literate/02_bayes_stats.md new file mode 100644 index 00000000..c507f600 --- /dev/null +++ b/assets/literate/02_bayes_stats.md @@ -0,0 +1,633 @@ + +# What is Bayesian Statistics? + +**Bayesian statistics** is an approach to inferential statistics based on Bayes' theorem, where available knowledge about +parameters in a statistical model is updated with the information in observed data. The background knowledge is expressed +as a prior distribution and combined with observational data in the form of a likelihood function to determine the posterior +distribution. The posterior can also be used for making predictions about future events. + +**Bayesian statistics** is a departure from classical inferential statistics that prohibits probability statements about +parameters and is based on asymptotically sampling infinite samples from a theoretical population and finding parameter values +that maximize the likelihood function. Mostly notorious is null-hypothesis significance testing (NHST) based on *p*-values. +Bayesian statistics **incorporate uncertainty** (and prior knowledge) by allowing probability statements about parameters, +and the process of parameter value inference is a direct result of the **Bayes' theorem**. + +Bayesian statistics is **revolutionizing all fields of evidence-based science**[^evidencebased] (van de Schoot et al., 2021). +Dissatisfaction with traditional methods of statistical inference (frequentist statistics) and the advent of computers with exponential +growth in computing power[^computingpower] provided a rise in Bayesian statistics because it is an approach aligned with the human +intuition of uncertainty, robust to scientific malpractices, but computationally intensive. + +But before we get into Bayesian statistics, we have to talk about **probability**: the engine of Bayesian inference. + +## What is Probability? + +> PROBABILITY DOES NOT EXIST! \\ \\ de Finetti (1974)[^deFinetti] + +These are the first words in the preface to the famous book by [Bruno de Finetti](https://en.wikipedia.org/wiki/Bruno_de_Finetti) +(figure below), one of the most important probability mathematician and philosopher. +Yes, probability does not exist. Or rather, probability as a physical quantity, objective chance, **does NOT exist**. +De Finetti showed that, in a precise sense, if we dispense with the question of objective chance *nothing is lost*. +The mathematics of inductive reasoning remains **exactly the same**. + +![De Finetti](/pages/images/finetti.jpg) + +\center{*Bruno de Finetti*} \\ + +Consider tossing a weighted coin. The attempts are considered independent and, as a result, exhibit another +important property: **the order does not matter**. To say that order does not matter is to say that if you take any +finite sequence of heads and tails and exchange the results however you want, +the resulting sequence will have the same probability. We say that this probability is +**invariant under permutations**. + +Or, to put it another way, the only thing that matters is the relative frequency. +Result that have the same frequency of heads and tails consequently have the same probability. +The frequency is considered a **sufficient statistic**. Saying that order doesn't matter or saying +that the only thing that matters is frequency are two ways of saying exactly the same thing. +This property is called **exchangeability** by de Finetti. And it is the most important property of +probability that makes it possible for us to manipulate it mathematically (or philosophically) even +if it does not exist as a physical "thing". + +Still developing the argument: +> "Probabilistic reasoning - always understood as subjective - stems +> merely from our being uncertain about something. It makes no difference whether the uncertainty +> relates to an unforeseeable future [^subjective], or to an unnoticed past, or to a past doubtfully reported +> or forgotten [^objective]... The only relevant thing is uncertainty - the extent of our own knowledge and ignorance. +> The actual fact of whether or not the events considered are in some sense determined, or known by other people, +> and so on, is of no consequence." (de Finetti, 1974) + +In conclusion: no matter what the probability is, you can use it anyway, even if it is an absolute frequency +(ex: probability that I will ride my bike naked is ZERO because the probability that an event that never occurred +will occur in the future it is ZERO) or a subjective guess (ex: maybe the probability is not ZERO, but 0.00000000000001; +very unlikely, but not impossible). + +### Mathematical Definition + +With the philosophical intuition of probability elaborated, we move on to **mathematical intuitions**. +The probability of an event is a real number[^realnumber], $\in \mathbb{R}$ between 0 and 1, where, roughly, +0 indicates the impossibility of the event and 1 indicates the certainty of the event. The greater the likelihood of an event, +the more likely it is that the event will occur. A simple example is the tossing of a fair (impartial) coin. Since the coin is fair, +both results ("heads" and "tails") are equally likely; the probability of "heads" is equal to the probability of "tails"; + and since no other result is possible[^mutually], the probability of "heads" or "tails" is $\frac{1}{2}$ +(which can also be written as 0.5 or 50%). + +Regarding notation, we define $A$ as an event and $P(A)$ as the probability of event $A$, thus: + +$$\{P(A) \in \mathbb{R} : 0 \leq P(A) \leq 1 \}.$$ + +This means the "probability of the event to occur is the set of all real numbers between 0 and 1; including 0 and 1". +In addition, we have three axioms[^axioms], originated from Kolmogorov(1933) (figure below): + +1. **Non-negativity**: For all $A$, $P(A) \geq 0$. Every probability is positive (greater than or equal to zero), regardless of the event. +2. **Additivity**: For two mutually exclusive $A$ and $B$ (cannot occur at the same time[^mutually2]): $P(A) = 1 - P(B)$ and $P(B) = 1 - P(A)$. +3. **Normalization**: The probability of all possible events $A_1, A_2, \dots$ must add up to 1: $\sum_{n \in \mathbb{N}} P(A_n) = 1$. + +![Andrey Nikolaevich Kolmogorov](/pages/images/kolmogorov.jpg) + +\center{*Andrey Nikolaevich Kolmogorov*} \\ + +With these three simple (and intuitive) axioms, we are able to **derive and construct all the mathematics of probability**. + +### Conditional Probability + +An important concept is the **conditional probability** that we can define as the "probability that one event will occur if another +has occurred or not". The notation we use is $ P(A \mid B)$, which reads as "the probability that we have observed $A$ given +we have already observed $B$". + +A good example is the [Texas Hold'em Poker game](https://en.wikipedia.org/wiki/Texas_hold_%27em), +where the player receives two cards and can use five "community cards" to set up +his "hand". The probability that you are dealt a King ($K$) is $\frac{4}{52}$: + +$$ P(K) = \left(\frac{4}{52}\right) = \left(\frac{1}{13}\right) \label{king} . $$ + +And the probability of being dealt an Ace is also the same as \eqref{king}, $\frac{4}{52}$: + +$$ P(A) = \left(\frac{4}{52}\right) = \left(\frac{1}{13}\right) \label{ace} . $$ + +However, the probability that you are dealt a King as a second card since you have been dealt an Ace as a first card is: + +$$ P(K \mid A) = \left(\frac{4}{51}\right) \label{kingace} . $$ + +Since we have one less card ($52 - 1 = 51$) because you have been dealt already an Ace (thus $A$ has been observed), +we have 4 Kings still in the deck, so the \eqref{kingace} is $\frac{4}{51}$. + +### Joint Probability + +Conditional probability leads us to another important concept: joint probability. **Joint probability +is the "probability that two events will both occur"**. Continuing with our Poker example, the probability +that you will receive two cards, Ace ($A$) and a King ($K$) as two starting cards is: + +$$ +\begin{aligned} +P(A,K) &= P(A) \cdot P(K \mid A) \label{aceandking}\\ +&= P \left(\frac{1}{13}\right) \cdot P \left(\frac{4}{51}\right)\\ +&= P \left(\frac{4}{51 \cdot 13}\right) \\ +&\approx 0.006 . +\end{aligned} +$$ + +Note that $P(A,K) = P(K,A)$: + +$$ +\begin{aligned} +P(K,A) &= P(K) \cdot P(A \mid K) \label{kingandace}\\ +&= P \left(\frac{1}{13}\right) \cdot P \left(\frac{4}{51}\right)\\ +&= P \left(\frac{4}{51 \cdot 13}\right) \\ +&\approx 0.006 . +\end{aligned} +$$ + +But this symmetry does not always exist (in fact it very rarely exists). The identity we have is as follows: + +$$ P(A) \cdot P(K \mid A) = P(K) \cdot P(A \mid K) . $$ + +So this symmetry only exists when the baseline rates for conditional events are equal: + +$$ P(A) = P(K). $$ + +Which is what happens in our example. + +#### Conditional Probability is not "commutative" + +$$ P(A \mid B) \neq P(B \mid A) \label{noncommutative} $$ + +Let's see a practical example. For example, I’m feeling good and start coughing in line at the supermarket. +What do you think will happen? Everyone will think I have COVID, which is equivalent to thinking about +$P(\text{cough} \mid \text{covid})$. Seeing the most common symptoms of COVID, **if you have COVID, the chance of +coughing is very high**. But we actually cough a lot more often than we have COVID -- $P(\text{cough}) \neq P(\text{COVID})$, so: + +$$ P(\text{COVID} \mid \text{cough}) \neq P(\text{cough} \mid \text{COVID}) . $$ + +### Bayes' Theorem + +This is the last concept of probability that we need to address before diving into Bayesian statistics, +but it is the most important. Note that it is not a semantic coincidence that Bayesian statistics and Bayes' theorem +have the same prefix. + +[Thomas Bayes](https://en.wikipedia.org/wiki/Thomas_Bayes) (1701 - 1761, figure below) was an English Presbyterian statistician, +philosopher and minister known for formulating a specific case of the theorem that bears his name: Bayes' theorem. +Bayes never published what would become his most famous accomplishment; his notes were edited and published +after his death by his friend Richard Price[^thomaspricelaplace]. In his later years, Bayes was deeply interested in probability. +Some speculate that he was motivated to refute David Hume's argument against belief in miracles based on evidence from the testimony in "An Inquiry Concerning Human Understanding". + +![Thomas Bayes](/pages/images/thomas_bayes.gif) + +\center{*Thomas Bayes*} \\ + +Let's move on to Theorem. Remember that we have the following identity in probability: + +$$ +\begin{aligned} +P(A,B) &= P(B,A) \\ +P(A) \cdot P(B \mid A) &= P(B) \cdot P(A \mid B) \label{jointidentity} . +\end{aligned} +$$ + +Ok, now move $P(B)$ in the right of \eqref{jointidentity} to the left as a division: + +$$ +\begin{aligned} +P(A) \cdot P(B \mid A) &= \overbrace{P(B)}^{\text{this goes to $\leftarrow$}} \cdot P(A \mid B) \\ +&\\ +\frac{P(A) \cdot P(B \mid A)}{P(B)} &= P(A \mid B) \\ +P(A \mid B) &= \frac{P(A) \cdot P(B \mid A)}{P(B)}. +\end{aligned} +$$ + +And the final result is: + +$$ P(A \mid B) = \frac{P(A) \cdot P(B \mid A)}{P(B)}. $$ + +Bayesian statistics uses this theorem as **inference engine** of **parameters** of a model +**conditioned** on **observed data**. + +### Discrete vs Continuous Parameters + +Everything that has been exposed so far is based on the assumption that the parameters are discrete. +This was done in order to provide a better intuition of what is probability. We do not always work with discrete parameters. +The parameters can be continuous, for example: age, height, weight, etc. But don't despair, all the rules and axioms of +probability are also valid for continuous parameters. The only thing we have to do is to exchange all the sums $\sum$ +for integrals $\int$. For example, the third axiom of **Normalization** for continuous random variables becomes: + +$$ \int_{x \in X} p(x) dx = 1 . $$ + +## Bayesian Statistics + +Now that you know what probability is and what Bayes' theorem is, I will propose the following model: + +$$ +\underbrace{P(\theta \mid y)}_{\text{Posterior}} = \frac{\overbrace{P(y \mid \theta)}^{\text{Likelihood}} \cdot \overbrace{P(\theta)}^{\text{Prior}}}{\underbrace{P(y)}_{\text{Normalizing Constant}}} \label{bayesianstats} , +$$ + +where: + +* $\theta$ -- parameter(s) of interest; +* $y$ -- observed data; +* **Prior** -- previous probability of the parameter value(s)[^prior] $\theta$; +* **Likelihood** -- probability of the observed data $y$ conditioned on the parameter value(s) $\theta$; +* **Posterior** -- posterior probability of the parameter value(s) $\theta$ after observing the data $y$; and +* **Normalizing Constant** -- $P(y)$ does not make intuitive sense. This probability is transformed and can be interpreted as something that exists only so that the result of $P(y \mid \theta) P(\theta)$ is somewhere between 0 and 1 -- a valid probability by the axioms. We will talk more about this constant in [5. **Markov Chain Monte Carlo (MCMC)**](/pages/5_MCMC/). + +Bayesian statistics allow us **to directly quantify the uncertainty** related to the value of one or more parameters of our model +conditioned to the observed data. This is the **main feature** of Bayesian statistics, for we are directly estimating +$P (\theta \mid y)$ using Bayes' theorem. The resulting estimate is totally intuitive: it simply quantifies the uncertainty +we have about the value of one or more parameters conditioned on the data, the assumptions of our model (likelihood) and +the previous probability(prior) we have about such values. + +## Frequentist Statistics + +To contrast with Bayesian statistics, let's look at the frequentist statistics, also known as "classical statistics". +And already take notice: **it is not something intuitive** like the Bayesian statistics. + +For frequentist statistics, the researcher is **prohibited from making probabilistic conjectures about parameters**. +Because they are not uncertain, quite the contrary they are determined quantities. The only issue is that we do not +directly observe the parameters, but they are deterministic and do not allow any margin of uncertainty. Therefore, for +the frequentist approach, parameters are unobserved amounts of interest in which we do not make probabilistic conjectures. + +What, then, is uncertain in frequentist statistics? Short answer: **the observed data**. For the frequentist approach, the +sample is uncertain. Thus, we can only make probabilistic conjectures about our sample. Therefore, the uncertainty is expressed +in the probability that I will obtain data similar to those that I obtained if I sampled from a population of interest infinite +samples of the same size as my sample[^warning]. Uncertainty is conditioned by a frequentist approach, in other words, uncertainty +only exists only if I consider an infinite sampling process and extract a frequency from that process. **The probability only +exists if it represents a frequency**. Frequentist statistics is based on an "infinite sampling process of a population that +I have never seen", strange that it may sounds. + +For the frequentist approach, there is no *posterior* or *prior* probability since both involve parameters, and we saw that this +is a no-no on frequentist soil. Everything that is necessary for statistical inference is **contained within likelihood**[^likelihoodsubj]. + +In addition, for reasons of ease of computation, since most of these methods were invented in the first half of the 20th century +(without any help of a computer), only the parameter value(s) that maximize(s) the likelihood function is(are) +computed[^likelihoodopt]. From this optimization process we extracted the **mode** of the likelihood function +(*i.e.* maximum value). The maximum likelihood estimate (MLE) is(are) the parameter value(s) so that a $N$-sized sample +randomly sampled from a population (*i.e.* the data you've collected) is the most likely $N$-sized sample from that population. +All other potential samples that could be extracted from this population will have a worse estimate than the sample you actually +have[^warning2]. In other words, we are conditioning the parameter value(s) on the observed data from the assumption that +we are sampling infinite $N$-sized samples from a theoretical population and treating the parameter values as fixed and +our sample as random (or uncertain). + +The mode works perfectly in the fairytale world, which assumes that everything follows a normal distribution, in which the mode is equal +to the median and to the mean -- $\text{mean} = \text{median} = \text{mode}$. There is only one problem, this assumption is rarely true +(see figure below), especially when we are dealing with multiple parameters with complex relationships between them (complex models). + +![Assumptions vs Reality](/pages/images/assumptions-vs-reality.jpeg) + +\center{*Assumptions vs Reality. Figure by [Katherine Hoffman](https://www.khstats.com/blog/tmle/tutorial/). Authorized Reproduction*} \\ + +A brief sociological and computational explanation is worthwhile of why frequentist (classical) statistics prohibit probabilistic +conjectures about parameters and we are only left with optimizing (finding the maximum value of a function) rather than approximating +or estimating a **complete likelihood density** (in other words, "to pull up the whole file" of the likelihood verisimilitude +instead of just the mode). + +On the sociological side of things, science at the beginning of the 20th century assumed that it should be strictly objective and +all subjectivity must be banned. Therefore, since the estimation of the a posterior probability of parameters involves +elucidating an a prior probability of parameters, such a method should not be allowed in science, as it brings subjectivity +(we know today that nothing in human behavior is purely objective, and subjectivity permeates all human endeavors). + +Regarding the computational side of things, in the 1930s without computers it was much easier to use strong assumptions about +the data to get a value from a statistical estimation using mathematical derivations than to calculate the statistical estimation +by hand without depending on such assumptions. For example: Student's famous $t$ test is a test that indicates when we can +reject that the mean of a certain parameter of interest between two groups is equal (famous null hypothesis - $H_0$). This +test starts from the assumption that if the parameter of interest is distributed according to a normal distribution (assumption 1 +-- normality of the dependent variable), if the variance of the parameter of interest varies homogeneously between groups +(assumption 2 -- homogeneity of the variances), and if the number of observations in the two groups are similar (assumption 3 +-- homogeneity of the size of the groups) the difference between the groups weighted by the variance of the groups follows a +Student-$t$ distribution (hence the name of the test). + +So statistical estimation comes down to calculating the average of two groups, the variance of both groups for a parameter +of interest and looking for the associated $p$-value in a table and see if we can reject the $H_0$. This was valid when +everything we had to do was calculated by hand. Today, with a computer 1 million times more powerful than the Apollo 11 computer +(one that took humanity to the moon) in your pocket [^computingpower], I don't know if it is still valid. + +### $p$-values + +> $p$-values are hard to understand, $p < 0.05$. + +![p-values are hard to understand](/pages/images/meme-pvalue2.jpg) + +Since I've mentioned the $p$ word, let me explain what it is. First, the correct[^booksp] statistics textbook definition: + +> $p$-value is the probability of obtaining test results at least as extreme as the results actually observed, under the assumption that the null hypothesis is correct. + +Unfortunately with frequentist statistics you have to choose one of two qualities for explanations: intuitive or accurate[^gelman]. + +If you write this definition in any test, book or scientific paper, you are 100% accurate and correct in defining what a $p$-value is. +Now, understanding this definition is complicated. For that, let's break this definition down into parts for a better understanding: + +* **"probability of obtaining test results.."**: notice that $p$-values are related your data and not your theory or hypothesis. +* **"...at least as extreme as the results actually observed..."**: "at least as extreme" implies defining a threshold for the characterization of some relevant finding, which is commonly called $\alpha$. We generally stipulate alpha at 5% ($\alpha = 0.05$) and anything more extreme than alpha (ie less than 5%) we characterize as **significant**. +* **"... under the assumption that the null hypothesis is correct."**: every statistical test that has a $p$-value has a null hypothesis (usually written as $H_0$). Null hypotheses, always have to do with some **null effect**. For example, the null hypothesis of the Shapiro-Wilk and Komolgorov-Smirnov test is "the data is distributed according to a Normal distribution" and that of the Levene test is "the group variances are equal". Whenever you see a $p$-value, ask yourself: "What is the null hypothesis that this test assumes is correct?". + +To understand the $p$-value any statistical test first find out what is the null hypothesis behind that test. The +definition of $p$-value will not change. In every test it is always the same. What changes with the test is the null hypothesis. +Each test has its $H_0$. For example, some common statistical tests ($\text{D}^*$ = data at least as extreme as the results actually observed): + +* $t$ Test: $P(\text{D}^* \mid \text{the difference between groups are zero})$ +* ANOVA: $P(\text{D}^* \mid \text{there is no difference between groups})$ +* Regression: $P(\text{D}^* \mid \text{the coefficient is zero})$ +* Shapiro-Wilk: $P(\text{D}^* \mid \text{the sample follows a normal distribution})$ + +$p$-value is the probability of the data you obtained given that the null hypothesis is true. For those who like +mathematical formalism: $p = P(\text{D}^* \mid H_0)$. In English, this expression means "the probability of $\text{D}^*$ conditioned to $H_0$". +Before moving on to some examples and attempts to formalize an intuition about $p$-values, it is important to note that +$p$-values say something about **data** and not **hypotheses**. For $p$-value, **the null hypothesis is true, and we are +only evaluating whether the data conforms to this null hypothesis or not**. If you leave this tutorial armed with this intuition, +the world will be rewarded with researchers better prepared to qualify and interpret evidence ($p <0.05$). + +\note{**Intuitive Example** of a $p$-value: + +Imagine that you have a coin that you suspect is biased towards a higher probability of flipping "heads" than "tails". +(Your null hypothesis is then that the coin is fair.) You flip the coin 100 times and get more heads than tails. +The $p$-value will not tell you if the coin is fair, but it will tell you the probability that you will get at least as +many heads as if the coin were fair. That's it - nothing more.} + +#### $p$-values -- A historical perspective + +There is no way to understand $p$-values ​​if we do not understand its origins and historical trajectory. +The first mention of the term was made by statistician Ronald Fisher[^fisher] (figure below). In 1925, Fisher (Fisher, 1925) defined +the $p$-value as an "index that measures the strength of the evidence against the null hypothesis". To quantify the +strength of the evidence against the null hypothesis, Fisher defended "$p <0.05$ (5% significance) as a standard level +to conclude that there is evidence against the tested hypothesis, although not as an absolute rule". Fisher did not stop +there but rated the strength of the evidence against the null hypothesis. He proposed "if $p$ is between 0.1 and 0.9 there +is certainly no reason to suspect the hypothesis tested. If it is below 0.02 it is strongly indicated that the hypothesis fails +to account for the whole of the facts. We shall not often be astray if we draw a conventional line at 0.05". Since Fisher made +this statement almost 100 years ago, the 0.05 threshold has been used by researchers and scientists worldwide and it has become +ritualistic to use 0.05 as a threshold as if other thresholds could not be used or even considered. + +![Ronald Fisher](/pages/images/fisher.jpg) + +\center{*Ronald Fisher*} \\ + +After that, the threshold of 0.05, now established as unquestionable, strongly influenced statistics and science. But there is +no reason against adopting other thresholds ($\alpha$) like 0.1 or 0.01 (Lakens et al., 2018). If well argued, the choice of +thresholds other than 0.05 can be welcomed by editors, reviewers and advisors. Since the $p$-value is a probability, it is a +continuous quantity. There is no reason to differentiate a $p$ of 0.049 against a $p$ of 0.051. Robert Rosenthal, a psychologist +already said "God loves $p$ 0.06 as much as a $p$ 0.05" (Rosnow & Rosenthal, 1989). + +In the last year of his life, Fisher published an article (Fisher, 1962) examining the possibilities of Bayesian methods, +but with a prior probabilities to be determined experimentally. Some authors even speculate (Jaynes, 2003) that if Fisher +were alive today, he would probably be a "Bayesian". + +### What the $p$-value is not + +![meme-pvalue](/pages/images/meme-pvalue.jpg) + +With the definition and a well-anchored intuition of what $p$-value is, we can move on to what the $p$-value **is not**! + +1. **$p$-value is not the probability of the null hypothesis** - Famous confusion between $P(D \mid H_0)$ and $P(H_0 \mid D)$. $p$-value is not the probability of the null hypothesis, but the probability of the data you obtained. To get the $P(H_0 \mid D)$ you need Bayesian statistics. + +2. **$p$-value is not the probability of the data being produced by chance** - No! Nobody said anything of "being produced by chance". Again: $p$-value is the probability to get results at least as extreme as those that were observed, given that the null hypothesis is true. + +3. **$p$-value measures the size of the effect of a statistical test** - Also wrong... $p$-value does not say anything about the size of the effect. Just about whether the observed data differs from what is expected under the null hypothesis. It is clear that large effects are more likely to be statistically significant than small effects. But this is not a rule and never judge a finding by its $p$-value, but by its effect size. In addition, $p$-values ​​can be "hacked" in a number of ways (Head et al., 2015) and often their value is a direct consequence of the sample size. + +### Confidence Intervals + +To conclude, let's talk about the famous **confidence intervals**, which are not a measure that quantifies the uncertainty of +the value of a parameter (remember probabilistic conjectures about parameters are prohibited in frequentist-land). Wait for it! +Here is the definition of confidence intervals: + +> "An X% confidence interval for a parameter $\theta$ is an interval $(L, U)$ generated +> by a procedure that in repeated sampling has an X% probability of containing the true value of $\theta$, for all possible +> values of $\theta$." +> \\ \\ +> Jerzy Neyman, the "father" of confidence intervals (see figure below) (Neyman, 1937). + +![Jerzy Neyman](/pages/images/neyman.jpeg) + +\center{*Jerzy Neyman*} \\ + +Again the idea of sampling an infinite number of times from a population you have never seen. For example: let's say +that you performed a statistical analysis to compare the effectiveness of a public policy in two groups and you obtained +the difference between the average of those groups. You can express this difference as a confidence interval. We generally +choose 95% confidence (since it is analogous as $p < 0.05$). You then write in your paper that the "observed difference between +groups is 10.5 - 23.5 (95% CI)." +This means that approximately 95 studies out of 100 would compute a confidence interval that contains the true mean difference +–- but it says nothing about which ones those are (whereas the data might). +In other words, 95% is not the probability of obtaining data such that the estimate of the true parameter is contained in the interval that we obtained, +it is the probability of obtaining data such that, if we compute another confidence interval in the same way, it contains the true parameter. +The interval that we got in this particular instance is irrelevant and might as well be thrown away. + +#### Confidence Intervals (Frequentist) vs Credible Intervals (Bayesian) + +Bayesian statistics have a concept similar to the confidence intervals of frequentist statistics. This concept is called +**credibility interval**. And, unlike the confidence interval, its definition is intuitive. **Credibility interval** measures +an interval in which we are sure that the value of the parameter of interest is, based on the likelihood conditioned on the observed +data - $P(y \mid \theta)$; and the prior probability of the parameter of interest - $P(\theta)$. It is basically a "slice" of +the posterior probability of the parameter restricted to a certain level of certainty. For example: a 95% credibility interval +shows the interval that we are 95% sure that captures the value of our parameter of interest. That simple... + +For example, see figure below, which shows a Log-Normal distribution with mean 0 and standard deviation 2. The green dot +shows the maximum likelihood estimation (MLE) of the value of $\theta$ which is simply the mode of distribution. And in +the shaded area we have the 50% credibility interval of the value of $\theta$, which is the interval between the 25% percentile +and the 75% percentile of the probability density of $\theta$. In this example, MLE leads to estimated values that are not +consistent with the actual probability density of the value of $\theta$. + +````julia:ex1 +using CairoMakie +using Distributions + +d = LogNormal(0, 2) +range_d = 0:0.001:4 +q25 = quantile(d, 0.25) +q75 = quantile(d, 0.75) +credint = range(q25; stop=q75, length=100) +f, ax, l = lines( + range_d, + pdf.(d, range_d); + linewidth=3, + axis=(; limits=(-0.2, 4.2, nothing, nothing), xlabel=L"\theta", ylabel="Density"), +) +scatter!(ax, mode(d), pdf(d, mode(d)); color=:green, markersize=12) +band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5)) +save(joinpath(@OUTPUT, "lognormal.svg"), f); # hide +```` + +\fig{lognormal} +\center{_**Log-Normal**: Maximum Likelihood Estimate vs Credible Intervals_} \\ + +Now an example of a multimodal distribution[^multimodal]. The figure below shows a bimodal distribution with two modes +2 and 10[^multimodal2] The green dot shows the maximum likelihood estimation (MLE) of the value of $\theta$ +which is the mode of distribution. See that even with 2 modes, maximum likelihood defaults to the highest mode[^multimodal3]. +And in the shaded area we have the 50% credibility interval of the value of $\theta$, which is the interval between the +25% percentile and the 75% percentile of the probability density of $\theta$. In this example, estimation by +maximum likelihood again lead us to estimated values ​​that are not consistent with the actual probability density +of the value of $\theta$. + +````julia:ex2 +d1 = Normal(10, 1) +d2 = Normal(2, 1) +mix_d = [0.4, 0.6] +d = MixtureModel([d1, d2], mix_d) +range_d = -2:0.01:14 +sim_d = rand(d, 10_000) +q25 = quantile(sim_d, 0.25) +q75 = quantile(sim_d, 0.75) +credint = range(q25; stop=q75, length=100) + +f, ax, l = lines( + range_d, + pdf.(d, range_d); + linewidth=3, + axis=(; + limits=(-2, 14, nothing, nothing), + xticks=[0, 5, 10], + xlabel=L"\theta", + ylabel="Density", + ), +) +scatter!(ax, mode(d2), pdf(d, mode(d2)); color=:green, markersize=12) +band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5)) +save(joinpath(@OUTPUT, "mixture.svg"), f); # hide +```` + +\fig{mixture} +\center{_**Mixture**: Maximum Likelihood Estimate vs Credible Intervals_} \\ + +## Bayesian Statistics vs Frequentist Statistics + +What we've seen so fat can be resumed in the table below: + +| | **Bayesian Statistics** | **Frequentist Statistics** | +|-------------------|---------------------------------------------------|---------------------------------------------------------------------| +| **Data** | Fixed -- Non-random | Uncertain -- Random | +| **Parameters** | Uncertain -- Random | Fixed -- Non-random | +| **Inference** | Uncertainty over parameter values | Uncertainty over a sampling procedure from an infinite population | +| **Probability** | Subjective | Objective (but with strong model assumptions) | +| **Uncertainty** | Credible Interval -- $P(\theta \mid y)$ | Confidence Interval -- $P(y \mid \theta)$ | + +## Advantages of Bayesian Statistics + +Finally, I summarize the main **advantages of Bayesian statistics**: + +* Natural approach to express uncertainty +* Ability to incorporate prior information +* Greater model flexibility +* Complete posterior distribution of parameters + * Confidence Intervals vs Credibility Intervals +* Natural propagation of uncertainty + +And I believe that I also need to show the main **disadvantage**: + +* Slow model estimation speed (30 seconds instead of 3 seconds using the frequentist approach) + +## The beginning of the end of Frequentist Statistics + +\center{*Götterdämmerung*} \\ + +Dear reader, know that you are at a time in history when Statistics is undergoing **major changes**. +I believe that frequentist statistics, especially the way we qualify evidence and hypotheses with $p$-values, +will transform in a "significant" way. Five years ago, the American Statistical Association (ASA, the world's +largest professional statistical organization) published a statement on $p$-values (Wasserstein & Lazar, 2016). +The statement says exactly what we talk about here. The main concepts of the null hypothesis significance test, and in +particular $p$-values, fail to provide what researchers require of them. Despite what many statistical books, teaching +materials and published articles say, $p$-values ​​below 0.05 do not "prove" the reality of anything. Nor, at this point, +do the $p$-values ​​above 0.05 refute anything. ASA's statement has more than 3,600 citations causing significant impact. +As an example, an international symposium was held in 2017 that led to a special open access edition of *The American +Statistician* dedicated to practical ways to abandon $p <0.05$ (Wasserstein, Schirm & Lazar 2019). + +Soon after, more attempts and claims followed. In September 2017, *Nature Human Behavior* published an editorial proposing +that the significance level of the $p$-value be reduced from $0.05$ to $0.005$ (Benjamin et al., 2018). +Several authors, including many highly influential and important statisticians, have argued that this simple step would help +tackle the problem of the science replicability crisis, which many believe is the main consequence of the abusive use of +$p$-values (Ioannidis, 2019). In addition, many have gone a step further and suggest that science discard once and for all +$p$-values (Nature, 2019). Many suggest (myself included) that the main inference tool be Bayesian statistics +(Amrhein, Greenland & McShane, 2019; Goodman, 2016; van de Schoot et al., 2021) + +## Turing + +[Turing](https://turing.ml/) (Ge, Xu & Ghahramani, 2018) is a **probabilistic programming interface written in Julia** +(Bezanson, Edelman, Karpinski & Shah, 2017). It enables **intuitive modeling syntax** with +**flexible composable probabilistic programming inference**. Turing supports a wide range of +**sampling based inference algorithms** by **combining model inference with differentiable +programming interfaces** in Julia. Yes, Julia is that amazing. The same differentiable stuff that you develop +for optimization in neural networks you can plug it in to a probabilistic programming framework and it will work +*without* much effort and boiler-plate code. Most importantly, Turing inference is **composable**: it combines Markov chain +sampling operations on subsets of model variables, e.g. using a combination of a Hamiltonian Monte Carlo (HMC) engine and +a particle Gibbs (PG) engine. This composable inference engine allows the user to **easily switch** between black-box style +inference methods such as HMC, and customized inference methods. + +I believe Turing is the most **important and popular probabilistic language framework in Julia**. It is what PyMC3 and Stan +are for Python and R, but for Julia. Furthermore, you don't have to do "cartwheels" with Theano backends and tensors like +in PyMC3 or learn a new language to declare your models like in Stan (or even have to debug C++ stuff). +Turing is **all** Julia. It uses Julia arrays, Julia distributions, Julia autodiff, Julia plots, Julia random number generator, +Julia MCMC algorithms etc. I think that developing and estimating Bayesian probabilistic models using Julia and Turing is +**powerful**, **intuitive**, **fun**, **expressive** and allows **easily new breakthroughs** simply by being 100% Julia and +embedded in Julia ecosystem. As discussed in [1. **Why Julia?**](/pages/1_why_Julia/), having multiple dispatch with +LLVM's JIT compilation allows us to combine code, types and algorithms in a very powerful and yet simple way. +By using Turing in this context, a researcher (or a curious Joe) can develop new methods and extend the frontiers +of Bayesian inference with new models, samplers, algorithms, or any mix-match of those. + +## Footnotes + +[^evidencebased]: personally, like a good Popperian, I don't believe there is science without being evidence-based; what does not use evidence can be considered as logic, philosophy or social practices (no less or more important than science, just a demarcation of what is science and what is not; eg, mathematics and law). +[^computingpower]: your smartphone (iPhone 12 - 4GB RAM) has 1,000,000x (1 million) more computing power than the computer that was aboard the Apollo 11 (4kB RAM) which took the man to the moon. Detail: this on-board computer was responsible for lunar module navigation, route and controls. +[^deFinetti]: if the reader wants an in-depth discussion see Nau (2001). +[^subjective]: my observation: related to the subjective Bayesian approach. +[^objective]: my observation: related to the objective frequentist approach. +[^realnumber]: a number that can be expressed as a point on a continuous line that originates from minus infinity and ends and plus infinity $(-\infty, +\infty)$; for those who like computing it is a floating point `float` or` double`. +[^mutually]: *i.e.* the events are "mutually exclusive". That means that only $A$ or $B$ can occur in the whole sample space. +[^axioms]: in mathematics, axioms are assumptions assumed to be true that serve as premises or starting points for the elaboration of arguments and theorems. Often the axioms are questionable, for example non-Euclidean geometry refutes Euclid's fifth axiom on parallel lines. So far there is no questioning that has supported the scrutiny of time and science about the three axioms of probability. +[^mutually2]: for example, the result of a given coin is one of two mutually exclusive events: heads or tails. +[^thomaspricelaplace]: the formal name of the theorem is Bayes-Price-Laplace, as Thomas Bayes was the first to discover, Richard Price took his drafts, formalized in mathematical notation and presented to the Royal Society of London, and Pierre Laplace rediscovered the theorem without having had previous contact in the late 18th century in France by using probability for statistical inference with Census data in the Napoleonic era. +[^prior]: I will cover prior probabilities in the content of tutorial [4. **How to use Turing**](/pages/4_Turing/). +[^warning]: I warned you that it was not intuitive... +[^likelihoodsubj]: something worth noting: likelihood also carries **a lot of subjectivity**. +[^likelihoodopt]: for those who have a thing for mathematics (like myself), we calculate at which point of $\theta$ the derivative of the likelihood function is zero - $\mathcal{L}^\prime = 0$. So we are talking really about an optimization problem that for some likelihood functions we can have a closed-form analytical solution. +[^warning2]: have I forgot to warn you that it is not so intuitive? +[^booksp]: there are several statistics textbooks that have wrong definitions of what a $p$-value is. If you don't believe me, see Wasserstein & Lazar (2016). +[^gelman]: this duality is attributed to Andrew Gelman -- Bayesian statistician. +[^fisher]: Ronald Fisher's personality and life controversy deserves a footnote. His contributions were undoubtedly crucial to the advancement of science and statistics. His intellect was brilliant and his talent already flourished young: before turning 33 years old he had proposed the maximum likelihood estimation method (MLE) (Stigler, 2007) and also created the concept of degrees of freedom when proposing a correction in Pearson's chi-square test (Baird, 1983). He also invented the Analysis of Variance (ANOVA) and was the first to propose randomization as a way of carrying out experiments, being considered the "father" of randomized clinical trials (RCTs). Not everything is golden in Fisher's life, he was a eugenicist and had a very strong view on ethnicity and race, advocating the superiority of certain ethnicities. Furthermore, he was extremely invariant, chasing, harming and mocking any critic of his theories and publications. What we see today in the monopoly of the Neyman-Pearson paradigm (Neyman & Pearson, 1933) with $p$-values ​​and null hypotheses the result of this Fisherian effort to silence critics and let only his voice echo. +[^multimodal]: which is not uncommon to see in the real world. +[^multimodal2]: for the curious it is a mixture of two normal distributions both with standard deviation 1, but with different means. To complete it assigns the weights of 60% for the distribution with an average of 2 and 40% for the distribution with an average of 10. +[^multimodal3]: to be more precise, estimation by maximum likelihood in non-convex functions cannot find an analytical solution and, if we are going to use another iterative maximization procedure, there is a risk of it becoming stuck in the second -- lower-valued -- mode of distribution. + +## References + +Amrhein, V., Greenland, S., & McShane, B. (2019). Scientists rise up against statistical significance. *Nature*, 567(7748), 305–307. https://doi.org/10.1038/d41586-019-00857-9 + +Baird, D. (1983). The fisher/pearson chi-squared controversy: A turning point for inductive inference. *The British Journal for the Philosophy of Science*, 34(2), 105–118. + +Benjamin, D. J., Berger, J. O., Johannesson, M., Nosek, B. A., Wagenmakers, E.-J., Berk, R., … Johnson, V. E. (2018). Redefine statistical significance. *Nature Human Behaviour*, 2(1), 6–10. https://doi.org/10.1038/s41562-017-0189-z + +Bezanson, J., Edelman, A., Karpinski, S., & Shah, V. B. (2017). Julia: A fresh approach to numerical computing. SIAM Review, 59(1), 65–98. + +de Finetti, B. (1974). *Theory of Probability*. New York: John Wiley & Sons. + +Eckhardt, R. (1987). Stan Ulam, John von Neumann, and the Monte Carlo Method. *Los Alamos Science*, 15(30), 131–136. + +Fisher, R. A. (1925). *Statistical methods for research workers*. Oliver; Boyd. + +Fisher, R. A. (1962). Some Examples of Bayes’ Method of the Experimental Determination of Probabilities A Priori. *Journal of the Royal Statistical Society. Series B (Methodological)*, 24(1), 118–124. Retrieved from https://www.jstor.org/stable/2983751 + +Ge, H., Xu, K., & Ghahramani, Z. (2018). Turing: A Language for Flexible Probabilistic Inference. International Conference on Artificial Intelligence and Statistics, 1682–1690. http://proceedings.mlr.press/v84/ge18b.html + +Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). *Bayesian Data Analysis*. Chapman and Hall/CRC. + +Goodman, S. N. (2016). Aligning statistical and scientific reasoning. *Science*, 352(6290), 1180–1181. https://doi.org/10.1126/science.aaf5406 + +Head, M. L., Holman, L., Lanfear, R., Kahn, A. T., & Jennions, M. D. (2015). The extent and consequences of p-hacking in science. *PLoS Biol*, 13(3), e1002106. + +Ioannidis, J. P. A. (2019). What Have We (Not) Learnt from Millions of Scientific Papers with P Values? *The American Statistician*, 73(sup1), 20–25. https://doi.org/10.1080/00031305.2018.1447512 + +It’s time to talk about ditching statistical significance. (2019). *Nature*, 567(7748, 7748), 283–283. https://doi.org/10.1038/d41586-019-00874-8 + +Jaynes, E. T. (2003). *Probability theory: The logic of science*. Cambridge university press. + +Kolmogorov, A. N. (1933). *Foundations of the Theory of Probability*. Berlin: Julius Springer. + +Lakens, D., Adolfi, F. G., Albers, C. J., Anvari, F., Apps, M. A. J., Argamon, S. E., … Zwaan, R. A. (2018). Justify your alpha. *Nature Human Behaviour*, 2(3), 168–171. https://doi.org/10.1038/s41562-018-0311-x + +Nau, R. F. (2001). De Finetti was Right: Probability Does Not Exist. *Theory and Decision*, 51(2), 89–124. https://doi.org/10.1023/A:1015525808214 + +Neyman, J. (1937). Outline of a theory of statistical estimation based on the classical theory of probability. *Philosophical Transactions of the Royal Society of London*. Series A, Mathematical and Physical Sciences, 236(767), 333–380. + +Neyman, J., & Pearson, E. S. (1933). On the problem of the most efficient tests of statistical hypotheses. *Philosophical Transactions of the Royal Society of London*. Series A, Containing Papers of a Mathematical or Physical Character, 231(694-706), 289–337. + +Rosnow, R. L., & Rosenthal, R. (1989). Statistical procedures and the justification of knowledge in psychological science. *American Psychologist*, 44, 1276–1284. + +Stigler, S. M. (2007). The epic story of maximum likelihood. *Statistical Science*, 22(4), 598–620. + +van de Schoot, R., Depaoli, S., King, R., Kramer, B., Märtens, K., Tadesse, M. G., … Yau, C. (2021). Bayesian statistics and modelling. *Nature Reviews Methods Primers*, 1(1, 1), 1–26. https://doi.org/10.1038/s43586-020-00001-2 + +Wasserstein, R. L., & Lazar, N. A. (2016). The ASA’s Statement on p-Values: Context, Process, and Purpose. *American Statistician*, 70(2), 129–133. https://doi.org/10.1080/00031305.2016.1154108 + +Wasserstein, R. L., Schirm, A. L., & Lazar, N. A. (2019). Moving to a World Beyond "p < 0.05." *American Statistician*, 73, 1–19. https://doi.org/10.1080/00031305.2019.1583913 + diff --git a/assets/literate/02_bayes_stats_script.jl b/assets/literate/02_bayes_stats_script.jl new file mode 100644 index 00000000..4e9c9bc1 --- /dev/null +++ b/assets/literate/02_bayes_stats_script.jl @@ -0,0 +1,44 @@ +# This file was generated, do not modify it. + +using CairoMakie +using Distributions + +d = LogNormal(0, 2) +range_d = 0:0.001:4 +q25 = quantile(d, 0.25) +q75 = quantile(d, 0.75) +credint = range(q25; stop=q75, length=100) +f, ax, l = lines( + range_d, + pdf.(d, range_d); + linewidth=3, + axis=(; limits=(-0.2, 4.2, nothing, nothing), xlabel=L"\theta", ylabel="Density"), +) +scatter!(ax, mode(d), pdf(d, mode(d)); color=:green, markersize=12) +band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5)) +save(joinpath(@OUTPUT, "lognormal.svg"), f); # hide + +d1 = Normal(10, 1) +d2 = Normal(2, 1) +mix_d = [0.4, 0.6] +d = MixtureModel([d1, d2], mix_d) +range_d = -2:0.01:14 +sim_d = rand(d, 10_000) +q25 = quantile(sim_d, 0.25) +q75 = quantile(sim_d, 0.75) +credint = range(q25; stop=q75, length=100) + +f, ax, l = lines( + range_d, + pdf.(d, range_d); + linewidth=3, + axis=(; + limits=(-2, 14, nothing, nothing), + xticks=[0, 5, 10], + xlabel=L"\theta", + ylabel="Density", + ), +) +scatter!(ax, mode(d2), pdf(d, mode(d2)); color=:green, markersize=12) +band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5)) +save(joinpath(@OUTPUT, "mixture.svg"), f); # hide diff --git a/assets/literate/03_prob_dist.md b/assets/literate/03_prob_dist.md new file mode 100644 index 00000000..27d1dfd0 --- /dev/null +++ b/assets/literate/03_prob_dist.md @@ -0,0 +1,363 @@ + +# Common Probability Distributions + +Bayesian statistics uses probability distributions as the inference "engine" for the estimation of the parameter values +along with their uncertainties. + +Imagine that probability distributions are small pieces of "Lego". We can build whatever we want with these little pieces. +We can make a castle, a house, a city; literally anything we want. The same is true for probabilistic models in Bayesian +statistics. We can build models from the simplest to the most complex using probability distributions and their relationships +to each other. In this tutorial we will give a brief overview of the main probabilistic distributions, their mathematical +notation and their main uses in Bayesian statistics. + +A probability distribution is the mathematical function that gives the probabilities of occurrence +of different possible outcomes for an experiment. It is a mathematical description of a random phenomenon in terms of its +sample space and the probabilities of events (subsets of the sample space). + +We generally use the notation `X ~ Dist (par1, par2, ...)`. Where `X` is the variable,` Dist` is the name of the distribution, +and `par` are the parameters that define how the distribution behaves. Any probabilistic distribution can be "parameterized" +by specifying parameters that allow us to shape some aspects of the distribution for some specific purpose. + +Let's start with discrete distributions and then we'll address the continuous ones. + +## Discrete + +Discrete probability distributions are those where the results are discrete numbers (also called whole numbers): +$\dots, -2, 1, 0, 1, 2, \dots, N$ and $N \in \mathbb{Z}$. In discrete distributions we say the probability that +a distribution takes certain values as "mass". The probability mass function $\text {PMF}$ is the function that +specifies the probability of the random variable $X$ taking the value $x$: + +$$ \text{PMF}(x) = P(X = x) $$ + +### Discrete Uniform + +The discrete uniform distribution is a symmetric probability distribution in which a finite number of values are equally +likely to be observed. Each of the $n$ values has an equal probability $\frac{1}{n}$. Another way of saying +"discrete uniform distribution" would be "a known and finite number of results equally likely to happen". + +The discrete uniform distribution has two parameters and its notation is $\text{Unif} (a, b)$: + +* Lower Bound ($a$) +* Upper Bound ($b$) + +Example: a 6-sided dice. + +````julia:ex1 +using CairoMakie +using Distributions + +f, ax, b = barplot( + DiscreteUniform(1, 6); + axis=(; + title="6-sided Dice", + xlabel=L"\theta", + ylabel="Mass", + xticks=1:6, + limits=(nothing, nothing, 0, 0.3), + ), +) +save(joinpath(@OUTPUT, "discrete_uniform.svg"), f); # hide +```` + +\fig{discrete_uniform} +\center{*Discrete Uniform between 1 and 6*} \\ + +### Bernoulli + +Bernoulli's distribution describes a binary event of a successful experiment. We usually represent $0$ as failure and $1$ +as success, so the result of a Bernoulli distribution is a binary variable $Y \in \{ 0, 1 \}$. + +The Bernoulli distribution is widely used to model discrete binary outcomes in which there are only two possible results. + +Bernoulli's distribution has only a single parameter and its notation is $\text{Bernoulli}(p)$: + +* Success Probability ($p$) + +Example: Whether the patient survived or died or whether the customer completes their purchase or not. + +````julia:ex2 +f, ax1, b = barplot( + Bernoulli(0.5); + width=0.3, + axis=(; + title=L"p=0.5", + xlabel=L"\theta", + ylabel="Mass", + xticks=0:1, + limits=(nothing, nothing, 0, 1), + ), +) +ax2 = Axis( + f[1, 2]; title=L"p=0.2", xlabel=L"\theta", xticks=0:1, limits=(nothing, nothing, 0, 1) +) +barplot!(ax2, Bernoulli(0.2); width=0.3) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "bernoulli.svg"), f); # hide +```` + +\fig{bernoulli} +\center{*Bernoulli with $p = \{ 0.5, 0.2 \}$*} \\ + +### Binomial + +The binomial distribution describes an event of the number of successes in a sequence of $n$ **independent** experiment(s), +each asking a yes-no question with a probability of success $p$. Note that the Bernoulli distribution is a special case +of the binomial distribution where the number of experiments is $1$. + +The binomial distribution has two parameters and its notation is $\text{Bin} (n, p)$ or $ \text{Binomial} (n, p)$: + +* Number of Experiment(s) ($n$) +* Probability of Success ($p$) + +Example: number of heads in 5 coin flips. + +````julia:ex3 +f, ax1, b = barplot( + Binomial(5, 0.5); axis=(; title=L"p=0.5", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"p=0.2", xlabel=L"\theta") +barplot!(ax2, Binomial(5, 0.2)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "binomial.svg"), f); # hide +```` + +\fig{binomial} +\center{*Binomial with $n=5$ and $p = \{ 0.5, 0.2 \}$*} \\ + +### Poisson + +The Poisson distribution expresses the probability that a given number of events will occur in a fixed interval of time +or space if those events occur with a known constant average rate and regardless of the time since the last event. The +Poisson distribution can also be used for the number of events at other specified intervals, such as distance, area or volume. + +The Poisson distribution has one parameter and its notation is $\text{Poisson} (\lambda)$: + +* Rate ($\lambda$) + +Example: Number of emails you receive daily. Number of holes you find on the street. + +````julia:ex4 +f, ax1, b = barplot( + Poisson(1); axis=(; title=L"\lambda=1", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"\lambda=4", xlabel=L"\theta") +barplot!(ax2, Poisson(4)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "poisson.svg"), f); # hide +```` + +\fig{poisson} +\center{*Poisson with $\lambda = \{ 1, 4 \}$*} \\ + +### Negative Binomial + +The negative binomial distribution describes an event of the number of failures before the $k$th success in a sequence of $n$ independent experiment(s), +each asking a yes-no question with probability $p$. +Note that it becomes identical to the Poisson distribution at the limit of $k \to \infty$. +This makes the negative binomial a robust option to replace a Poisson +distribution to model phenomena with a overdispersion* (excess expected variation in data). + +The negative binomial distribution has two parameters and its notation is $\text{NB} (k, p)$ or $\text{Negative-Binomial} (k, p)$: + +* Number of Success(es) ($k$) +* Probability of Success ($p$) + +Any phenomenon that can be modeled with a Poisson distribution, can be modeled with a negative binomial distribution +(Gelman et al., 2013; 2020). + +Example: Annual count of tropical cyclones. + +````julia:ex5 +f, ax1, b = barplot( + NegativeBinomial(1, 0.5); axis=(; title=L"k=1", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"k=2", xlabel=L"\theta") +barplot!(ax2, NegativeBinomial(2, 0.5)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "negbinomial.svg"), f); # hide +```` + +\fig{negbinomial} +\center{*Negative Binomial with $p=0.5$ and $r = \{ 1, 2 \}$*} \\ + +## Continuous + +Continuous probability distributions are those where the results are values in a continuous range (also called real numbers): +$(-\infty, +\infty) \in \mathbb{R}$. In continuous distributions we call the probability that a distribution takes certain +values as "density". As we are talking about real numbers we are not able to obtain the probability that a random variable +$X$ takes the value of $x$. This will always be $0$, as there is no way to specify an exact value of $x$. $x$ lives in the +real numbers line, so we need to specify the probability that $X$ takes values in a **range** $[a,b]$. The probability +density function $\text {PDF}$ is defined as: + +$$ \text{PDF}(x) = P(a \leq X \leq b) = \int_a^b f(x) dx $$ + +### Normal / Gaussian + +This distribution is generally used in the social and natural sciences to represent continuous variables in which its +distributions are not known. This assumption is due to the central limit theorem. The central limit theorem states that, +in some conditions, the average of many samples (observations) of a random variable with finite mean and variance is +itself a random variable whose distribution converges to a normal distribution as the number of samples increases. +Therefore, physical quantities that are expected to be the sum of many independent processes (such as measurement errors) +often have distributions that are expected to be nearly normal. + +The normal distribution has two parameters and its notation is $\text{Normal} (\mu, \sigma^2)$ or $\text{N}(\mu, \sigma^2)$: + +* Mean ($\mu$): distribution mean which is also both the mode and the median of the distribution +* Standard Deviation ($\sigma$): the variance of the distribution ($\sigma^2$) is a measure of the dispersion of the observations in relation to the mean + +Example: Height, Weight, etc. + +````julia:ex6 +f, ax, l = lines( + Normal(0, 1); + label=L"\sigma=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)), +) +lines!(ax, Normal(0, 0.5); label=L"\sigma=0.5", linewidth=5) +lines!(ax, Normal(0, 2); label=L"\sigma=2", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "normal.svg"), f); # hide +```` + +\fig{normal} +\center{*Normal with $\mu=0$ and $\sigma = \{ 1, 0.5, 2 \}$*} \\ + +### Log-normal + +The Log-normal distribution is a continuous probability distribution of a random variable whose logarithm is normally +distributed. Thus, if a random variable $X$ is normally distributed by its natural log, then $Y =\log(X)$ will have +a normal distribution. + +A random variable with logarithmic distribution accepts only positive real values. It is a convenient and useful model +for measurements in the physical sciences and engineering, as well as medicine, economics and other fields, +eg. for energies, concentrations, lengths, financial returns and other values. + +A log-normal process is the statistical realization of the multiplicative product of many independent random variables, +each one being positive. + +The log-normal distribution has two parameters and its notation is $\text{Log-Normal} (\mu, \sigma^2)$: + +* Mean ($\mu$): natural logarithm of the mean the distribution +* Standard Deviation ($\sigma$): natural logarithm of the variance of the distribution ($\sigma^2$) is a measure of the dispersion of the observations in relation to the mean + +````julia:ex7 +f, ax, l = lines( + LogNormal(0, 1); + label=L"\sigma=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 3, nothing, nothing)), +) +lines!(ax, LogNormal(0, 0.25); label=L"\sigma=0.25", linewidth=5) +lines!(ax, LogNormal(0, 0.5); label=L"\sigma=0.5", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "lognormal.svg"), f); # hide +```` + +\fig{lognormal} +\center{*Log-Normal with $\mu=0$ and $\sigma = \{ 1, 0.25, 0.5 \}$*} \\ + +### Exponential + +The exponential distribution is the probability distribution of time between events +that occur continuously and independently at a constant average rate. + +The exponential distribution has one parameter and its notation is $\text{Exp} (\lambda)$: + +* Rate ($\lambda$) + +Example: How long until the next earthquake. How long until the next bus arrives. + +````julia:ex8 +f, ax, l = lines( + Exponential(1); + label=L"\lambda=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 4.5, nothing, nothing)), +) +lines!(ax, Exponential(0.5); label=L"\lambda=0.5", linewidth=5) +lines!(ax, Exponential(1.5); label=L"\lambda=2", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "exponential.svg"), f); # hide +```` + +\fig{exponential} +\center{*Exponential with $\lambda = \{ 1, 0.5, 1.5 \}$*} \\ + +### Student-$t$ distribution + +Student-$t$ distribution appears when estimating the average of a population normally distributed in situations +where the sample size is small and the population standard deviation is unknown. + +If we take a sample of $n$ observations from a normal distribution, then the distribution +Student-$t$ with $\nu = n-1$ degrees of freedom can be defined as the distribution of the location of the +sample mean relative to the true mean, divided by the standard deviation of the sample, after multiplying by +the standardizing term $\sqrt{n}$. + +The Student-$t$ distribution is symmetrical and bell-shaped, like the normal distribution, but has longer tails, +which means that it is more likely to produce values ​​that are far from its mean. + +The Student-$t$ distribution has one parameter and its notation is $\text{Student-$t$} (\nu)$: + +* Degrees of Freedom ($\nu$): controls how much it resembles a normal distribution + +Example: A database full of outliers. + +````julia:ex9 +f, ax, l = lines( + TDist(2); + label=L"\nu=2", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)), +) +lines!(ax, TDist(8); label=L"\nu=8", linewidth=5) +lines!(ax, TDist(30); label=L"\nu=30", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "tdist.svg"), f); # hide +```` + +\fig{tdist} +\center{*Student-$t$ with $\nu = \{ 2, 8, 30 \}$*} \\ + +### Beta Distribution + +The beta distributions is a natural choice to model anything that is constrained to take values between 0 and 1. +So it is a good candidate for probabilities and proportions. + +The beta distribution has two parameters and its notation is $\text{Beta} (a, b)$: + +* Shape parameter ($a$ or sometimes $\alpha$): controls how much the shape is shifted towards 1 +* Shape parameter ($b$ or sometimes $\beta$): controls how much the shape is shifted towards 0 + +Example: A basketball player has made already scored 5 free throws while missing 3 in a total of 8 attempts +-- $\text{Beta}(3, 5)$. + +````julia:ex10 +f, ax, l = lines( + Beta(1, 1); + label=L"a=b=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 1, nothing, nothing)), +) +lines!(ax, Beta(3, 2); label=L"a=3, b=2", linewidth=5) +lines!(ax, Beta(2, 3); label=L"a=2, b=3", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "beta.svg"), f); # hide +```` + +\fig{beta} +\center{*Beta with different values of $a$ and $b$*} \\ + +## Distribution Zoo + +I did not cover all existing distributions. There is a whole plethora of probabilistic distributions. + +To access the entire "distribution zoo" use this tool from [Ben Lambert](https://ben-lambert.com/bayesian/) +(statistician from *Imperial College of London*): + +## References + +Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian Data Analysis. Chapman and Hall/CRC. + +Gelman, A., Hill, J., & Vehtari, A. (2020). Regression and other stories. Cambridge University Press. + diff --git a/assets/literate/03_prob_dist_script.jl b/assets/literate/03_prob_dist_script.jl new file mode 100644 index 00000000..562c7aea --- /dev/null +++ b/assets/literate/03_prob_dist_script.jl @@ -0,0 +1,113 @@ +# This file was generated, do not modify it. + +using CairoMakie +using Distributions + +f, ax, b = barplot( + DiscreteUniform(1, 6); + axis=(; + title="6-sided Dice", + xlabel=L"\theta", + ylabel="Mass", + xticks=1:6, + limits=(nothing, nothing, 0, 0.3), + ), +) +save(joinpath(@OUTPUT, "discrete_uniform.svg"), f); # hide + +f, ax1, b = barplot( + Bernoulli(0.5); + width=0.3, + axis=(; + title=L"p=0.5", + xlabel=L"\theta", + ylabel="Mass", + xticks=0:1, + limits=(nothing, nothing, 0, 1), + ), +) +ax2 = Axis( + f[1, 2]; title=L"p=0.2", xlabel=L"\theta", xticks=0:1, limits=(nothing, nothing, 0, 1) +) +barplot!(ax2, Bernoulli(0.2); width=0.3) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "bernoulli.svg"), f); # hide + +f, ax1, b = barplot( + Binomial(5, 0.5); axis=(; title=L"p=0.5", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"p=0.2", xlabel=L"\theta") +barplot!(ax2, Binomial(5, 0.2)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "binomial.svg"), f); # hide + +f, ax1, b = barplot( + Poisson(1); axis=(; title=L"\lambda=1", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"\lambda=4", xlabel=L"\theta") +barplot!(ax2, Poisson(4)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "poisson.svg"), f); # hide + +f, ax1, b = barplot( + NegativeBinomial(1, 0.5); axis=(; title=L"k=1", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"k=2", xlabel=L"\theta") +barplot!(ax2, NegativeBinomial(2, 0.5)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "negbinomial.svg"), f); # hide + +f, ax, l = lines( + Normal(0, 1); + label=L"\sigma=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)), +) +lines!(ax, Normal(0, 0.5); label=L"\sigma=0.5", linewidth=5) +lines!(ax, Normal(0, 2); label=L"\sigma=2", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "normal.svg"), f); # hide + +f, ax, l = lines( + LogNormal(0, 1); + label=L"\sigma=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 3, nothing, nothing)), +) +lines!(ax, LogNormal(0, 0.25); label=L"\sigma=0.25", linewidth=5) +lines!(ax, LogNormal(0, 0.5); label=L"\sigma=0.5", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "lognormal.svg"), f); # hide + +f, ax, l = lines( + Exponential(1); + label=L"\lambda=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 4.5, nothing, nothing)), +) +lines!(ax, Exponential(0.5); label=L"\lambda=0.5", linewidth=5) +lines!(ax, Exponential(1.5); label=L"\lambda=2", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "exponential.svg"), f); # hide + +f, ax, l = lines( + TDist(2); + label=L"\nu=2", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)), +) +lines!(ax, TDist(8); label=L"\nu=8", linewidth=5) +lines!(ax, TDist(30); label=L"\nu=30", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "tdist.svg"), f); # hide + +f, ax, l = lines( + Beta(1, 1); + label=L"a=b=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 1, nothing, nothing)), +) +lines!(ax, Beta(3, 2); label=L"a=3, b=2", linewidth=5) +lines!(ax, Beta(2, 3); label=L"a=2, b=3", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "beta.svg"), f); # hide diff --git a/assets/literate/04_Turing.md b/assets/literate/04_Turing.md new file mode 100644 index 00000000..628b471b --- /dev/null +++ b/assets/literate/04_Turing.md @@ -0,0 +1,350 @@ + +# How to use Turing + +[**Turing**](http://turing.ml/) is an ecosystem of Julia packages for Bayesian Inference using +[probabilistic programming](https://en.wikipedia.org/wiki/Probabilistic_programming). +Turing provides an easy and intuitive way of specifying models. + +## Probabilistic Programming + +What is **probabilistic programming** (PP)? It is a **programming paradigm** in which probabilistic models +are specified and inference for these models is performed **automatically** (Hardesty, 2015). In more clear terms, +PP and PP Languages (PPLs) allows us to specify **variables as random variables** (like Normal, Binomial etc.) with +**known or unknown parameters**. Then, we **construct a model** using these variables by specifying how the variables + related to each other, and finally **automatic inference of the variables' unknown parameters** is then performed. + +In a Bayesian approach this means specifying **priors**, **likelihoods** and letting the PPL compute the **posterior**. +Since the denominator in the posterior is often intractable, we use Markov Chain Monte Carlo[^MCMC] and some fancy +algorithm that uses the posterior geometry to guide the MCMC proposal using Hamiltonian dynamics called +Hamiltonian Monte Carlo (HMC) to approximate the posterior. This involves, besides a suitable PPL, automatic differentiation, +MCMC chains interface, and also an efficient HMC algorithm implementation. In order to provide all of these features, +Turing has a whole ecosystem to address each and every one of these components. + +## Turing's Ecosystem + +Before we dive into how to specify models in Turing, let's discuss Turing's **ecosystem**. +We have several Julia packages under Turing's GitHub organization [TuringLang](https://github.com/TuringLang), +but I will focus on 6 of those: + +* [`Turing.jl`](https://github.com/TuringLang/Turing.jl) +* [`MCMCChains.jl`](https://github.com/TuringLang/MCMCChains.jl) +* [`DynamicPPL.jl`](https://github.com/TuringLang/DynamicPPL.jl) +* [`AdvancedHMC.jl`](https://github.com/TuringLang/AdvancedHMC.jl) +* [`DistributionsAD.jl`](https://github.com/TuringLang/DistributionsAD.jl) +* [`Bijectors.jl`](https://github.com/TuringLang/Bijectors.jl) + +The first one is [`Turing.jl`](https://github.com/TuringLang/Turing.jl) (Ge, Xu & Ghahramani, 2018) +itself, the main package that we use to +**interface with all the Turing ecosystem** of packages and the backbone of the PPL Turing. + +The second, [`MCMCChains.jl`](https://github.com/TuringLang/MCMCChains.jl), is an interface to **summarizing MCMC +simulations** and has several utility functions for **diagnostics** and **visualizations**. + +The third package is [`DynamicPPL.jl`](https://github.com/TuringLang/DynamicPPL.jl) (Tarek, Xu, Trapp, Ge & Ghahramani, 2020) +which specifies a domain-specific language and backend for Turing (which itself is a PPL). The main feature of `DynamicPPL.jl` +is that is is entirely written in Julia and also it is modular. + +[`AdvancedHMC.jl`](https://github.com/TuringLang/AdvancedHMC.jl) (Xu, Ge, Tebbutt, Tarek, Trapp & Ghahramani, 2020) provides a robust, +modular and efficient implementation +of advanced HMC algorithms. The state-of-the-art HMC algorithm is the **N**o-**U**-**T**urn **S**ampling +(NUTS)[^MCMC] (Hoffman & Gelman, 2011) which is available in `AdvancedHMC.jl`. + +The fourth package, [`DistributionsAD.jl`](https://github.com/TuringLang/DistributionsAD.jl) defines the necessary functions to enable +automatic differentiation (AD) of the `logpdf` function from [`Distributions.jl`](https://github.com/JuliaStats/Distributions.jl) +using the packages [`Tracker.jl`](https://github.com/FluxML/Tracker.jl), [`Zygote.jl`](https://github.com/FluxML/Zygote.jl), +[`ForwardDiff.jl`](https://github.com/JuliaDiff/ForwardDiff.jl) and [`ReverseDiff.jl`](https://github.com/JuliaDiff/ReverseDiff.jl). +The main goal of `DistributionsAD.jl` is to make the output of `logpdf` differentiable with respect to all continuous parameters +of a distribution as well as the random variable in the case of continuous distributions. This is the package that guarantees the +"automatic inference" part of the definition of a PPL. + +Finally, [`Bijectors.jl`](https://github.com/TuringLang/Bijectors.jl) implements a set of functions for transforming constrained +random variables (e.g. simplexes, intervals) to Euclidean space. Note that `Bijectors.jl` is still a work-in-progress and +in the future we'll have better implementation for more constraints, *e.g.* positive ordered vectors of random variables. + +Most of the time we will not be dealing with these packages directly, since `Turing.jl` will take care of the interfacing +for us. So let's talk about `Turing.jl`. + +## `Turing.jl` + +`Turing.jl` is the main package in the Turing ecosystem and the backbone that glues all the other packages together. +Turing's "workflow" begin with a model specification. We specify the model inside a macro `@model` where we can assign variables +in two ways: + +* using `~`: which means that a variable follows some probability distribution (Normal, Binomial etc.) and its value is random under that distribution +* using `=`: which means that a variable does not follow a probability distribution and its value is deterministic (like the normal `=` assignment in programming languages) + +Turing will perform automatic inference on all variables that you specify using `~`. +Here is a simple example of how we would model a six-sided dice. Note that a "fair" dice will be distributed as a discrete uniform +probability with the lower bound as 1 and the upper bound as 6: + +$$ X \sim \text{Uniform}(1,6) \label{uniformdice} $$ + +Note that the expectation of a random variable $X \sim \text{Uniform}(a,b)$ is: + +$$ E(X) = \frac{a+b}{2} = \frac{7}{2} = 3.5 \label{expectationdice} $$ + +Graphically this means: + +````julia:ex1 +using CairoMakie +using Distributions + +dice = DiscreteUniform(1, 6) +f, ax, b = barplot( + dice; + label="six-sided Dice", + axis=(; xlabel=L"\theta", ylabel="Mass", xticks=1:6, limits=(nothing, nothing, 0, 0.3)), +) +vlines!(ax, [mean(dice)]; linewidth=5, color=:red, label=L"E(\theta)") +axislegend(ax) +save(joinpath(@OUTPUT, "dice.svg"), f); # hide +```` + +\fig{dice} +\center{*A "fair" six-sided Dice: Discrete Uniform between 1 and 6*} \\ + +So let's specify our first Turing model. It will be named `dice_throw` and will have a single parameter `y` +which is a $N$-dimensional vector of integers representing the observed data, *i.e.* the outcomes of $N$ six-sided dice throws: + +````julia:ex2 +using Turing +setprogress!(false) # hide + +@model function dice_throw(y) + #Our prior belief about the probability of each result in a six-sided dice. + #p is a vector of length 6 each with probability p that sums up to 1. + p ~ Dirichlet(6, 1) + + #Each outcome of the six-sided dice has a probability p. + for i in eachindex(y) + y[i] ~ Categorical(p) + end +end; +```` + +Here we are using the [Dirichlet distribution](https://en.wikipedia.org/wiki/Dirichlet_distribution) which +is the multivariate generalization of the [Beta distribution](https://en.wikipedia.org/wiki/Beta_distribution). +The Dirichlet distribution is often used as the conjugate prior for Categorical or Multinomial distributions. Our dice +is modelled as a [Categorical distribution](https://en.wikipedia.org/wiki/Categorical_distribution) +with six possible results $y \in \{ 1, 2, 3, 4, 5, 6 \}$ with some probability vector +$\mathbf{p} = (p_1, \dots, p_6)$. Since all mutually exclusive outcomes must sum up to 1 to be a valid probability, we impose the constraint that +all $p$s must sum up to 1 -- $\sum^n_{i=1} p_i = 1$. We could have used a vector of six Beta random variables but it would be hard and +inefficient to enforce this constraint. Instead, I've opted for a Dirichlet with a weekly informative prior towards a +"fair" dice which is encoded as a `Dirichlet(6,1)`. This is translated as a 6-dimensional vector of elements that sum to one: + +````julia:ex3 +mean(Dirichlet(6, 1)) +```` + +And, indeed, it sums up to one: + +````julia:ex4 +sum(mean(Dirichlet(6, 1))) +```` + +Also, since the outcome of a [Categorical distribution](https://en.wikipedia.org/wiki/Categorical_distribution) is an integer +and `y` is a $N$-dimensional vector of integers we need to apply some sort of broadcasting here. +We could use the familiar dot `.` broadcasting operator in Julia: +`y .~ Categorical(p)` to signal that all elements of `y` are distributed as a Categorical distribution. +But doing that does not allow us to do predictive checks (more on this below). So, instead we use a `for`-loop. + +### Simulating Data + +Now let's set a seed for the pseudo-random number generator and simulate 1,000 throws of a six-sided dice: + +````julia:ex5 +using Random + +Random.seed!(123); + +my_data = rand(DiscreteUniform(1, 6), 1_000); +```` + +The vector `my_data` is a 1,000-length vector of `Int`s ranging from 1 to 6, just like how a regular six-sided dice outcome would be: + +````julia:ex6 +first(my_data, 5) +```` + +Once the model is specified we instantiate the model with the single parameter `y` as the simulated `my_data`: + +````julia:ex7 +model = dice_throw(my_data); +```` + +Next, we call Turing's `sample()` function that takes a Turing model as a first argument, along with a +sampler as the second argument, and the third argument is the number of iterations. Here, I will use the `NUTS()` sampler from +`AdvancedHMC.jl` and 1,000 iterations. Please note that, as default, Turing samplers will discard the first thousand (1,000) iterations as warmup. +So the sampler will output 1,000 samples starting from sample 1,001 until sample 2,000: + +````julia:ex8 +chain = sample(model, NUTS(), 1_000); +```` + +Now let's inspect the chain. We can do that with the function `describe()` that will return a 2-element vector of +`ChainDataFrame` (this is the type defined by `MCMCChains.jl` to store Markov chain's information regarding the inferred +parameters). The first `ChainDataFrame` has information regarding the parameters' summary statistics (`mean`, `std`, `r_hat`, ...) +and the second is the parameters' quantiles. Since `describe(chain)` returns a 2-element vector, I will assign the output to two variables: + +````julia:ex9 +summaries, quantiles = describe(chain); +```` + +We won't be focusing on quantiles, so let's put it aside for now. Let's then take a look at the parameters' summary statistics: + +````julia:ex10 +summaries +```` + +Here `p` is a 6-dimensional vector of probabilities, which each one associated with a mutually exclusive outcome of a six-sided +dice throw. As we expected, the probabilities are almost equal to $\frac{1}{6}$, like a "fair" six-sided dice that we simulated +data from (sampling from `DiscreteUniform(1, 6)`). Indeed, just for a sanity check, the mean of the estimates of `p` sums up to 1: + +````julia:ex11 +sum(summaries[:, :mean]) +```` + +In the future if you have some crazy huge models and you just want a **subset** of parameters from your chains? +Just do `group(chain, :parameter)` or index with `chain[:, 1:6, :]`: + +````julia:ex12 +summarystats(chain[:, 1:3, :]) +```` + +or `chain[[:parameters,...]]`: + +````julia:ex13 +summarystats(chain[[:var"p[1]", :var"p[2]"]]) +```` + +And, finally let's compute the expectation of the estimated six-sided dice, $E(\tilde{X})$, using the standard expectation +definition of expectation for a discrete random variable: + +$$ E(X) = \sum_{x \in X} x \cdot P(x) $$ + +````julia:ex14 +sum([idx * i for (i, idx) in enumerate(summaries[:, :mean])]) +```` + +Bingo! The estimated expectation is very *close* to the theoretical expectation of $\frac{7}{2} = 3.5$, as we've show +in \eqref{expectationdice}. + +### Visualizations + +Note that the type of our `chain` is a `Chains` object from `MCMCChains.jl`: + +````julia:ex15 +typeof(chain) +```` + +Since `Chains` is a [`Tables.jl`-compatible](https://github.com/JuliaData/Tables.jl/blob/main/INTEGRATIONS.md) data structure, +we can use all of the plotting capabilities from [`AlgebraOfGraphics.jl`](https://aog.makie.org/stable/). + +````julia:ex16 +using AlgebraOfGraphics +using AlgebraOfGraphics: density +#exclude additional information such as log probability +params = names(chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt1 = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines) +plt2 = data(chain) * chain_mapping * density() +f = Figure(; resolution=(800, 600)) +draw!(f[1, 1], plt1) +draw!(f[1, 2], plt2; axis=(; ylabel="density")) +save(joinpath(@OUTPUT, "chain.svg"), f); # hide +```` + +\fig{chain} +\center{*Visualization of a MCMC Chain simulation*} \\ + +On the figure above we can see, for each parameter in the model, on the left the +parameter's traceplot and on the right the parameter's density[^visualization]. + +## Prior and Posterior Predictive Checks + +Predictive checks are a great way to **validate a model**. +The idea is to **generate data from the model** using **parameters from draws from the prior or posterior**. +**Prior predictive check** is when we simulate data using model parameter values drawn from the **prior** distribution, +and **posterior predictive check** is is when we simulate data using model parameter values drawn from the **posterior** +distribution. + +The workflow we do when specifying and sampling Bayesian models is not linear or acyclic (Gelman et al., 2020). This means +that we need to iterate several times between the different stages in order to find a model that captures +best the data generating process with the desired assumptions. The figure below demonstrates the workflow [^workflow]. + +![Bayesian Workflow](/pages/images/bayesian_workflow.png) + +\center{*Bayesian Workflow. Adapted from Gelman et al. (2020)*} \\ + +This is quite easy in Turing. Our six-sided dice model already has a **posterior distribution** which is the object `chain`. +We need to create a **prior distribution** for our model. To accomplish this, instead of supplying a MCMC sampler like +`NUTS()`, we supply the "sampler" `Prior()` inside Turing's `sample()` function: + +````julia:ex17 +prior_chain = sample(model, Prior(), 2_000); +```` + +Now we can perform predictive checks using both the prior (`prior_chain`) or posterior (`chain`) distributions. +To draw from the prior and posterior predictive distributions we instantiate a "predictive model", *i.e.* a Turing model but with the observations set to `missing`, and then calling `predict()` on the predictive model and the previously drawn samples. +First let's do the *prior* predictive check: + +````julia:ex18 +missing_data = similar(my_data, Missing) # vector of `missing` +model_missing = dice_throw(missing_data) # instantiate the "predictive model +prior_check = predict(model_missing, prior_chain); +```` + +Here we are creating a `missing_data` object which is a `Vector` of the same length as the `my_data` and populated with type `missing` as values. +We then instantiate a new `dice_throw` model with the `missing_data` vector as the `y` argument. +Finally, we call `predict()` on the predictive model and the previously drawn samples, which in our case are the samples from the prior distribution (`prior_chain`). + +Note that `predict()` returns a `Chains` object from `MCMCChains.jl`: + +````julia:ex19 +typeof(prior_check) +```` + +And we can call `summarystats()`: + +````julia:ex20 +summarystats(prior_check[:, 1:5, :]) # just the first 5 prior samples +```` + +We can do the same with `chain` for a *posterior* predictive check: + +````julia:ex21 +posterior_check = predict(model_missing, chain); +summarystats(posterior_check[:, 1:5, :]) # just the first 5 posterior samples +```` + +## Conclusion + +This is the basic overview of Turing usage. I hope that I could show you how simple and intuitive is to +specify probabilistic models using Turing. First, specify a **model** with the macro `@model`, then **sample from it** by +specifying the **data**, **sampler** and **number of interactions**. All **probabilistic parameters** (the ones that you've specified +using `~`) will be **inferred** with a full **posterior density**. Finally, you inspect the **parameters' statistics** like +**mean** and **standard deviation**, along with **convergence diagnostics** like `r_hat`. Conveniently, you can **plot** stuff +easily if you want to. You can also do **predictive checks** using either the **posterior** or **prior** model's distributions. + +## Footnotes + +[^MCMC]: see [5. **Markov Chain Monte Carlo (MCMC)**](/pages/5_MCMC/). +[^visualization]: we'll cover those plots and diagnostics in [5. **Markov Chain Monte Carlo (MCMC)**](/pages/5_MCMC/). +[^workflow]: note that this workflow is a extremely simplified adaptation from the original workflow on which it was based. I suggest the reader to consult the original workflow of Gelman et al. (2020). + +## References + +Ge, H., Xu, K., & Ghahramani, Z. (2018). Turing: A Language for Flexible Probabilistic Inference. International Conference on Artificial Intelligence and Statistics, 1682–1690. http://proceedings.mlr.press/v84/ge18b.html + +Gelman, A., Vehtari, A., Simpson, D., Margossian, C. C., Carpenter, B., Yao, Y., … Modr’ak, M. (2020, November 3). Bayesian Workflow. Retrieved February 4, 2021, from http://arxiv.org/abs/2011.01808 + +Hardesty (2015). "Probabilistic programming does in 50 lines of code what used to take thousands". phys.org. April 13, 2015. Retrieved April 13, 2015. https://phys.org/news/2015-04-probabilistic-lines-code-thousands.html + +Hoffman, M. D., & Gelman, A. (2011). The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo. Journal of Machine Learning Research, 15(1), 1593–1623. Retrieved from http://arxiv.org/abs/1111.4246 + +Tarek, M., Xu, K., Trapp, M., Ge, H., & Ghahramani, Z. (2020). DynamicPPL: Stan-like Speed for Dynamic Probabilistic Models. ArXiv:2002.02702 [Cs, Stat]. http://arxiv.org/abs/2002.02702 + +Xu, K., Ge, H., Tebbutt, W., Tarek, M., Trapp, M., & Ghahramani, Z. (2020). AdvancedHMC.jl: A robust, modular and efficient implementation of advanced HMC algorithms. Symposium on Advances in Approximate Bayesian Inference, 1–10. http://proceedings.mlr.press/v118/xu20a.html + diff --git a/assets/literate/04_Turing_script.jl b/assets/literate/04_Turing_script.jl new file mode 100644 index 00000000..8a7ad3c2 --- /dev/null +++ b/assets/literate/04_Turing_script.jl @@ -0,0 +1,85 @@ +# This file was generated, do not modify it. + +using CairoMakie +using Distributions + +dice = DiscreteUniform(1, 6) +f, ax, b = barplot( + dice; + label="six-sided Dice", + axis=(; xlabel=L"\theta", ylabel="Mass", xticks=1:6, limits=(nothing, nothing, 0, 0.3)), +) +vlines!(ax, [mean(dice)]; linewidth=5, color=:red, label=L"E(\theta)") +axislegend(ax) +save(joinpath(@OUTPUT, "dice.svg"), f); # hide + +using Turing +setprogress!(false) # hide + +@model function dice_throw(y) + #Our prior belief about the probability of each result in a six-sided dice. + #p is a vector of length 6 each with probability p that sums up to 1. + p ~ Dirichlet(6, 1) + + #Each outcome of the six-sided dice has a probability p. + for i in eachindex(y) + y[i] ~ Categorical(p) + end +end; + +mean(Dirichlet(6, 1)) + +sum(mean(Dirichlet(6, 1))) + +using Random + +Random.seed!(123); + +my_data = rand(DiscreteUniform(1, 6), 1_000); + +first(my_data, 5) + +model = dice_throw(my_data); + +chain = sample(model, NUTS(), 1_000); + +summaries, quantiles = describe(chain); + +summaries + +sum(summaries[:, :mean]) + +summarystats(chain[:, 1:3, :]) + +summarystats(chain[[:var"p[1]", :var"p[2]"]]) + +sum([idx * i for (i, idx) in enumerate(summaries[:, :mean])]) + +typeof(chain) + +using AlgebraOfGraphics +using AlgebraOfGraphics: density +#exclude additional information such as log probability +params = names(chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt1 = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines) +plt2 = data(chain) * chain_mapping * density() +f = Figure(; resolution=(800, 600)) +draw!(f[1, 1], plt1) +draw!(f[1, 2], plt2; axis=(; ylabel="density")) +save(joinpath(@OUTPUT, "chain.svg"), f); # hide + +prior_chain = sample(model, Prior(), 2_000); + +missing_data = similar(my_data, Missing) # vector of `missing` +model_missing = dice_throw(missing_data) # instantiate the "predictive model +prior_check = predict(model_missing, prior_chain); + +typeof(prior_check) + +summarystats(prior_check[:, 1:5, :]) # just the first 5 prior samples + +posterior_check = predict(model_missing, chain); +summarystats(posterior_check[:, 1:5, :]) # just the first 5 posterior samples diff --git a/assets/literate/05_MCMC.md b/assets/literate/05_MCMC.md new file mode 100644 index 00000000..d773e708 --- /dev/null +++ b/assets/literate/05_MCMC.md @@ -0,0 +1,1497 @@ + +# Markov Chain Monte Carlo (MCMC) + +The main computational barrier for Bayesian statistics is the denominator $P(\text{data})$ of the Bayes formula: + +$$ P(\theta \mid \text{data})=\frac{P(\theta) \cdot P(\text{data} \mid \theta)}{P(\text{data})} \label{bayes} $$ + +In discrete cases we can turn the denominator into a sum of all parameters using the chain rule of probability: + +$$ P(A,B \mid C)=P(A \mid B,C) \times P(B \mid C) \label{chainrule} $$ + +This is also called marginalization: + +$$ P(\text{data})=\sum_{\theta} P(\text{data} \mid \theta) \times P(\theta) \label{discretemarginalization} $$ + +However, in the continuous cases the denominator $P(\text{data})$ becomes a very large and complicated integral to calculate: + +$$ P(\text{data})=\int_{\theta} P(\text{data} \mid \theta) \times P(\theta)d \theta \label{continuousmarginalization} $$ + +In many cases this integral becomes *intractable* (incalculable) and therefore we must find other ways to calculate +the posterior probability $P(\theta \mid \text{data})$ in \eqref{bayes} without using the denominator $P(\text{data})$. + +## What is the denominator $P(\text{data})$ for? + +Quick answer: to normalize the posterior in order to make it a valid probability distribution. This means that the sum of all probabilities +of the possible events in the probability distribution must be equal to 1: + +- in the case of discrete probability distribution: $\sum_{\theta} P(\theta \mid \text{data}) = 1$ +- in the case of continuous probability distribution: $\int_{\theta} P(\theta \mid \text{data})d \theta = 1$ + +## What if we remove this denominator? + +When we remove the denominator $(\text{data})$ we have that the posterior $P(\theta \mid \text{data})$ is **proportional** to the +prior multiplied by the likelihood $P(\theta) \cdot P(\text{data} \mid \theta)$[^propto]. + +$$ P(\theta \mid \text{data}) \propto P(\theta) \cdot P(\text{data} \mid \theta) \label{proptobayes} $$ + +This [YouTube video](https://youtu.be/8FbqSVFzmoY) explains the denominator problem very well, see below: + +~~~ +
+~~~ + +## Markov Chain Monte Carlo (MCMC) + +This is where Markov Chain Monte Carlo comes in. MCMC is a broad class of computational tools for approximating integrals and generating samples from +a posterior probability (Brooks, Gelman, Jones & Meng, 2011). MCMC is used when it is not possible to sample $\theta$ directly from the subsequent +probabilistic distribution $P(\theta \mid \text{data})$. Instead, we sample in an iterative manner such that at each step of the process we expect the +distribution from which we sample $P^* (\theta^* \mid \text{data})$ (here $*$ means simulated) becomes increasingly similar to the posterior +$P(\theta \mid \text{data})$. All of this is to eliminate the (often impossible) calculation of the denominator $P(\text{data})$. + +The idea is to define an ergodic Markov chain (that is to say that there is a single stationary distribution) of which the set of possible states +is the sample space and the stationary distribution is the distribution to be approximated (or sampled). Let $X_0, X_1, \dots, X_n$ be a +simulation of the chain. The Markov chain converges to the stationary distribution of any initial state $X_0$ after a large enough number +of iterations $r$, the distribution of the state $X_r$ will be similar to the stationary distribution, so we can use it as a sample. +Markov chains have a property that the probability distribution of the next state depends only on the current state and not on the +sequence of events that preceded: $P(X_{n+1}=x \mid X_{0},X_{1},X_{2},\ldots ,X_{n}) = P(X_{n+1}=x \mid X_{n})$. This property is +called Markovian, after the mathematician [Andrey Markov](https://en.wikipedia.org/wiki/Andrey_Markov) (see figure below). +Similarly, repeating this argument with $X_r$ as the starting point, we can use $X_{2r}$ as a sample, and so on. +We can then use the state sequence $X_r, X_{2r}, X_{3r}, \dots$ as almost independent samples of the stationary distribution +of the Markov chain. + +![Andrey Markov](/pages/images/andrey_markov.jpg) + +\center{*Andrey Markov*} \\ + +The effectiveness of this approach depends on: + +1. how big $r$ must be to ensure a suitably good sample; and + +2. computational power required for each iteration of the Markov chain. + +In addition, it is customary to discard the first iterations of the algorithm as they are usually not representative +of the distribution to be approximated. In the initial iterations of MCMC algorithms, generally the Markov current is +in a *warm-up* process[^warmup] and its state is far from ideal to start a reliable sampling. It is generally recommended +to discard half of the iterations (Gelman, Carlin, Stern, Dunson, Vehtari, & Rubin, 2013a). For example: +if the Markov chain has 4,000 iterations, we discard the first 2,000 as warm-up. + +### Monte Carlo Method + +Stanislaw Ulam (figure below), who participated in the Manhattan project and when trying to calculate the neutron diffusion +process for the hydrogen bomb ended up creating a class of methods called **_Monte Carlo_**. + +![Stanislaw Ulam](/pages/images/stanislaw.jpg) + +\center{*Stanislaw Ulam*} \\ + +Monte Carlo methods have the underlying concept of using randomness to solve problems that can be deterministic in principle. +They are often used in physical and mathematical problems and are most useful when it is difficult or impossible to use other approaches. +Monte Carlo methods are used mainly in three classes of problems: optimization, numerical integration and generating sample from a +probability distribution. + +The idea for the method came to Ulam while playing solitaire during his recovery from surgery, as he thought about playing hundreds +of games to statistically estimate the probability of a successful outcome. As he himself mentions in Eckhardt (1987): + +> "The first thoughts and attempts I made to practice [the Monte Carlo method] were suggested by a question which occurred to me +> in 1946 as I was convalescing from an illness and playing solitaires. The question was what are the chances that a Canfield solitaire +> laid out with 52 cards will come out successfully? After spending a lot of time trying to estimate them by pure combinatorial +> calculations, I wondered whether a more practical method than "abstract thinking" might not be to lay it out say one hundred times and +> simply observe and count the number of successful plays. This was already possible to envisage with the beginning of the new era of +> fast computers, and I immediately thought of problems of neutron diffusion and other questions of mathematical physics, and more +> generally how to change processes described by certain differential equations into an equivalent form interpretable as a succession +> of random operations. Later... [in 1946, I ] described the idea to John von Neumann and we began to plan actual calculations." + +Because it was secret, von Neumann and Ulam's work required a codename. A colleague of von Neumann and Ulam, Nicholas Metropolis +(figure below), suggested using the name "Monte Carlo", which refers to Casino Monte Carlo in Monaco, where Ulam's uncle +(Michał Ulam) borrowed money from relatives to gamble. + +![Nicholas Metropolis](/pages/images/nicholas_metropolis.png) + +\center{*Nicholas Metropolis*} \\ + +The [applications of the Monte Carlo method](https://en.wikipedia.org/wiki/Monte_Carlo_method#Applications) are numerous: +physical sciences, engineering, climate change, computational biology, computer graphics, applied statistics, artificial intelligence, +search and rescue, finance and business and law. In the scope of these tutorials we will focus on applied statistics and specifically +in the context of Bayesian inference: providing a random sample of the posterior distribution. + +### Simulations + +I will do some simulations to illustrate MCMC algorithms and techniques. So, here's the initial setup: + +````julia:ex1 +using CairoMakie +using Distributions +using Random + +Random.seed!(123); +```` + +Let's start with a toy problem of a multivariate normal distribution of $X$ and $Y$, where + +$$ +\begin{bmatrix} +X \\ +Y +\end{bmatrix} \sim \text{Multivariate Normal} \left( +\begin{bmatrix} +\mu_X \\ +\mu_Y +\end{bmatrix}, \mathbf{\Sigma} +\right) \\ +\mathbf{\Sigma} \sim +\begin{pmatrix} +\sigma^2_{X} & \sigma_{X}\sigma_{Y} \rho \\ +\sigma_{X}\sigma_{Y} \rho & \sigma^2_{Y} +\end{pmatrix} +\label{mvnormal} +$$ + +If we assign $\mu_X = \mu_Y = 0$ and $\sigma_X = \sigma_Y = 1$ (mean 0 and standard deviation 1 +for both $X$ and $Y$), we have the following formulation from \eqref{mvnormal}: + +$$ +\begin{bmatrix} +X \\ +Y +\end{bmatrix} \sim \text{Multivariate Normal} \left( +\begin{bmatrix} +0 \\ +0 +\end{bmatrix}, \mathbf{\Sigma} +\right), \\ +\mathbf{\Sigma} \sim +\begin{pmatrix} +1 & \rho \\ +\rho & 1 +\end{pmatrix} +\label{stdmvnormal} +$$ + +All that remains is to assign a value for $\rho$ in \eqref{stdmvnormal} for the correlation between $X$ and $Y$. +For our example we will use correlation of 0.8 ($\rho = 0.8$): + +$$ +\mathbf{\Sigma} \sim +\begin{pmatrix} +1 & 0.8 \\ +0.8 & 1 +\end{pmatrix} +\label{Sigma} +$$ + +````julia:ex2 +const N = 100_000 +const μ = [0, 0] +const Σ = [1 0.8; 0.8 1] + +const mvnormal = MvNormal(μ, Σ) +```` + +In the figure below it is possible to see a contour plot of the PDF of a multivariate normal distribution composed of two normal +variables $X$ and $Y$, both with mean 0 and standard deviation 1. +The correlation between $X$ and $Y$ is $\rho = 0.8$: + +````julia:ex3 +x = -3:0.01:3 +y = -3:0.01:3 +dens_mvnormal = [pdf(mvnormal, [i, j]) for i in x, j in y] +f, ax, c = contourf(x, y, dens_mvnormal; axis=(; xlabel=L"X", ylabel=L"Y")) +Colorbar(f[1, 2], c) +save(joinpath(@OUTPUT, "countour_mvnormal.svg"), f); # hide +```` + +\fig{countour_mvnormal} +\center{*Countour Plot of the PDF of a Multivariate Normal Distribution*} \\ + +Also a surface plot can be seen below for you to get a 3-D intuition of what is going on: + +````julia:ex4 +f, ax, s = surface( + x, + y, + dens_mvnormal; + axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 8), +) +save(joinpath(@OUTPUT, "surface_mvnormal.svg"), f); # hide +```` + +\fig{surface_mvnormal} +\center{*Surface Plot of the PDF of a Multivariate Normal Distribution*} \\ + +### Metropolis and Metropolis-Hastings + +The first MCMC algorithm widely used to generate samples from Markov chain originated in physics in the 1950s +(in a very close relationship with the atomic bomb at the Manhattan project) and is called **Metropolis** +(Metropolis, Rosenbluth, Rosenbluth, Teller, & Teller, 1953) in honor of the first author [Nicholas Metropolis](https://en.wikipedia.org/wiki/Nicholas_Metropolis) +(figure above). In summary, the Metropolis algorithm is an adaptation of a random walk with +an acceptance/rejection rule to converge to the target distribution. + +The Metropolis algorithm uses a **proposal distribution** $J_t(\theta^*)$ ($J$ stands for *jumping distribution* +and $t$ indicates which state of the Markov chain we are in) to define next values of the distribution +$P^*(\theta^* \mid \text{data})$. This distribution must be symmetrical: + +$$ J_t (\theta^* \mid \theta^{t-1}) = J_t(\theta^{t-1} \mid \theta^*) \label{symjump} $$ + +In the 1970s, a generalization of the Metropolis algorithm emerged that **does not** require that the proposal distributions +be symmetric. The generalization was proposed by [Wilfred Keith Hastings](https://en.wikipedia.org/wiki/W._K._Hastings) +(Hastings, 1970) (figure below) and is called **Metropolis-Hastings algorithm**. + +![Wilfred Hastings](/pages/images/hastings.jpg) + +\center{*Wilfred Hastings*} \\ + +#### Metropolis Algorithm + +The essence of the algorithm is a random walk through the parameters' sample space, where the probability of the Markov chain +changing state is defined as: + +$$ P_{\text{change}} = \min \left( {\frac{P (\theta_{\text{proposed}})}{P (\theta_{\text{current}})}}, 1 \right) \label{proposal} $$ + +This means that the Markov chain will only change to a new state under two conditions: + +1. When the probability of the parameters proposed by the random walk $P(\theta_{\text{proposed}})$ is **greater** than the probability of the parameters of the current state $P(\theta_{\text{current}})$, we change with 100% probability. Note that if $P(\theta_{\text{proposed}}) > P(\theta_{\text{current}})$ then the function $\min$ chooses the value 1 which means 100%. +2. When the probability of the parameters proposed by the random walk $P(\theta_{\text{proposed}})$ is **less** than the probability of the parameters of the current state $P(\theta_{\text{current}})$, we changed with a probability equal to the proportion of that difference. Note that if $P(\theta_{\text{proposed}}) < P(\theta_{\text{current}})$ then the function $\min$ **does not** choose the value 1, but the value $\frac{P(\theta_{\text{proposed}})}{P(\theta_{\text{current}})}$ which equates the proportion of the probability of the proposed parameters to the probability of the parameters at the current state. + +Anyway, at each iteration of the Metropolis algorithm, even if the Markov chain changes state or not, we sample the parameter +$\theta$ anyway. That is, if the chain does not change to a new state, $\theta$ will be sampled twice (or +more if the current is stationary in the same state). + +The Metropolis-Hastings algorithm can be described in the following way [^metropolis] ($\theta$ is the parameter, or set of +parameters, of interest and $y$ is the data): + +1. Define a starting point $\theta^0$ of which $p(\theta^0 \mid y) > 0$, or sample it from an initial distribution $p_0(\theta)$. $p_0(\theta)$ can be a normal distribution or a prior distribution of $\theta$ ($p(\theta)$). + +2. For $t = 1, 2, \dots$: + + - Sample a proposed $\theta^*$ from a proposal distribution in time $t$, $J_t (\theta^* \mid \theta^{t-1})$. + + - Calculate the ratio of probabilities: + + - **Metropolis**: $r = \frac{p(\theta^* \mid y)}{p(\theta^{t-1} \mid y)}$ + - **Metropolis-Hastings**: $r = \frac{\frac{p(\theta^* \mid y)}{J_t(\theta^* \mid \theta^{t-1})}}{\frac{p(\theta^{t-1} \mid y)}{J_t(\theta^{t-1} \mid \theta^*)}}$ + + - Assign: + + $ + \theta^t = + \begin{cases} + \theta^* & \text{with probability } \min (r, 1) \\ + \theta^{t-1} & \text{otherwise} + \end{cases} + $ + +#### Limitations of the Metropolis Algorithm + +The limitations of the Metropolis-Hastings algorithm are mainly computational. With randomly generated proposals, +it usually takes a large number of iterations to enter areas of higher (more likely) posterior densities. Even +efficient Metropolis-Hastings algorithms sometimes accept less than 25% of the proposals (Roberts, Gelman & Gilks, 1997). +In lower-dimensional situations, the increased computational power can compensate for the lower efficiency to some extent. +But in higher-dimensional and more complex modeling situations, bigger and faster computers alone are rarely +enough to overcome the challenge. + +#### Metropolis -- Implementation + +In our toy example we will assume that $J_t (\theta^* \mid \theta^{t-1})$ is symmetric, thus +$J_t(\theta^* \mid \theta^{t-1}) = J_t (\theta^{t-1} \mid \theta^*)$, so I'll just implement +the Metropolis algorithm (not the Metropolis-Hastings algorithm). + +Below I created a Metropolis sampler for our toy example. At the end it prints the acceptance rate of +the proposals. Here I am using the same proposal distribution for both $X$ and $Y$: a uniform distribution +parameterized with a `width` parameter: + +$$ +X \sim \text{Uniform} \left( X - \frac{\text{width}}{2}, X + \frac{\text{width}}{2} \right) \\ +Y \sim \text{Uniform} \left( Y - \frac{\text{width}}{2}, Y + \frac{\text{width}}{2} \right) +$$ + +I will use the already known `Distributions.jl` `MvNormal` from the plots above along with the `logpdf()` +function to calculate the PDF of the proposed and current $\theta$s. It is easier to work with +probability logs than with the absolute values[^numerical]. Mathematically we will compute: + +$$ +\begin{aligned} +r &= \frac{ +\operatorname{PDF}\left( +\text{Multivariate Normal} \left( +\begin{bmatrix} +x_{\text{proposed}} \\ +y_{\text{proposed}} +\end{bmatrix} +\right) +\Bigg| +\text{Multivariate Normal} \left( +\begin{bmatrix} +\mu_X \\ +\mu_Y +\end{bmatrix}, \mathbf{\Sigma} +\right) +\right)} +{ +\operatorname{PDF}\left( +\text{Multivariate Normal} \left( +\begin{bmatrix} +x_{\text{current}} \\ +y_{\text{current}} +\end{bmatrix} +\right) +\Bigg| +\text{Multivariate Normal} \left( +\begin{bmatrix} +\mu_X \\ +\mu_Y +\end{bmatrix}, \mathbf{\Sigma} +\right) +\right)}\\ +&=\frac{\operatorname{PDF}_{\text{proposed}}}{\operatorname{PDF}_{\text{current}}}\\ +&= \exp\Big( +\log\left(\operatorname{PDF}_{\text{proposed}}\right) +- +\log\left(\operatorname{PDF}_{\text{current}}\right) +\Big) +\end{aligned} +$$ + +Here is a simple implementation in Julia: + +````julia:ex5 +function metropolis( + S::Int64, + width::Float64, + ρ::Float64; + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + accepted = 0::Int64 + x = start_x + y = start_y + @inbounds draws[1, :] = [x y] + for s in 2:S + x_ = rand(rgn, Uniform(x - width, x + width)) + y_ = rand(rgn, Uniform(y - width, y + width)) + r = exp(logpdf(binormal, [x_, y_]) - logpdf(binormal, [x, y])) + + if r > rand(rgn, Uniform()) + x = x_ + y = y_ + accepted += 1 + end + @inbounds draws[s, :] = [x y] + end + println("Acceptance rate is: $(accepted / S)") + return draws +end +```` + +Now let's run our `metropolis()` algorithm for the bivariate normal case with +`S = 10_000`, `width = 2.75` and `ρ = 0.8`: + +````julia:ex6 +const S = 10_000 +const width = 2.75 +const ρ = 0.8 + +X_met = metropolis(S, width, ρ); +```` + +Take a quick peek into `X_met`, we'll see it's a matrix of $X$ and $Y$ values as columns and the time $t$ as rows: + +````julia:ex7 +X_met[1:10, :] +```` + +Also note that the acceptance of the proposals was 21%, the expected for Metropolis algorithms (around 20-25%) +(Roberts et. al, 1997). + +We can construct `Chains` object using `MCMCChains.jl`[^mcmcchains] by passing a matrix along with the parameters names as +symbols inside the `Chains()` constructor: + +````julia:ex8 +using MCMCChains + +chain_met = Chains(X_met, [:X, :Y]); +```` + +Then we can get summary statistics regarding our Markov chain derived from the Metropolis algorithm: + +````julia:ex9 +summarystats(chain_met) +```` + +Both of `X` and `Y` have mean close to 0 and standard deviation close to 1 (which +are the theoretical values). +Take notice of the `ess` (effective sample size - ESS) is approximate 1,000. +So let's calculate the efficiency of our Metropolis algorithm by dividing +the ESS by the number of sampling iterations that we've performed: + +$$ \text{efficiency} = \frac{\text{ESS}}{\text{iterations}} \label{ESS} $$ + +````julia:ex10 +mean(summarystats(chain_met)[:, :ess_tail]) / S +```` + +Our Metropolis algorithm has around 10.2% efficiency. Which, in my honest opinion, *sucks*...(😂) + +##### Metropolis -- Visual Intuition + +I believe that a good visual intuition, even if you have not understood any mathematical formula, is the key for you to start a +fruitful learning journey. So I made some animations! + +The animation in figure below shows the first 100 simulations of the Metropolis algorithm used to generate `X_met`. +Note that in several iterations the proposal is rejected and the algorithm samples the parameters $\theta_1$ and $\theta_2$ +from the previous state (which becomes the current one, since the proposal is refused). The blue ellipsis represents +the 90% HPD of our toy example's bivariate normal distribution. + +Note: `HPD` stands for *Highest Probability Density* (which in our case the posterior's 90% probability range). + +````julia:ex11 +using LinearAlgebra: eigvals, eigvecs +#source: https://discourse.julialang.org/t/plot-ellipse-in-makie/82814/4 +function getellipsepoints(cx, cy, rx, ry, θ) + t = range(0, 2 * pi; length=100) + ellipse_x_r = @. rx * cos(t) + ellipse_y_r = @. ry * sin(t) + R = [cos(θ) sin(θ); -sin(θ) cos(θ)] + r_ellipse = [ellipse_x_r ellipse_y_r] * R + x = @. cx + r_ellipse[:, 1] + y = @. cy + r_ellipse[:, 2] + return (x, y) +end +function getellipsepoints(μ, Σ; confidence=0.95) + quant = sqrt(quantile(Chisq(2), confidence)) + cx = μ[1] + cy = μ[2] + + egvs = eigvals(Σ) + if egvs[1] > egvs[2] + idxmax = 1 + largestegv = egvs[1] + smallesttegv = egvs[2] + else + idxmax = 2 + largestegv = egvs[2] + smallesttegv = egvs[1] + end + + rx = quant * sqrt(largestegv) + ry = quant * sqrt(smallesttegv) + + eigvecmax = eigvecs(Σ)[:, idxmax] + θ = atan(eigvecmax[2] / eigvecmax[1]) + if θ < 0 + θ += 2 * π + end + + return getellipsepoints(cx, cy, rx, ry, θ) +end + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "met_anim.gif"); framerate=5) do frame + for i in 1:100 + scatter!(ax, (X_met[i, 1], X_met[i, 2]); color=(:red, 0.5)) + linesegments!(X_met[i:(i + 1), 1], X_met[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; +```` + +\fig{met_anim} +\center{*Animation of the First 100 Samples Generated from the Metropolis Algorithm*} \\ + +Now let's take a look how the first 1,000 simulations were, excluding 1,000 initial iterations as warm-up. + +````julia:ex12 +const warmup = 1_000 + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_met[warmup:(warmup + 1_000), 1], + X_met[warmup:(warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "met_first1000.svg"), f); # hide +```` + +\fig{met_first1000} +\center{*First 1,000 Samples Generated from the Metropolis Algorithm after warm-up*} \\ + +And, finally, lets take a look in the all 9,000 samples generated after the warm-up of 1,000 iterations. + +````julia:ex13 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_met[warmup:end, 1], X_met[warmup:end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "met_all.svg"), f); # hide +```` + +\fig{met_all} +\center{*All 9,000 Samples Generated from the Metropolis Algorithm after warm-up*} \\ + +### Gibbs + +To circumvent the problem of low acceptance rate of the Metropolis (and Metropolis-Hastings) algorithm, +the Gibbs algorithm was developed, which does not have an acceptance/rejection rule for new proposals +to change the state of the Markov chain. **All proposals are accepted**. + +Gibbs' algorithm had an original idea conceived by physicist Josiah Willard Gibbs (figure below), +in reference to an analogy between a sampling algorithm and statistical physics (a branch of physics which +is based on statistical mechanics). The algorithm was described by brothers Stuart and Donald Geman +in 1984 (Geman & Geman, 1984), about eight decades after Gibbs's death. + +![Josiah Gibbs](/pages/images/josiah_gibbs.jpg) + +\center{*Josiah Gibbs*} \\ + +The Gibbs algorithm is very useful in multidimensional sample spaces (in which there are more than 2 parameters to be +sampled for the posterior probability). It is also known as **alternating conditional sampling**, since we always sample +a parameter **conditioned** to the probability of the other parameters. + +The Gibbs algorithm can be seen as a **special case** of the Metropolis-Hastings algorithm because all proposals are +accepted (Gelman, 1992). + +#### Gibbs Algorithm + +The essence of Gibbs' algorithm is an iterative sampling of parameters conditioned to other parameters +$P(\theta_1 \mid \theta_2, \dots \theta_n)$. + +Gibbs's algorithm can be described in the following way[^gibbs] ($\theta$ is the parameter, or set of +parameters, of interest and $y$ is the data): + +1. Define $P(\theta_1), P(\theta_2), \dots, P(\theta_n)$: the prior probability of each of the $\theta_n$ parameters. + +2. Sample a starting point $\theta^0_1, \theta^0_2, \dots, \theta^0_n$. We usually sample from a normal distribution or from a distribution specified as the prior distribution of $\theta_n$. + +3. For $t = 1, 2, \dots$: + + $\begin{aligned} + \theta^t_1 &\sim p(\theta_1 \mid \theta^0_2, \dots, \theta^0_n) \\ + \theta^t_2 &\sim p(\theta_2 \mid \theta^{t-1}_1, \dots, \theta^0_n) \\ + &\vdots \\ + \theta^t_n &\sim p(\theta_n \mid \theta^{t-1}_1, \dots, \theta^{t-1}_{n-1}) + \end{aligned}$ + +#### Limitations of the Gibbs Algorithm + +The main limitation of the Gibbs algorithm is with regard to alternative conditional sampling. + +If we compare with the Metropolis algorithm (and consequently Metropolis-Hastings) we have random proposals +from a proposal distribution in which we sample each parameter unconditionally to other parameters. In order +for the proposals to take us to the posterior probability's correct locations to sample, we have an +acceptance/rejection rule for these proposals, otherwise the samples of the Metropolis algorithm would not +approach the target distribution of interest. The state changes of the Markov chain are then carried out +multidimensionally [^gibbs2]. As you saw in the Metropolis' figures, in a 2-D space (as is our example's bivariate +normal distribution), when there is a change of state in the Markov chain, the new proposal location considers +both $\theta_1$ and $\theta_2$, causing **diagonal** movement in space 2-D sample. In other words, +the proposal is done regarding all dimensions of the parameter space. + +In the case of the Gibbs algorithm, in our toy example, this movement occurs only in a single parameter, +*i.e* single dimension, as we sample sequentially and conditionally to other parameters. This causes **horizontal** +movements (in the case of $\theta_1$) and **vertical movements** (in the case of $\theta_2$), but never +diagonal movements like the ones we saw in the Metropolis algorithm. + +#### Gibbs -- Implementation + +Here are some new things compared to the Metropolis algorithm implementation. First to conditionally +sample the parameters $P(\theta_1 \mid \theta_2)$ and $P(\theta_2 \mid \theta_1)$, we need to create +two new variables `β` and `λ`. These variables represent the correlation between $X$ and $Y$ ($\theta_1$ +and $\theta_2$ respectively) scaled by the ratio of the variance of $X$ and $Y$. +And then we use these variables in the sampling of $\theta_1$ and $\theta_2$: + +$$ +\begin{aligned} +\beta &= \rho \cdot \frac{\sigma_Y}{\sigma_X} = \rho \\ +\lambda &= \rho \cdot \frac{\sigma_X}{\sigma_Y} = \rho \\ +\sigma_{YX} &= 1 - \rho^2\\ +\sigma_{XY} &= 1 - \rho^2\\ +\theta_1 &\sim \text{Normal} \bigg( \mu_X + \lambda \cdot (y^* - \mu_Y), \sigma_{XY} \bigg) \\ +\theta_2 &\sim \text{Normal} \bigg( \mu_y + \beta \cdot (x^* - \mu_X), \sigma_{YX} \bigg) +\end{aligned} +$$ + +````julia:ex14 +function gibbs( + S::Int64, + ρ::Float64; + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + x = start_x + y = start_y + β = ρ * σ_y / σ_x + λ = ρ * σ_x / σ_y + sqrt1mrho2 = sqrt(1 - ρ^2) + σ_YX = σ_y * sqrt1mrho2 + σ_XY = σ_x * sqrt1mrho2 + @inbounds draws[1, :] = [x y] + for s in 2:S + if s % 2 == 0 + y = rand(rgn, Normal(μ_y + β * (x - μ_x), σ_YX)) + else + x = rand(rgn, Normal(μ_x + λ * (y - μ_y), σ_XY)) + end + @inbounds draws[s, :] = [x y] + end + return draws +end +```` + +Generally a Gibbs sampler is not implemented in this way. Here I coded the Gibbs algorithm so that it samples a parameter for each iteration. +To be more computationally efficient we would sample all parameters are on each iteration. I did it on purpose because I want +to show in the animations the real trajectory of the Gibbs sampler in the sample space (vertical and horizontal, not diagonal). +So to remedy this I will provide `gibbs()` double the amount of `S` (20,000 in total). Also take notice that we are now proposing +new parameters' values conditioned on other parameters, so there is not an acceptance/rejection rule here. + +````julia:ex15 +X_gibbs = gibbs(S * 2, ρ); +```` + +As before lets' take a quick peek into `X_gibbs`, we'll see it's a matrix of $X$ and $Y$ values as columns and the time $t$ as rows: + +````julia:ex16 +X_gibbs[1:10, :] +```` + +Again, we construct a `Chains` object by passing a matrix along with the parameters names as +symbols inside the `Chains()` constructor: + +````julia:ex17 +chain_gibbs = Chains(X_gibbs, [:X, :Y]); +```` + +Then we can get summary statistics regarding our Markov chain derived from the Gibbs algorithm: + +````julia:ex18 +summarystats(chain_gibbs) +```` + +Both of `X` and `Y` have mean close to 0 and standard deviation close to 1 (which +are the theoretical values). +Take notice of the `ess` (effective sample size - ESS) that is around 2,100. +Since we used `S * 2` as the number of samples, in order for we to compare with Metropolis, +we would need to divide the ESS by 2. So our ESS is between 1,000, which is similar +to Metropolis' ESS. +Now let's calculate the efficiency of our Gibbs algorithm by dividing +the ESS by the number of sampling iterations that we've performed also +accounting for the `S * 2`: + +````julia:ex19 +(mean(summarystats(chain_gibbs)[:, :ess_tail]) / 2) / S +```` + +Our Gibbs algorithm has around 10.6% efficiency. Which, in my honest opinion, despite the +small improvement still *sucks*...(😂) + +##### Gibbs -- Visual Intuition + +Oh yes, we have animations for Gibbs also! + +The animation in figure below shows the first 100 simulations of the Gibbs algorithm used to generate `X_gibbs`. +Note that all proposals are accepted now, so the at each iteration we sample new parameters values. +The blue ellipsis represents the 90% HPD of our toy example's bivariate normal distribution. + +````julia:ex20 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "gibbs_anim.gif"); framerate=5) do frame + for i in 1:200 + scatter!(ax, (X_gibbs[i, 1], X_gibbs[i, 2]); color=(:red, 0.5)) + linesegments!(X_gibbs[i:(i + 1), 1], X_gibbs[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; +```` + +\fig{gibbs_anim} +\center{*Animation of the First 100 Samples Generated from the Gibbs Algorithm*} \\ + +Now let's take a look how the first 1,000 simulations were, excluding 1,000 initial iterations as warm-up. + +````julia:ex21 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_gibbs[(2 * warmup):(2 * warmup + 1_000), 1], + X_gibbs[(2 * warmup):(2 * warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "gibbs_first1000.svg"), f); # hide +```` + +\fig{gibbs_first1000} +\center{*First 1,000 Samples Generated from the Gibbs Algorithm after warm-up*} \\ + +And, finally, lets take a look in the all 9,000 samples generated after the warm-up of 1,000 iterations. + +````julia:ex22 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_gibbs[(2 * warmup):end, 1], X_gibbs[(2 * warmup):end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "gibbs_all.svg"), f); # hide +```` + +\fig{gibbs_all} +\center{*All 9,000 Samples Generated from the Gibbs Algorithm after warm-up*} \\ + +### What happens when we run Markov chains in parallel? + +Since the Markov chains are **independent**, we can run them in **parallel**. The key to this is +**defining different starting points for each Markov chain** (if you use a sample of a previous distribution +of parameters as a starting point this is not a problem). We will use the same toy example of a bivariate normal +distribution $X$ and $Y$ that we used in the previous examples, but now with **4 Markov chains in parallel +with different starting points**[^markovparallel]. + +First, let's defined 4 different pairs of starting points using a nice Cartesian product +from Julia's `Base.Iterators`: + +````julia:ex23 +const starts = collect(Iterators.product((-2.5, 2.5), (2.5, -2.5))) +```` + +Also, I will restrict this simulation to 100 samples: + +````julia:ex24 +const S_parallel = 100; +```` + +Additionally, note that we are using different `seed`s: + +````julia:ex25 +X_met_1 = metropolis( + S_parallel, width, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1]) +); +X_met_2 = metropolis( + S_parallel, width, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2]) +); +X_met_3 = metropolis( + S_parallel, width, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3]) +); +X_met_4 = metropolis( + S_parallel, width, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4]) +); +```` + +There have been some significant changes in the approval rate for Metropolis proposals. All were around 13%-24%, +this is due to the low number of samples (only 100 for each Markov chain), if the samples were larger we would see +these values converge to close to 20% according to the previous example of 10,000 samples with a single stream +(Roberts et. al, 1997). + +Now let's take a look on how those 4 Metropolis Markov chains sample the parameter space starting from different positions. +Each chain will have its own marker and path color, so that we can see their different behavior: + +````julia:ex26 +using Colors +const logocolors = Colors.JULIA_LOGO_COLORS; +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "parallel_met.gif"); framerate=5) do frame + for i in 1:99 + scatter!(ax, (X_met_1[i, 1], X_met_1[i, 2]); color=(logocolors.blue, 0.5)) + linesegments!( + X_met_1[i:(i + 1), 1], X_met_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5) + ) + scatter!(ax, (X_met_2[i, 1], X_met_2[i, 2]); color=(logocolors.red, 0.5)) + linesegments!( + X_met_2[i:(i + 1), 1], X_met_2[i:(i + 1), 2]; color=(logocolors.red, 0.5) + ) + scatter!(ax, (X_met_3[i, 1], X_met_3[i, 2]); color=(logocolors.green, 0.5)) + linesegments!( + X_met_3[i:(i + 1), 1], X_met_3[i:(i + 1), 2]; color=(logocolors.green, 0.5) + ) + scatter!(ax, (X_met_4[i, 1], X_met_4[i, 2]); color=(logocolors.purple, 0.5)) + linesegments!( + X_met_4[i:(i + 1), 1], X_met_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5) + ) + recordframe!(frame) + end +end; +```` + +\fig{parallel_met} +\center{*Animation of 4 Parallel Metropolis Markov Chains*} \\ + +Now we'll do the the same for Gibbs, taking care to provide also different `seed`s and starting points: + +````julia:ex27 +X_gibbs_1 = gibbs( + S_parallel * 2, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1]) +); +X_gibbs_2 = gibbs( + S_parallel * 2, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2]) +); +X_gibbs_3 = gibbs( + S_parallel * 2, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3]) +); +X_gibbs_4 = gibbs( + S_parallel * 2, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4]) +); +```` + +Now let's take a look on how those 4 Gibbs Markov chains sample the parameter space starting from different positions. +Each chain will have its own marker and path color, so that we can see their different behavior: + +````julia:ex28 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "parallel_gibbs.gif"); framerate=5) do frame + for i in 1:199 + scatter!(ax, (X_gibbs_1[i, 1], X_gibbs_1[i, 2]); color=(logocolors.blue, 0.5)) + linesegments!( + X_gibbs_1[i:(i + 1), 1], X_gibbs_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5) + ) + scatter!(ax, (X_gibbs_2[i, 1], X_gibbs_2[i, 2]); color=(logocolors.red, 0.5)) + linesegments!( + X_gibbs_2[i:(i + 1), 1], X_gibbs_2[i:(i + 1), 2]; color=(logocolors.red, 0.5) + ) + scatter!(ax, (X_gibbs_3[i, 1], X_gibbs_3[i, 2]); color=(logocolors.green, 0.5)) + linesegments!( + X_gibbs_3[i:(i + 1), 1], X_gibbs_3[i:(i + 1), 2]; color=(logocolors.green, 0.5) + ) + scatter!(ax, (X_gibbs_4[i, 1], X_gibbs_4[i, 2]); color=(logocolors.purple, 0.5)) + linesegments!( + X_gibbs_4[i:(i + 1), 1], X_gibbs_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5) + ) + recordframe!(frame) + end +end; +```` + +\fig{parallel_gibbs} +\center{*Animation of 4 Parallel Gibbs Markov Chains*} \\ + +## Hamiltonian Monte Carlo -- HMC + +The problems of low acceptance rates of proposals for Metropolis techniques and the low performance of the Gibbs algorithm +in multidimensional problems (in which the posterior's topology is quite complex) led to the emergence of a new MCMC technique +using Hamiltonian dynamics (in honor of the Irish physicist [ William Rowan Hamilton](https://en.wikipedia.org/wiki/William_Rowan_Hamilton) +(1805-1865) (figure below). The name for this technique is *Hamiltonian Monte Carlo* -- HMC. + +![William Rowan Hamilton](/pages/images/hamilton.png) + +\center{*William Rowan Hamilton*} \\ + +The HMC is an adaptation of the Metropolis technique and employs a guided scheme for generating new proposals: +this improves the proposal's acceptance rate and, consequently, efficiency. More specifically, the HMC uses +the posterior log gradient to direct the Markov chain to regions of higher posterior density, where most samples +are collected. As a result, a Markov chain with the well-adjusted HMC algorithm will accept proposals at a much higher +rate than the traditional Metropolis algorithm (Roberts et. al, 1997). + +HMC was first described in the physics literature (Duane, Kennedy, Pendleton & Roweth, 1987) (which they called the +*"Hybrid" Monte Carlo* -- HMC). Soon after, HMC was applied to statistical problems by Neal (1994) (which he called +*Hamiltonean Monte Carlo* -- HMC). For an in-depth discussion regarding HMC (which is not the focus of this tutorial), +I recommend Neal (2011) and Betancourt (2017). + +HMC uses Hamiltonian dynamics applied to particles exploring the topology of a posterior density. In some simulations +Metropolis has an acceptance rate of approximately 23%, while HMC 65% (Gelman et al., 2013b). In addition to better +exploring the posterior's topology and tolerating complex topologies, HMC is much more efficient than Metropolis and +does not suffer from the Gibbs' parameter correlation problem. + +For each component $\theta_j$, the HMC adds a momentum variable $\phi_j$. The subsequent density $P(\theta \mid y)$ +is increased by an independent distribution $P(\phi)$ of the momentum, thus defining a joint distribution: + +$$ P(\theta, \phi \mid y) = P(\phi) \cdot P(\theta \mid y) \label{hmcjoint} $$ + +HMC uses a proposal distribution that changes depending on the current state in the Markov chain. The HMC discovers +the direction in which the posterior distribution increases, called *gradient*, and distorts the distribution of proposals +towards the *gradient*. In the Metropolis algorithm, the distribution of the proposals would be a (usually) Normal distribution +centered on the current position, so that jumps above or below the current position would have the same probability of being +proposed. But the HMC generates proposals quite differently. + +You can imagine that for high-dimensional posterior densities that have *narrow diagonal valleys* and even *curved valleys*, +the HMC dynamics will find proposed positions that are much more **promising** than a naive symmetric proposal distribution, +and more promising than the Gibbs sampling, which can get stuck in *diagonal walls*. + +The probability of the Markov chain changing state in the HMC algorithm is defined as: + +$$ +P_{\text{change}} = \min\left({\frac{P(\theta_{\text{proposed}}) \cdot +P(\phi_{\text{proposed}})}{P(\theta_{\text{current}})\cdot P(\phi_{\text{current}})}}, 1\right) \label{hmcproposal} +$$ + +where $\phi$ is the momentum. + +### Momentum Distribution -- $P(\phi)$ + +We normally give $\phi$ a normal multivariate distribution with a mean of 0 and a covariance of $\mathbf{M}$, a "mass matrix". +To keep things a little bit simpler, we use a diagonal mass matrix $\mathbf{M}$. This makes the components of $\phi$ independent +with $\phi_j \sim \text{Normal}(0, M_{jj})$ + +### HMC Algorithm + +The HMC algorithm is very similar to the Metropolis algorithm but with the inclusion of the momentum $\phi$ as a way of +quantifying the gradient of the posterior distribution: + +1. Sample $\phi$ from a $\text{Normal}(0, \mathbf{M})$ + +2. Simultaneously sample $\theta$ and $\phi$ with $L$ *leapfrog steps* each scaled by a $\epsilon$ factor. In a *leapfrog step*, both $\theta$ and $\phi$ are changed, in relation to each other. Repeat the following steps $L$ times: + +2.1. Use the gradient of log posterior of $\theta$ to produce a *half-step* of $\phi$: + + $$\phi \leftarrow \phi + \frac{1}{2} \epsilon \frac{d \log p(\theta \mid y)}{d \theta}$$ + +2.2 Use the momentum vector $\phi$ to update the parameter vector $\theta$: + + $$ \theta \leftarrow \theta + \epsilon \mathbf{M}^{-1} \phi $$ + +2.3. Use again the gradient of log posterior of $\theta$ to another *half-step* of $\phi$: + + $$ \phi \leftarrow \phi + \frac{1}{2} \epsilon \frac{d \log p(\theta \mid y)}{d \theta} $$ + +3. Assign $\theta^{t-1}$ and $\phi^{t-1}$ as the values of the parameter vector and the momentum vector, respectively, at the beginning of the *leapfrog* process (step 2) and $\theta^*$ and $\phi^*$ as the values after $L$ steps. As an acceptance/rejection rule calculate: + + $$ r = \frac{p(\theta^* \mid y) p(\phi^*)}{p(\theta^{t-1} \mid y) p(\phi^{-1})} $$ + +4. Assign: + + $$\theta^t = + \begin{cases} + \theta^* & \text{with probability } \min (r, 1) \\ + \theta^{t-1} & \text{otherwise} + \end{cases}$$ + +### HMC -- Implementation + +Alright let's code the HMC algorithm for our toy example's bivariate normal distribution: + +````julia:ex29 +using ForwardDiff: gradient +function hmc( + S::Int64, + width::Float64, + ρ::Float64; + L=40, + ϵ=0.001, + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + accepted = 0::Int64 + x = start_x + y = start_y + @inbounds draws[1, :] = [x y] + M = [1.0 0.0; 0.0 1.0] + ϕ_d = MvNormal([0.0, 0.0], M) + for s in 2:S + x_ = rand(rgn, Uniform(x - width, x + width)) + y_ = rand(rgn, Uniform(y - width, y + width)) + ϕ = rand(rgn, ϕ_d) + kinetic = sum(ϕ .^ 2) / 2 + log_p = logpdf(binormal, [x, y]) - kinetic + ϕ += 0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_]) + for l in 1:L + x_, y_ = [x_, y_] + (ϵ * M * ϕ) + ϕ += +0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_]) + end + ϕ = -ϕ # make the proposal symmetric + kinetic = sum(ϕ .^ 2) / 2 + log_p_ = logpdf(binormal, [x_, y_]) - kinetic + r = exp(log_p_ - log_p) + + if r > rand(rgn, Uniform()) + x = x_ + y = y_ + accepted += 1 + end + @inbounds draws[s, :] = [x y] + end + println("Acceptance rate is: $(accepted / S)") + return draws +end +```` + +In the `hmc()` function above I am using the `gradient()` function from +[`ForwardDiff.jl`](https://github.com/JuliaDiff/ForwardDiff.jl) (Revels, Lubin & Papamarkou, 2016) +which is Julia's package for forward mode auto differentiation (autodiff). +The `gradient()` function accepts a function as input and an array $\mathbf{X}$. It literally evaluates the function $f$ +at $\mathbf{X}$ and returns the gradient $\nabla f(\mathbf{X})$. +This is one the advantages of Julia: I don't need to implement an autodiff for `logpdf()`s of any distribution, it will +be done automatically for any one from `Distributions.jl`. You can also try reverse mode autodiff with +[`ReverseDiff.jl`](https://github.com/JuliaDiff/ReverseDiff.jl) if you want to; and it will also very easy to get a gradient. +Now, we've got carried away with Julia's amazing autodiff potential... Let me show you an example of a gradient of a log PDF +evaluated at some value. I will use our `mvnormal` bivariate normal distribution as an example and evaluate its gradient +at $x = 1$ and $y = -1$: + +````julia:ex30 +gradient(x -> logpdf(mvnormal, x), [1, -1]) +```` + +So the gradient tells me that the partial derivative of $x = 1$ with respect to our `mvnormal` distribution is `-5` +and the partial derivative of $y = -1$ with respect to our `mvnormal` distribution is `5`. + +Now let's run our HMC Markov chain. +We are going to use $L = 40$ and (don't ask me how I found out) $\epsilon = 0.0856$: + +````julia:ex31 +X_hmc = hmc(S, width, ρ; ϵ=0.0856, L=40); +```` + +Our acceptance rate is 20.79%. +As before lets' take a quick peek into `X_hmc`, we'll see it's a matrix of $X$ and $Y$ values as columns and the time $t$ as rows: + +````julia:ex32 +X_hmc[1:10, :] +```` + +Again, we construct a `Chains` object by passing a matrix along with the parameters names as +symbols inside the `Chains()` constructor: + +````julia:ex33 +chain_hmc = Chains(X_hmc, [:X, :Y]); +```` + +Then we can get summary statistics regarding our Markov chain derived from the HMC algorithm: + +````julia:ex34 +summarystats(chain_hmc) +```` + +Both of `X` and `Y` have mean close to 0 and standard deviation close to 1 (which +are the theoretical values). +Take notice of the `ess` (effective sample size - ESS) that is around 1,600. +Now let's calculate the efficiency of our HMC algorithm by dividing +the ESS by the number of sampling iterations: + +````julia:ex35 +mean(summarystats(chain_hmc)[:, :ess_tail]) / S +```` + +We see that a simple naïve (and not well-calibrated[^calibrated]) HMC has 70% more efficiency from both Gibbs and Metropolis. +≈ 10% versus ≈ 17%. Great! 😀 + +#### HMC -- Visual Intuition + +This wouldn't be complete without animations for HMC! + +The animation in figure below shows the first 100 simulations of the HMC algorithm used to generate `X_hmc`. +Note that we have a gradient-guided proposal at each iteration, so the animation would resemble more like +a very lucky random-walk Metropolis [^luckymetropolis]. +The blue ellipsis represents the 90% HPD of our toy example's bivariate normal distribution. + +````julia:ex36 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "hmc_anim.gif"); framerate=5) do frame + for i in 1:100 + scatter!(ax, (X_hmc[i, 1], X_hmc[i, 2]); color=(:red, 0.5)) + linesegments!(X_hmc[i:(i + 1), 1], X_hmc[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; +```` + +\fig{hmc_anim} +\center{*Animation of the First 100 Samples Generated from the HMC Algorithm*} \\ + +As usual, let's take a look how the first 1,000 simulations were, excluding 1,000 initial iterations as warm-up. + +````julia:ex37 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_hmc[warmup:(warmup + 1_000), 1], + X_hmc[warmup:(warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "hmc_first1000.svg"), f); # hide +```` + +\fig{hmc_first1000} +\center{*First 1,000 Samples Generated from the HMC Algorithm after warm-up*} \\ + +And, finally, lets take a look in the all 9,000 samples generated after the warm-up of 1,000 iterations. + +````julia:ex38 +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_hmc[warmup:end, 1], X_hmc[warmup:end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "hmc_all.svg"), f); # hide +```` + +\fig{hmc_all} +\center{*All 9,000 Samples Generated from the HMC Algorithm after warm-up*} \\ + +### HMC -- Complex Topologies + +There are cases where HMC will be much better than Metropolis or Gibbs. In particular, these cases focus on complicated +and difficult-to-explore posterior topologies. In these contexts, an algorithm that can guide the proposals to regions +of higher density (such as the case of the HMC) is able to explore much more efficient (less iterations for convergence) +and effective (higher rate of acceptance of the proposals). + +See figure below for an example of a bimodal posterior density with also the marginal histogram of $X$ and $Y$: + +$$ +X = \text{Normal} \left( +\begin{bmatrix} +10 \\ +2 +\end{bmatrix}, +\begin{bmatrix} +1 & 0 \\ +0 & 1 +\end{bmatrix} +\right), \quad +Y = \text{Normal} \left( +\begin{bmatrix} +0 \\ +0 +\end{bmatrix}, +\begin{bmatrix} +8.4 & 2.0 \\ +2.0 & 1.7 +\end{bmatrix} +\right) +$$ + +````julia:ex39 +d1 = MvNormal([10, 2], [1 0; 0 1]) +d2 = MvNormal([0, 0], [8.4 2.0; 2.0 1.7]) + +d = MixtureModel([d1, d2]) + +x = -6:0.01:15 +y = -2.5:0.01:4.2 +dens_mixture = [pdf(d, [i, j]) for i in x, j in y] + +f, ax, s = surface( + x, + y, + dens_mixture; + axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 4), +) +save(joinpath(@OUTPUT, "bimodal.svg"), f); # hide +```` + +\fig{bimodal} +\center{*Multivariate Bimodal Normal*} \\ + +And to finish an example of Neal's funnel (Neal, 2003) in the figure below. This is a very difficult posterior to be sampled +even for HMC, as it varies in geometry in the dimensions $X$ and $Y$. This means that the HMC sampler has to change the +*leapfrog steps* $L$ and the scaling factor $\epsilon$ every time, since at the top of the image (the top of the funnel) +a large value of $L$ is needed along with a small $\epsilon$; and at the bottom (at the bottom of the funnel) the opposite: +small $L$ and large $\epsilon$. + +````julia:ex40 +funnel_y = rand(Normal(0, 3), 10_000) +funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2) + +f, ax, s = scatter( + funnel_x, + funnel_y; + color=(:steelblue, 0.3), + axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)), +) +save(joinpath(@OUTPUT, "funnel.svg"), f); # hide +```` + +\fig{funnel} +\center{*Neal's Funnel*} \\ + +## "I understood nothing..." + +If you haven't understood anything by now, don't despair. Skip all the formulas and get the visual intuition of the algorithms. +See the limitations of Metropolis and Gibbs and compare the animations and figures with those of the HMC. The superiority +of efficiency (more samples with low autocorrelation - ESS) and effectiveness (more samples close to the most likely areas +of the target distribution) is self-explanatory by the images. + +In addition, you will probably **never** have to code your HMC algorithm (Gibbs, Metropolis or any other MCMC) by hand. +For that there are packages like Turing's [`AdvancedHMC.jl`](https://github.com/TuringLang/AdvancedHMC.jl) +In addition, `AdvancedHMC` has a modified HMC with a technique called **N**o-**U**-**T**urn **S**ampling (NUTS)[^nuts] +(Hoffman & Gelman, 2011) that selects automatically the values ​​of $\epsilon$ (scaling factor) and $L$ (*leapfrog steps*). +The performance of the HMC is highly sensitive to these two "hyperparameters" (parameters that must be specified by the user). +In particular, if $L$ is too small, the algorithm exhibits undesirable behavior of a random walk, while if $L$ is too large, +the algorithm wastes computational efficiency. NUTS uses a recursive algorithm to build a set of likely candidate points that span +a wide range of proposal distribution, automatically stopping when it starts to go back and retrace its steps +(why it doesn't turn around - *No U-turn*), in addition NUTS also automatically calibrates simultaneously $L$ and $\epsilon$. + +## MCMC Metrics + +All Markov chains have some convergence and diagnostics metrics that you should be aware of. Note that this is still an ongoing +area of intense research and new metrics are constantly being proposed +(*e.g.* Lambert & Vehtari (2020) or Vehtari, Gelman., Simpson, Carpenter & Bürkner (2021)) +To show MCMC metrics let me recover our six-sided dice example from [4. **How to use Turing**](/pages/4_Turing/): + +````julia:ex41 +using Turing +setprogress!(false) # hide + +@model function dice_throw(y) + #Our prior belief about the probability of each result in a six-sided dice. + #p is a vector of length 6 each with probability p that sums up to 1. + p ~ Dirichlet(6, 1) + + #Each outcome of the six-sided dice has a probability p. + return y ~ filldist(Categorical(p), length(y)) +end; +```` + +Let's once again generate 1,000 throws of a six-sided dice: + +````julia:ex42 +data_dice = rand(DiscreteUniform(1, 6), 1_000); +```` + +Like before we'll sample using `NUTS()` and 1,000 iterations. Note that you can use Metropolis with `MH()`, Gibbs with `Gibbs()` +and HMC with `HMC()` if you want to. You can check all Turing's different MCMC samplers in +[Turing's documentation](https://turing.ml/dev/docs/using-turing/guide). + +````julia:ex43 +model = dice_throw(data_dice) +chain = sample(model, NUTS(), 1_000); +summarystats(chain) +```` + +We have the following columns that outputs some kind of MCMC summary statistics: + +* `mcse`: **M**onte **C**arlo **S**tandard **E**rror, the uncertainty about a statistic in the sample due to sampling error. +* `ess`: **E**ffective **S**ample **S**ize, a rough approximation of the number of effective samples sampled by the MCMC estimated by the value of `rhat`. +* `rhat`: a metric of convergence and stability of the Markov chain. + +The most important metric to take into account is the `rhat` which is a metric that measures whether the Markov chains +are stable and converged to a value during the total progress of the sampling procedure. It is basically the proportion +of variation when comparing two halves of the chains after discarding the warmups. A value of 1 implies convergence +and stability. By default, `rhat` must be less than 1.01 for the Bayesian estimation to be valid +(Brooks & Gelman, 1998; Gelman & Rubin, 1992). + +Note that all of our model's parameters have achieve a nice `ess` and, more important, `rhat` in the desired range, a solid +indicator that the Markov chain is stable and has converged to the estimated parameter values. + +### What looks like when your model doesn't converge + +Depending on the model and data, it is possible that HMC (even with NUTS) will not achieve convergence. +NUTS will not converge if it encounters divergences either due to a very pathological posterior density topology +or if you supply improper parameters. To exemplify let me run a "bad" chain by passing the `NUTS()` constructor +a target acceptance rate of `0.3` with only 500 sampling iterations (including warmup of 1,000 iters): + +````julia:ex44 +bad_chain = sample(model, NUTS(0.3), 500) +summarystats(bad_chain) +```` + +Here we can see that the `ess` and `rhat` of the `bad_chain` are *really* bad! +There will be several divergences that we can access in the column `numerical_error` of a `Chains` object. Here we have 64 divergences. + +````julia:ex45 +sum(bad_chain[:numerical_error]) +```` + +Also we can see the Markov chain acceptance rate in the column `acceptance_rate`. This is the `bad_chain` acceptance rate: + +````julia:ex46 +mean(bad_chain[:acceptance_rate]) +```` + +And now the "good" `chain`: + +````julia:ex47 +mean(chain[:acceptance_rate]) +```` + +What a difference huh? 80% versus 1.5%. + +### MCMC Visualizations + +Besides the `rhat` values, we can also visualize the Markov chain with a *traceplot*. +The *traceplot* is the overlap of the MCMC chain sampling for each estimated parameter (vertical axis). +The idea is that the chains mix and that there is no slope or visual pattern along the iterations (horizontal axis). +This demonstrates that the chains have mixed converged to a certain value of the parameter and remained in that region +during a good part (or all) of the Markov chain sampling. We can do that with the `MCMCChains.jl`'s function `traceplot()`. +Let's look the "good" `chain` first: + +````julia:ex48 +using AlgebraOfGraphics +params = names(chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines) +f = Figure(; resolution=(1200, 900)) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "traceplot_chain.svg"), f); # hide +```` + +\fig{traceplot_chain} +\center{*Traceplot for `chain`*} \\ + +And now the `bad_chain`: + +````julia:ex49 +params = names(bad_chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt = data(bad_chain) * mapping(:iteration) * chain_mapping * visual(Lines) +f = Figure(; resolution=(1200, 900)) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "traceplot_bad_chain.svg"), f); # hide +```` + +\fig{traceplot_bad_chain} +\center{*Traceplot for `bad_chain`*} \\ + +We can see that the `bad_chain` is all over the place and has definitely not converged or became stable. + +## How to make your Markov chains converge + +**First**: Before making fine adjustments to the number of chains or the number of iterations (among others ...) +know that Turing's NUTS sampler is very efficient and effective in exploring the most diverse and complex "crazy" +topologies of posteriors' target distributions. The standard arguments `NUTS()` work perfectly for 99% of cases +(even in complex models). That said, **most of the time when you have sampling and computational problems in your +Bayesian model, the problem is in the model specification and not in the MCMC sampling algorithm**. +This is known as *Folk Theorem* (Gelman, 2008). In the words of Andrew Gelman: + +> "When you have computational problems, often there's a problem with your model". +> \\ \\ +> Andrew Gelman (Gelman, 2008) + +If your Bayesian model has problems with convergence there are some steps that can be tried[^QR]. +Listed here from the simplest to the most complex: + +1. **Increase the number of iterations and chains**: First option is to increase the number of MCMC iterations and it is also possible to increase the number of parallel chains to be sampled. +2. **Model reparameterization**: the second option is to reparameterize the model. There are two ways to parameterize the model: the first with centered parameterization (CP) and the second with non-centered parameterization (NCP). NCP is most useful in Multilevel Models, therefore we will cover NCP in [10. **Multilevel Models**](/pages/10_multilevel_models/). +3. **Collect more data**: sometimes the model is too complex and we need a larger sample to get stable estimates. +4. **Rethink the model**: convergence failure when we have adequate sampling is usually due to a specification of priors and likelihood function that are not compatible with the data. In this case, it is necessary to rethink the data's generative process in which the model's assumptions are anchored. + +## Footnotes +[^propto]: the symbol $\propto$ (`\propto`) should be read as "proportional to". +[^warmup]: some references call this process *burnin*. +[^metropolis]: if you want a better explanation of the Metropolis and Metropolis-Hastings algorithms I suggest to see Chib & Greenberg (1995). +[^numerical]: Due to easier computational complexity and to avoid [numeric overflow](https://en.wikipedia.org/wiki/Integer_overflow) we generally use sum of logs instead of multiplications, specially when dealing with probabilities, *i.e.* $\mathbb{R} \in [0, 1]$. +[^mcmcchains]: this is one of the packages of Turing's ecosystem. I recommend you to take a look into [4. **How to use Turing**](/pages/4_Turing/). +[^gibbs]: if you want a better explanation of the Gibbs algorithm I suggest to see Casella & George (1992). +[^gibbs2]: this will be clear in the animations and images. +[^markovparallel]: note that there is some shenanigans here to take care. You would also want to have different seeds for the random number generator in each Markov chain. This is why `metropolis()` and `gibbs()` have a `seed` parameter. +[^calibrated]: as detailed in the following sections, HMC is quite sensitive to the choice of $L$ and $\epsilon$ and I haven't tried to get the best possible combination of those. +[^luckymetropolis]: or a not-drunk random-walk Metropolis 😂. +[^nuts]: NUTS is an algorithm that uses the symplectic leapfrog integrator and builds a binary tree composed of leaf nodes that are simulations of Hamiltonian dynamics using $2^j$ *leapfrog steps* in forward or backward directions in time where $j$ is the integer representing the iterations of the construction of the binary tree. Once the simulated particle starts to retrace its trajectory, the tree construction is interrupted and the ideal number of $L$ *leapfrog steps* is defined as $2^j$ in time $j-1$ from the beginning of the retrogression of the trajectory. So the simulated particle never "turns around" so "No U-turn". For more details on the algorithm and how it relates to Hamiltonian dynamics see Hoffman & Gelman (2011). +[^QR]: furthermore, if you have high-correlated variables in your model, you can try a QR decomposition of the data matrix $X$. I will cover QR decomposition and other Turing's computational "tricks of the trade" in [11. **Computational Tricks with Turing**](/pages/11_Turing_tricks/). + +## References + +Betancourt, M. (2017, January 9). A Conceptual Introduction to Hamiltonian Monte Carlo. Retrieved November 6, 2019, from http://arxiv.org/abs/1701.02434 + +Brooks, S., Gelman, A., Jones, G., & Meng, X.-L. (2011). Handbook of Markov Chain Monte Carlo. Retrieved from https://books.google.com?id=qfRsAIKZ4rIC + +Brooks, S. P., & Gelman, A. (1998). General Methods for Monitoring Convergence of Iterative Simulations. Journal of Computational and Graphical Statistics, 7(4), 434–455. https://doi.org/10.1080/10618600.1998.10474787 + +Casella, G., & George, E. I. (1992). Explaining the gibbs sampler. The American Statistician, 46(3), 167–174. https://doi.org/10.1080/00031305.1992.10475878 + +Chib, S., & Greenberg, E. (1995). Understanding the Metropolis-Hastings Algorithm. The American Statistician, 49(4), 327–335. https://doi.org/10.1080/00031305.1995.10476177 + +Duane, S., Kennedy, A. D., Pendleton, B. J., & Roweth, D. (1987). Hybrid Monte Carlo. Physics Letters B, 195(2), 216–222. https://doi.org/10.1016/0370-2693(87)91197-X + +Eckhardt, R. (1987). Stan Ulam, John von Neumann, and the Monte Carlo Method. Los Alamos Science, 15(30), 131–136. + +Gelman, A. (1992). Iterative and Non-Iterative Simulation Algorithms. Computing Science and Statistics (Interface Proceedings), 24, 457–511. PROCEEDINGS PUBLISHED BY VARIOUS PUBLISHERS. + +Gelman, A. (2008). The folk theorem of statistical computing. Retrieved from https://statmodeling.stat.columbia.edu/2008/05/13/the_folk_theore/ + +Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013a). Basics of Markov Chain Simulation. In Bayesian Data Analysis. Chapman and Hall/CRC. + +Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013b). Bayesian Data Analysis. Chapman and Hall/CRC. + +Gelman, A., & Rubin, D. B. (1992). Inference from Iterative Simulation Using Multiple Sequences. Statistical Science, 7(4), 457–472. https://doi.org/10.1214/ss/1177011136 + +Geman, S., & Geman, D. (1984). Stochastic Relaxation, Gibbs Distributions, and the Bayesian Restoration of Images. IEEE Transactions on Pattern Analysis and Machine Intelligence, PAMI-6(6), 721–741. https://doi.org/10.1109/TPAMI.1984.4767596 + +Hastings, W. K. (1970). Monte Carlo sampling methods using Markov chains and their applications. Biometrika, 57(1), 97–109. https://doi.org/10.1093/biomet/57.1.97 + +Hoffman, M. D., & Gelman, A. (2011). The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo. Journal of Machine Learning Research, 15(1), 1593–1623. Retrieved from http://arxiv.org/abs/1111.4246 + +Lambert, B., & Vehtari, A. (2020). $R^*$: A robust MCMC convergence diagnostic with uncertainty using decision tree classifiers. ArXiv:2003.07900 [Stat]. http://arxiv.org/abs/2003.07900 + +Metropolis, N., Rosenbluth, A. W., Rosenbluth, M. N., Teller, A. H., & Teller, E. (1953). Equation of State Calculations by Fast Computing Machines. The Journal of Chemical Physics, 21(6), 1087–1092. https://doi.org/10.1063/1.1699114 + +Neal, Radford M. (1994). An Improved Acceptance Procedure for the Hybrid Monte Carlo Algorithm. Journal of Computational Physics, 111(1), 194–203. https://doi.org/10.1006/jcph.1994.1054 + +Neal, Radford M. (2003). Slice Sampling. The Annals of Statistics, 31(3), 705–741. Retrieved from https://www.jstor.org/stable/3448413 + +Neal, Radford M. (2011). MCMC using Hamiltonian dynamics. In S. Brooks, A. Gelman, G. L. Jones, & X.-L. Meng (Eds.), Handbook of markov chain monte carlo. + +Revels, J., Lubin, M., & Papamarkou, T. (2016). Forward-Mode Automatic Differentiation in Julia. ArXiv:1607.07892 [Cs]. http://arxiv.org/abs/1607.07892 + +Roberts, G. O., Gelman, A., & Gilks, W. R. (1997). Weak convergence and optimal scaling of random walk Metropolis algorithms. Annals of Applied Probability, 7(1), 110–120. https://doi.org/10.1214/aoap/1034625254 + +Vehtari, A., Gelman, A., Simpson, D., Carpenter, B., & Bürkner, P.-C. (2021). Rank-Normalization, Folding, and Localization: An Improved Rˆ for Assessing Convergence of MCMC. Bayesian Analysis, 1(1), 1–28. https://doi.org/10.1214/20-BA1221 + diff --git a/assets/literate/05_MCMC_script.jl b/assets/literate/05_MCMC_script.jl new file mode 100644 index 00000000..9f91f569 --- /dev/null +++ b/assets/literate/05_MCMC_script.jl @@ -0,0 +1,511 @@ +# This file was generated, do not modify it. + +using CairoMakie +using Distributions +using Random + +Random.seed!(123); + +const N = 100_000 +const μ = [0, 0] +const Σ = [1 0.8; 0.8 1] + +const mvnormal = MvNormal(μ, Σ) + +x = -3:0.01:3 +y = -3:0.01:3 +dens_mvnormal = [pdf(mvnormal, [i, j]) for i in x, j in y] +f, ax, c = contourf(x, y, dens_mvnormal; axis=(; xlabel=L"X", ylabel=L"Y")) +Colorbar(f[1, 2], c) +save(joinpath(@OUTPUT, "countour_mvnormal.svg"), f); # hide + +f, ax, s = surface( + x, + y, + dens_mvnormal; + axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 8), +) +save(joinpath(@OUTPUT, "surface_mvnormal.svg"), f); # hide + +function metropolis( + S::Int64, + width::Float64, + ρ::Float64; + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + accepted = 0::Int64 + x = start_x + y = start_y + @inbounds draws[1, :] = [x y] + for s in 2:S + x_ = rand(rgn, Uniform(x - width, x + width)) + y_ = rand(rgn, Uniform(y - width, y + width)) + r = exp(logpdf(binormal, [x_, y_]) - logpdf(binormal, [x, y])) + + if r > rand(rgn, Uniform()) + x = x_ + y = y_ + accepted += 1 + end + @inbounds draws[s, :] = [x y] + end + println("Acceptance rate is: $(accepted / S)") + return draws +end + +const S = 10_000 +const width = 2.75 +const ρ = 0.8 + +X_met = metropolis(S, width, ρ); + +X_met[1:10, :] + +using MCMCChains + +chain_met = Chains(X_met, [:X, :Y]); + +summarystats(chain_met) + +mean(summarystats(chain_met)[:, :ess_tail]) / S + +using LinearAlgebra: eigvals, eigvecs +#source: https://discourse.julialang.org/t/plot-ellipse-in-makie/82814/4 +function getellipsepoints(cx, cy, rx, ry, θ) + t = range(0, 2 * pi; length=100) + ellipse_x_r = @. rx * cos(t) + ellipse_y_r = @. ry * sin(t) + R = [cos(θ) sin(θ); -sin(θ) cos(θ)] + r_ellipse = [ellipse_x_r ellipse_y_r] * R + x = @. cx + r_ellipse[:, 1] + y = @. cy + r_ellipse[:, 2] + return (x, y) +end +function getellipsepoints(μ, Σ; confidence=0.95) + quant = sqrt(quantile(Chisq(2), confidence)) + cx = μ[1] + cy = μ[2] + + egvs = eigvals(Σ) + if egvs[1] > egvs[2] + idxmax = 1 + largestegv = egvs[1] + smallesttegv = egvs[2] + else + idxmax = 2 + largestegv = egvs[2] + smallesttegv = egvs[1] + end + + rx = quant * sqrt(largestegv) + ry = quant * sqrt(smallesttegv) + + eigvecmax = eigvecs(Σ)[:, idxmax] + θ = atan(eigvecmax[2] / eigvecmax[1]) + if θ < 0 + θ += 2 * π + end + + return getellipsepoints(cx, cy, rx, ry, θ) +end + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "met_anim.gif"); framerate=5) do frame + for i in 1:100 + scatter!(ax, (X_met[i, 1], X_met[i, 2]); color=(:red, 0.5)) + linesegments!(X_met[i:(i + 1), 1], X_met[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; + +const warmup = 1_000 + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_met[warmup:(warmup + 1_000), 1], + X_met[warmup:(warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "met_first1000.svg"), f); # hide + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_met[warmup:end, 1], X_met[warmup:end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "met_all.svg"), f); # hide + +function gibbs( + S::Int64, + ρ::Float64; + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + x = start_x + y = start_y + β = ρ * σ_y / σ_x + λ = ρ * σ_x / σ_y + sqrt1mrho2 = sqrt(1 - ρ^2) + σ_YX = σ_y * sqrt1mrho2 + σ_XY = σ_x * sqrt1mrho2 + @inbounds draws[1, :] = [x y] + for s in 2:S + if s % 2 == 0 + y = rand(rgn, Normal(μ_y + β * (x - μ_x), σ_YX)) + else + x = rand(rgn, Normal(μ_x + λ * (y - μ_y), σ_XY)) + end + @inbounds draws[s, :] = [x y] + end + return draws +end + +X_gibbs = gibbs(S * 2, ρ); + +X_gibbs[1:10, :] + +chain_gibbs = Chains(X_gibbs, [:X, :Y]); + +summarystats(chain_gibbs) + +(mean(summarystats(chain_gibbs)[:, :ess_tail]) / 2) / S + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "gibbs_anim.gif"); framerate=5) do frame + for i in 1:200 + scatter!(ax, (X_gibbs[i, 1], X_gibbs[i, 2]); color=(:red, 0.5)) + linesegments!(X_gibbs[i:(i + 1), 1], X_gibbs[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_gibbs[(2 * warmup):(2 * warmup + 1_000), 1], + X_gibbs[(2 * warmup):(2 * warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "gibbs_first1000.svg"), f); # hide + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_gibbs[(2 * warmup):end, 1], X_gibbs[(2 * warmup):end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "gibbs_all.svg"), f); # hide + +const starts = collect(Iterators.product((-2.5, 2.5), (2.5, -2.5))) + +const S_parallel = 100; + +X_met_1 = metropolis( + S_parallel, width, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1]) +); +X_met_2 = metropolis( + S_parallel, width, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2]) +); +X_met_3 = metropolis( + S_parallel, width, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3]) +); +X_met_4 = metropolis( + S_parallel, width, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4]) +); + +using Colors +const logocolors = Colors.JULIA_LOGO_COLORS; +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "parallel_met.gif"); framerate=5) do frame + for i in 1:99 + scatter!(ax, (X_met_1[i, 1], X_met_1[i, 2]); color=(logocolors.blue, 0.5)) + linesegments!( + X_met_1[i:(i + 1), 1], X_met_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5) + ) + scatter!(ax, (X_met_2[i, 1], X_met_2[i, 2]); color=(logocolors.red, 0.5)) + linesegments!( + X_met_2[i:(i + 1), 1], X_met_2[i:(i + 1), 2]; color=(logocolors.red, 0.5) + ) + scatter!(ax, (X_met_3[i, 1], X_met_3[i, 2]); color=(logocolors.green, 0.5)) + linesegments!( + X_met_3[i:(i + 1), 1], X_met_3[i:(i + 1), 2]; color=(logocolors.green, 0.5) + ) + scatter!(ax, (X_met_4[i, 1], X_met_4[i, 2]); color=(logocolors.purple, 0.5)) + linesegments!( + X_met_4[i:(i + 1), 1], X_met_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5) + ) + recordframe!(frame) + end +end; + +X_gibbs_1 = gibbs( + S_parallel * 2, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1]) +); +X_gibbs_2 = gibbs( + S_parallel * 2, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2]) +); +X_gibbs_3 = gibbs( + S_parallel * 2, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3]) +); +X_gibbs_4 = gibbs( + S_parallel * 2, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4]) +); + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "parallel_gibbs.gif"); framerate=5) do frame + for i in 1:199 + scatter!(ax, (X_gibbs_1[i, 1], X_gibbs_1[i, 2]); color=(logocolors.blue, 0.5)) + linesegments!( + X_gibbs_1[i:(i + 1), 1], X_gibbs_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5) + ) + scatter!(ax, (X_gibbs_2[i, 1], X_gibbs_2[i, 2]); color=(logocolors.red, 0.5)) + linesegments!( + X_gibbs_2[i:(i + 1), 1], X_gibbs_2[i:(i + 1), 2]; color=(logocolors.red, 0.5) + ) + scatter!(ax, (X_gibbs_3[i, 1], X_gibbs_3[i, 2]); color=(logocolors.green, 0.5)) + linesegments!( + X_gibbs_3[i:(i + 1), 1], X_gibbs_3[i:(i + 1), 2]; color=(logocolors.green, 0.5) + ) + scatter!(ax, (X_gibbs_4[i, 1], X_gibbs_4[i, 2]); color=(logocolors.purple, 0.5)) + linesegments!( + X_gibbs_4[i:(i + 1), 1], X_gibbs_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5) + ) + recordframe!(frame) + end +end; + +using ForwardDiff: gradient +function hmc( + S::Int64, + width::Float64, + ρ::Float64; + L=40, + ϵ=0.001, + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + accepted = 0::Int64 + x = start_x + y = start_y + @inbounds draws[1, :] = [x y] + M = [1.0 0.0; 0.0 1.0] + ϕ_d = MvNormal([0.0, 0.0], M) + for s in 2:S + x_ = rand(rgn, Uniform(x - width, x + width)) + y_ = rand(rgn, Uniform(y - width, y + width)) + ϕ = rand(rgn, ϕ_d) + kinetic = sum(ϕ .^ 2) / 2 + log_p = logpdf(binormal, [x, y]) - kinetic + ϕ += 0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_]) + for l in 1:L + x_, y_ = [x_, y_] + (ϵ * M * ϕ) + ϕ += +0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_]) + end + ϕ = -ϕ # make the proposal symmetric + kinetic = sum(ϕ .^ 2) / 2 + log_p_ = logpdf(binormal, [x_, y_]) - kinetic + r = exp(log_p_ - log_p) + + if r > rand(rgn, Uniform()) + x = x_ + y = y_ + accepted += 1 + end + @inbounds draws[s, :] = [x y] + end + println("Acceptance rate is: $(accepted / S)") + return draws +end + +gradient(x -> logpdf(mvnormal, x), [1, -1]) + +X_hmc = hmc(S, width, ρ; ϵ=0.0856, L=40); + +X_hmc[1:10, :] + +chain_hmc = Chains(X_hmc, [:X, :Y]); + +summarystats(chain_hmc) + +mean(summarystats(chain_hmc)[:, :ess_tail]) / S + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "hmc_anim.gif"); framerate=5) do frame + for i in 1:100 + scatter!(ax, (X_hmc[i, 1], X_hmc[i, 2]); color=(:red, 0.5)) + linesegments!(X_hmc[i:(i + 1), 1], X_hmc[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_hmc[warmup:(warmup + 1_000), 1], + X_hmc[warmup:(warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "hmc_first1000.svg"), f); # hide + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_hmc[warmup:end, 1], X_hmc[warmup:end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "hmc_all.svg"), f); # hide + +d1 = MvNormal([10, 2], [1 0; 0 1]) +d2 = MvNormal([0, 0], [8.4 2.0; 2.0 1.7]) + +d = MixtureModel([d1, d2]) + +x = -6:0.01:15 +y = -2.5:0.01:4.2 +dens_mixture = [pdf(d, [i, j]) for i in x, j in y] + +f, ax, s = surface( + x, + y, + dens_mixture; + axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 4), +) +save(joinpath(@OUTPUT, "bimodal.svg"), f); # hide + +funnel_y = rand(Normal(0, 3), 10_000) +funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2) + +f, ax, s = scatter( + funnel_x, + funnel_y; + color=(:steelblue, 0.3), + axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)), +) +save(joinpath(@OUTPUT, "funnel.svg"), f); # hide + +using Turing +setprogress!(false) # hide + +@model function dice_throw(y) + #Our prior belief about the probability of each result in a six-sided dice. + #p is a vector of length 6 each with probability p that sums up to 1. + p ~ Dirichlet(6, 1) + + #Each outcome of the six-sided dice has a probability p. + return y ~ filldist(Categorical(p), length(y)) +end; + +data_dice = rand(DiscreteUniform(1, 6), 1_000); + +model = dice_throw(data_dice) +chain = sample(model, NUTS(), 1_000); +summarystats(chain) + +bad_chain = sample(model, NUTS(0.3), 500) +summarystats(bad_chain) + +sum(bad_chain[:numerical_error]) + +mean(bad_chain[:acceptance_rate]) + +mean(chain[:acceptance_rate]) + +using AlgebraOfGraphics +params = names(chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines) +f = Figure(; resolution=(1200, 900)) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "traceplot_chain.svg"), f); # hide + +params = names(bad_chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt = data(bad_chain) * mapping(:iteration) * chain_mapping * visual(Lines) +f = Figure(; resolution=(1200, 900)) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "traceplot_bad_chain.svg"), f); # hide diff --git a/assets/literate/06_linear_reg.md b/assets/literate/06_linear_reg.md new file mode 100644 index 00000000..5ac2f555 --- /dev/null +++ b/assets/literate/06_linear_reg.md @@ -0,0 +1,170 @@ + +# Bayesian Linear Regression + +> "All models are wrong but some are useful" +> \\ \\ +> George Box (Box, 1976) + +This tutorial begins with a very provocative quote from the statistician +[George Box](https://en.wikipedia.org/wiki/George_E._P._Box) (figure below) +on statistical models. Yes, all models are somehow wrong. But they are very useful. +The idea is that the reality is too complex for us to understand when analyzing it in +a naked and raw way. We need to somehow simplify it into individual components and analyze +their relationships. But there is a danger here: any simplification of reality promotes loss +of information in some way. Therefore, we always have a delicate balance between simplifications +of reality through models and the inherent loss of information. Now you ask me: +"how are they useful?" Imagine that you are in total darkness and you have a very powerful +flashlight but with a narrow beam. Are you going to throw the flashlight away because it +can't light everything around you and stay in the dark? You must use the flashlight to aim +at interesting places in the darkness in order to illuminate them. You will never find a +flashlight that illuminates everything with the clarity you need to analyze all the fine details +of reality. Just as you will never find a unique model that will explain the whole reality around +you. You need different flashlights just like you need different models. Without them you will +be in total darkness. + +![George Box](/pages/images/george_box.jpg) + +\center{*George Box*} \\ + +## Linear Regression + +Let's talk about a class of model known as linear regression. The idea here is to model a continuous dependent variable +with a linear combination of independent variables. + +$$ \mathbf{y} = \alpha + \mathbf{X} \boldsymbol{\beta} + \epsilon \label{linear reg} $$ + +where: + +* $\mathbf{y}$ -- dependent variable +* $\alpha$ -- intercept +* $\boldsymbol{\beta}$ -- coefficient vector +* $\mathbf{X}$ -- data matrix +* $\epsilon$ -- model error + +To estimate the $\boldsymbol{\beta}$ coefficients we use a Gaussian/normal likelihood function. +Mathematically the Bayesian regression model is: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Normal}\left( \alpha + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +Here we see that the likelihood function $P(\mathbf{y} \mid \boldsymbol{\theta})$ is a normal distribution in which $\mathbf{y}$ +depends on the parameters of the model $\alpha$ and $\boldsymbol{\beta}$, in addition to having an +error $\sigma$. We condition $\mathbf{y}$ onto the observed data $\mathbf{X}$ by inserting +$\alpha + \mathbf{X} \cdot \boldsymbol{\beta}$ as the linear predictor of the model (the mean $\mu$ parameter of the +model's Normal likelihood function, and $\sigma$ is the variance parameter). What remains is to specify which are the +priors of the model parameters: + +* Prior Distribution of $\alpha$ -- Knowledge we possess regarding the model's intercept. +* Prior Distribution of $\boldsymbol{\beta}$ -- Knowledge we possess regarding the model's independent variables' coefficients. +* Prior Distribution of $\sigma$ -- Knowledge we possess regarding the model's error. Important that the error can only be positive. In addition, it is intuitive to place a distribution that gives greater weight to values close to zero, but that also allows values that are far from zero, so a distribution with a long tail is welcome. Candidate distributions are $\text{Exponential}$ which is only supported on positive real numbers (so it solves the question of negative errors) or $\text{Cauchy}^+$ truncated to only positive numbers (remembering that the distribution Cauchy is Student's $t$ with degrees of freedom $\nu = 1$). + +Our goal is to instantiate a linear regression with the observed data ($\mathbf{y}$ and $\mathbf{X}$) and find the posterior +distribution of our model's parameters of interest ($\alpha$ and $\boldsymbol{\beta}$). This means to find the full posterior +distribution of: + +$$ P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta}, \sigma \mid \mathbf{y}) $$ + +This is easily accomplished with Turing: + +````julia:ex1 +using Turing +using LinearAlgebra: I +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function linreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + + #likelihood + return y ~ MvNormal(α .+ X * β, σ^2 * I) +end; +```` + +Here I am specifying very weakly informative priors: + +* $\alpha \sim \text{Normal}(\bar{\mathbf{y}}, 2.5 \cdot \sigma_{\mathbf{y}})$ -- This means a normal distribution centered on `y`'s mean with a standard deviation 2.5 times the standard deviation of `y`. That prior should with ease cover all possible values of $\alpha$. Remember that the normal distribution has support over all the real number line $\in (-\infty, +\infty)$. +* $\boldsymbol{\beta} \sim \text{Student-}t(0,1,3)$ -- The predictors all have a prior distribution of a Student-$t$ distribution centered on 0 with variance 1 and degrees of freedom $\nu = 3$. That wide-tailed $t$ distribution will cover all possible values for our coefficients. Remember the Student-$t$ also has support over all the real number line $\in (-\infty, +\infty)$. Also the `filldist()` is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution. +* $\sigma \sim \text{Exponential}(1)$ -- A wide-tailed-positive-only distribution perfectly suited for our model's error. + +Also, we are using the `MvNormal` construction where we specify both +a vector of means (first positional argument) +and a covariance matrix (second positional argument). +Regarding the covariance matrix `σ^2 * I`, +it uses the model's errors `σ`, here parameterized as a standard deviation, +squares it to produce a variance paramaterization, +and multiplies by `I`, which is Julia's `LinearAlgebra` standard module implementation +to represent an identity matrix of any size. + +## Example - Children's IQ Score + +For our example, I will use a famous dataset called `kidiq` (Gelman & Hill, 2007), which is data from a survey of adult American women and their respective children. Dated from 2007, it has 434 observations and 4 variables: + +* `kid_score`: child's IQ +* `mom_hs`: binary/dummy (0 or 1) if the child's mother has a high school diploma +* `mom_iq`: mother's IQ +* `mom_age`: mother's age + +Ok let's read our data with `CSV.jl` and output into a `DataFrame` from `DataFrames.jl`: + +````julia:ex2 +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv" +kidiq = CSV.read(HTTP.get(url).body, DataFrame) +describe(kidiq) +```` + +As you can see from the `describe()` output, the mean children's IQ is around 87 while the mother's is 100. Also the mother's +range from 17 to 29 years with mean of around 23 years old. Finally, note that 79% of mothers have a high school diploma. + +Now let's us instantiate our model with the data: + +````julia:ex3 +X = Matrix(select(kidiq, Not(:kid_score))) +y = kidiq[:, :kid_score] +model = linreg(X, y); +```` + +And, finally, we will sample from the Turing model. We will be using the default `NUTS()` sampler with `1_000` samples, but +now we will sample from 4 Markov chains using multiple threads `MCMCThreads()`: + +````julia:ex4 +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) +```` + +We had no problem with the Markov chains as all the `rhat` are well below `1.01` (or above `0.99`). +Our model has an error `σ` of around 18. So it estimates IQ±9. The intercept `α` is the basal child's IQ. +So each child has 22±9 IQ before we add the coefficients multiplied by the child's independent variables. +And from our coefficients $\boldsymbol{\beta}$, we can see that the `quantile()` tells us the uncertainty around their +estimates: + +````julia:ex5 +quantile(chain) +```` + +* `β[1]` -- first column of `X`, `mom_hs`, has 95% credible interval that is all over the place, including zero. So its effect on child's IQ is inconclusive. +* `β[2]` -- second column of `X`, `mom_iq`, has a 95% credible interval from 0.46 to 0.69. So we expect that every increase in the mother's IQ is associated with a 0.46 to 0.69 increase in the child's IQ. +* `β[3]` -- third column of `X`, `mom_age`, has also 95% credible interval that is all over the place, including zero. Like `mom_hs`, its effect on child's IQ is inconclusive. + +That's how you interpret 95% credible intervals from a `quantile()` output of a linear regression `Chains` object. + +## References + +Box, G. E. P. (1976). Science and Statistics. Journal of the American Statistical Association, 71(356), 791–799. https://doi.org/10.2307/2286841 + +Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press. + diff --git a/assets/literate/06_linear_reg_script.jl b/assets/literate/06_linear_reg_script.jl new file mode 100644 index 00000000..12bf2c97 --- /dev/null +++ b/assets/literate/06_linear_reg_script.jl @@ -0,0 +1,35 @@ +# This file was generated, do not modify it. + +using Turing +using LinearAlgebra: I +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function linreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + + #likelihood + return y ~ MvNormal(α .+ X * β, σ^2 * I) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv" +kidiq = CSV.read(HTTP.get(url).body, DataFrame) +describe(kidiq) + +X = Matrix(select(kidiq, Not(:kid_score))) +y = kidiq[:, :kid_score] +model = linreg(X, y); + +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) + +quantile(chain) diff --git a/assets/literate/07_logistic_reg.md b/assets/literate/07_logistic_reg.md new file mode 100644 index 00000000..2b2e4278 --- /dev/null +++ b/assets/literate/07_logistic_reg.md @@ -0,0 +1,281 @@ + +# Bayesian Logistic Regression + +Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The first is +**logistic regression** (also called binomial regression). + +A logistic regression behaves exactly like a linear model: it makes a prediction simply by computing a weighted +sum of the independent variables $\mathbf{X}$ by the estimated coefficients $\boldsymbol{\beta}$, plus an intercept +$\alpha$. However, instead of returning a continuous value $y$, such as linear regression, it returns the **logistic +function** of $y$: + +$$ \text{Logistic}(x) = \frac{1}{1 + e^{(-x)}} $$ + +We use logistic regression when our dependent variable is binary. It has only two distinct values, usually +encoded as $0$ or $1$. See the figure below for a graphical intuition of the logistic function: + +````julia:ex1 +using CairoMakie + +function logistic(x) + return 1 / (1 + exp(-x)) +end + +f, ax, l = lines(-10 .. 10, logistic; axis=(; xlabel=L"x", ylabel=L"\mathrm{Logistic}(x)")) +f +save(joinpath(@OUTPUT, "logistic.svg"), f); # hide +```` + +\fig{logistic} +\center{*Logistic Function*} \\ + +As we can see, the logistic function is basically a mapping of any real number to a +real number in the range between 0 and 1: + +$$ \text{Logistic}(x) = \{ \mathbb{R} \in [- \infty , + \infty] \} \to \{ \mathbb{R} \in (0, 1) \} $$ + +That is, the logistic function is the ideal candidate for when we need to convert something continuous without restrictions +to something continuous restricted between 0 and 1. That is why it is used when we need a model to have a probability as a +dependent variable (remembering that any real number between 0 and 1 is a valid probability). In the case of a binary dependent +variable, we can use this probability as the chance of the dependent variable taking a value of 0 or 1. + +## Comparison with Linear Regression + +Linear regression follows the following mathematical formulation: + +$$ \text{Linear} = \theta_0 + \theta_1 x_1 + \theta_2 x_2 + \dots + \theta_n x_n $$ + +* $\theta$ - model parameters + * $\theta_0$ - intercept + * $\theta_1, \theta_2, \dots$ - independent variables $x_1, x_2, \dots$ coefficients +* $n$ - total number of independent variables + +Logistic regression would add the logistic function to the linear term: + +* $\hat{p} = \text{Logistic}(\text{Linear}) = \frac{1}{1 + e^{-\operatorname{Linear}}}$ - predicted probability of the observation being the value 1 +* $\hat{\mathbf{y}}=\left\{\begin{array}{ll} 0 & \text { if } \hat{p} < 0.5 \\ 1 & \text { if } \hat{p} \geq 0.5 \end{array}\right.$ - predicted discrete value of $\mathbf{y}$ + +**Example**: + +$$ \text{Probability of Death} = \text{Logistic} \big(-10 + 10 \cdot \text{cancer} + 12 \cdot \text{diabetes} + 8 \cdot \text{obesity} \big) $$ + +## Bayesian Logistic Regression + +We can model logistic regression in two ways. The first option with a **Bernoulli likelihood** function and the second option with +a **binomial likelihood** function. + +With the **Bernoulli likelihood** we model a binary dependent variable $y$ which is the result of a Bernoulli trial with +a certain probability $p$. + +In a **binomial likelihood**, we model a continuous dependent variable $y$ which is the number of successes of $n$ +independent Bernoulli trials. + +### Using Bernoulli Likelihood + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Bernoulli}\left( p \right) \\ +\mathbf{p} &\sim \text{Logistic}(\alpha + \mathbf{X} \cdot \boldsymbol{\beta}) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) +\end{aligned} +$$ + +where: + +* $\mathbf{y}$ -- binary dependent variable. +* $\mathbf{p}$ -- probability of $\mathbf{y}$ taking the value of $\mathbf{y}$ -- success of an independent Bernoulli trial. +* $\text{Logistic}$ -- logistic function. +* $\alpha$ -- intercept. +* $\boldsymbol{\beta}$ -- coefficient vector. +* $\mathbf{X}$ -- data matrix. + +### Using Binomial Likelihood + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Binomial}\left( n, p \right) \\ +\mathbf{p} &\sim \text{Logistic}(\alpha + \mathbf{X} \cdot \boldsymbol{\beta}) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) +\end{aligned} +$$ + +where: + +* $\mathbf{y}$ -- binary dependent variable -- successes of $n$ independent Bernoulli trials. +* $n$ -- number of independent Bernoulli trials. +* $\mathbf{p}$ -- probability of $\mathbf{y}$ taking the value of $\mathbf{y}$ -- success of an independent Bernoulli trial. +* $\text{Logistic}$ -- logistic function. +* $\alpha$ -- intercept. +* $\boldsymbol{\beta}$ -- coefficient vector. +* $\mathbf{X}$ -- data matrix. + +In both likelihood options, what remains is to specify the model parameters' prior distributions: + +* Prior Distribution of $\alpha$ -- Knowledge we possess regarding the model's intercept. +* Prior Distribution of $\boldsymbol{\beta}$ -- Knowledge we possess regarding the model's independent variables' coefficients. + +Our goal is to instantiate a logistic regression with the observed data ($\mathbf{y}$ and $\mathbf{X}$) and find the posterior +distribution of our model's parameters of interest ($\alpha$ and $\boldsymbol{\beta}$). This means to find the full posterior +distribution of: + +$$ P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta} \mid \mathbf{y}) $$ + +Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error +parameter $\sigma$ in our logistic regression. This is due to neither the Bernoulli nor binomial distributions having +a "scale" parameter such as the $\sigma$ parameter in the Gaussian/normal distribution. + +Also note that the Bernoulli distribution is a special case of the binomial distribution where $n = 1$: + +$$ \text{Bernoulli}(p) = \text{Binomial}(1, p) $$ + +This is easily accomplished with Turing: + +````julia:ex2 +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function logreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + + #likelihood + return y ~ arraydist(LazyArray(@~ BernoulliLogit.(α .+ X * β))) +end; +```` + +Here I am specifying very weakly informative priors: + +* $\alpha \sim \text{Normal}(0, 2.5)$ -- This means a normal distribution centered on 0 with a standard deviation of 2.5. That prior should with ease cover all possible values of $\alpha$. Remember that the normal distribution has support over all the real number line $\in (-\infty, +\infty)$. +* $\boldsymbol{\beta} \sim \text{Student-}t(0,1,3)$ -- The predictors all have a prior distribution of a Student-$t$ distribution centered on 0 with variance 1 and degrees of freedom $\nu = 3$. That wide-tailed $t$ distribution will cover all possible values for our coefficients. Remember the Student-$t$ also has support over all the real number line $\in (-\infty, +\infty)$. Also the `filldist()` is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution. + +Turing's `arraydist()` function wraps an array of distributions returning a new distribution sampling from the individual +distributions. And the LazyArrays' `LazyArray()` constructor wrap a lazy object that wraps a computation producing an array +to an array. Last, but not least, the macro `@~` creates a broadcast and is a nice short hand for the familiar dot `.` +broadcasting operator in Julia. This is an efficient way to tell Turing that our `y` vector is distributed lazily as a +`BernoulliLogit` broadcasted to `α` added to the product of the data matrix `X` and `β` coefficient vector. + +If your dependent variable `y` is continuous and represents the number of successes of $n$ independent Bernoulli trials +you can use the binomial likelihood in your model: + +```julia +y ~ arraydist(LazyArray(@~ BinomialLogit.(n, α .+ X * β))) +``` + +## Example - Contaminated Water Wells + +For our example, I will use a famous dataset called `wells` (Gelman & Hill, 2007), which is data from a survey of 3,200 +residents in a small area of Bangladesh suffering from arsenic contamination of groundwater. Respondents with elevated +arsenic levels in their wells had been encouraged to switch their water source to a safe public or private well in the nearby +area and the survey was conducted several years later to learn which of the affected residents had switched wells. +It has 3,200 observations and the following variables: + +* `switch` -- binary/dummy (0 or 1) for well-switching. +* `arsenic` -- arsenic level in respondent's well. +* `dist` -- distance (meters) from the respondent's house to the nearest well with safe drinking water. +* `association` -- binary/dummy (0 or 1) if member(s) of household participate in community organizations. +* `educ` -- years of education (head of household). + +Ok let's read our data with `CSV.jl` and output into a `DataFrame` from `DataFrames.jl`: + +````julia:ex3 +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/wells.csv" +wells = CSV.read(HTTP.get(url).body, DataFrame) +describe(wells) +```` + +As you can see from the `describe()` output 58% of the respondents switched wells and 42% percent of respondents +somehow are engaged in community organizations. The average years of education of the household's head is approximate +5 years and ranges from 0 (no education at all) to 17 years. The distance to safe drinking water is measured in meters +and averages 48m ranging from less than 1m to 339m. Regarding arsenic levels I cannot comment because the only thing I +know that it is toxic and you probably would never want to have your well contaminated with it. Here, we believe that all +of those variables somehow influence the probability of a respondent switch to a safe well. + +Now let's us instantiate our model with the data: + +````julia:ex4 +X = Matrix(select(wells, Not(:switch))) +y = wells[:, :switch] +model = logreg(X, y); +```` + +And, finally, we will sample from the Turing model. We will be using the default `NUTS()` sampler with `1_000` samples, with +4 Markov chains using multiple threads `MCMCThreads()`: + +````julia:ex5 +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) +```` + +We had no problem with the Markov chains as all the `rhat` are well below `1.01` (or above `0.99`). +Note that the coefficients are in log-odds scale. They are the natural log of the odds[^logit], and odds is +defined as: + +$$ \text{odds} = \frac{p}{1-p} $$ + +where $p$ is a probability. So log-odds is defined as: + +$$ \log(\text{odds}) = \log \left( \frac{p}{1-x} \right) $$ + +So in order to get odds from a log-odds we must undo the log operation with a exponentiation. +This translates to: + +$$ \text{odds} = \exp ( \log ( \text{odds} )) $$ + +We can do this with a transformation in a `DataFrame` constructed from a `Chains` object: + +````julia:ex6 +using Chain + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end +```` + +Our interpretation of odds is the same as in betting games. Anything below 1 signals a unlikely probability that $y$ will be $1$. +And anything above 1 increases the probability of $y$ being $1$, while 1 itself is a neutral odds for $y$ being either $1$ or $0$. +Since I am not a gambling man, let's talk about probabilities. So I will create a function called `logodds2prob()` that converts +log-odds to probabilities: + +````julia:ex7 +function logodds2prob(logodds::Float64) + return exp(logodds) / (1 + exp(logodds)) +end + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false) +end +```` + +There you go, much better now. Let's analyze our results. The intercept `α` is the basal `switch` probability which has +a median value of 46%. All coefficients whose 95% credible intervals captures the value $\frac{1}{2} = 0.5$ tells +that the effect on the propensity of `switch` is inconclusive. It is pretty much similar to a 95% credible interval +that captures the 0 in the linear regression coefficients. So this rules out `β[3]` which is the third column of `X` +-- `assoc`. The other remaining 95% credible intervals can be interpreted as follows: + +* `β[1]` -- first column of `X`, `arsenic`, has 95% credible interval 0.595 to 0.634. This means that each increase in one unit of `arsenic` is related to an increase of 9.6% to 13.4% propension of `switch` being 1. +* `β[2]` -- second column of `X`, `dist`, has a 95% credible interval from 0.497 to 0.498. So we expect that each increase in one meter of `dist` is related to a decrease of 0.01% propension of `switch` being 1. +* `β[4]` -- fourth column of `X`, `educ`, has a 95% credible interval from 0.506 to 0.515. Each increase in one year of `educ` is related to an increase of 0.6% to 1.5% propension of `switch` being 1. + +That's how you interpret 95% credible intervals from a `quantile()` output of a logistic regression `Chains` +object converted from log-odds to probability. + +## Footnotes + +[^logit]: actually the [logit](https://en.wikipedia.org/wiki/Logit) function or the log-odds is the logarithm of the odds $\frac{p}{1-p}$ where $p$ is a probability. + +## References + +Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press. + diff --git a/assets/literate/07_logistic_reg_script.jl b/assets/literate/07_logistic_reg_script.jl new file mode 100644 index 00000000..298eeaae --- /dev/null +++ b/assets/literate/07_logistic_reg_script.jl @@ -0,0 +1,57 @@ +# This file was generated, do not modify it. + +using CairoMakie + +function logistic(x) + return 1 / (1 + exp(-x)) +end + +f, ax, l = lines(-10 .. 10, logistic; axis=(; xlabel=L"x", ylabel=L"\mathrm{Logistic}(x)")) +f +save(joinpath(@OUTPUT, "logistic.svg"), f); # hide + +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function logreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + + #likelihood + return y ~ arraydist(LazyArray(@~ BernoulliLogit.(α .+ X * β))) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/wells.csv" +wells = CSV.read(HTTP.get(url).body, DataFrame) +describe(wells) + +X = Matrix(select(wells, Not(:switch))) +y = wells[:, :switch] +model = logreg(X, y); + +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) + +using Chain + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end + +function logodds2prob(logodds::Float64) + return exp(logodds) / (1 + exp(logodds)) +end + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false) +end diff --git a/assets/literate/08_ordinal_reg.md b/assets/literate/08_ordinal_reg.md new file mode 100644 index 00000000..1b314e7a --- /dev/null +++ b/assets/literate/08_ordinal_reg.md @@ -0,0 +1,451 @@ + +# Bayesian Ordinal Regression + +Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The second is +**ordinal regression**. + +A ordinal regression behaves exactly like a linear model: it makes a prediction simply by computing a weighted +sum of the independent variables $\mathbf{X}$ by the estimated coefficients $\boldsymbol{\beta}$, +but now we have not only one intercept but several intercepts $\alpha_k$ for $k \in K$. + +We use ordinal regression when our dependent variable is ordinal. +That means it has different that have a "natural order"**. +Most important, the distance between values is not the same. +For example, imagine a pain score scale that goes from 1 to 10. +The distance between 1 and 2 is different from the distance 9 to 10. +Another example is opinion pools with their ubiquitous disagree-agree range +of plausible values. +These are also known as Likert scale variables. +The distance between "disagree" to "not agree or disagree" is different +than the distance between "agree" and "strongly agree". + +This assumption is what we call the "metric" assumption, +also called as the equidistant assumption. +Almost always when we model an ordinal dependent variable this assumption is violated. +Thus, we cannot blindly employ linear regression here. + +## How to deal with Ordered Discrete Dependent Variable? + +So, how we deal with ordered discrete responses in our dependent variable? +This is similar with the previous logistic regression approach. +We have to do a **non-linear transformation of the dependent variable**. + +### Cumulative Distribution Function (CDF) + +In the case of **ordinal regression**, we need to first transform the dependent variable into a **cumulative scale**. +We need to calculate the **cumulative distribution function** (CDF) of our dependent variable: + +$$P(Y \leq y) = \sum^y_{i=y_{\text{min}}} P(Y = i)$$ + +The **CDF is a monotonic increasing function** that depicts the **probability of our random variable $Y$ taking values less than a certain value**. +In our case, the discrete ordinal case, these values can be represented as positive integers ranging from 1 to the length of possible values. +For instance, a 6-categorical ordered response variable will have 6 possible values, and all their CDFs will lie between 0 and 1. +Furthermore, their sum will be 1; since it represents the total probability of the variable taking any of the possible values, i.e. 100%. + +### Log-cumulative-odds + +That is still not enough, we need to apply the **logit function to the CDF**: + +$$\mathrm{logit}(x) = \mathrm{logistic}^{-1}(x) = \ln\left(\frac{x}{1 -x}\right)$$ + +where $\ln$ is the natural logarithm function. + +The logit is the **inverse of the logistic transformation**, +it takes as a input any number between 0 and 1 +(where a probability is the perfect candidate) and outputs a real number, +which we call the **log-odds**. + +Since we are taking the log-odds of the CDF, we can call this complete transformation as +**log-odds of the CDF**, or **log-cumulative-odds**. + +### $K-1$ Intercepts + +Now, the next question is: what do we do with the log-cumulative-odds? + +**We need the log-cumulative-odds because it allows us to construct different intercepts for the possible values our ordinal dependent variable**. +We create an unique intercept for each possible outcome $k \in K$. + +Notice that the highest probable value of $Y$ will always have a log-cumulative-odds of $\infty$, since for $p=1$: + +$$\ln \frac{p}{1-p} = \ln \frac{1}{1-1} = \ln 0 = \infty$$ + +Thus, we only need $K-1$ intercepts for a $K$ possible dependent variables' response values. +These are known as **cut points**. + +Each intercept implies a CDF for each value $K$. +This allows us to **violate the equidistant assumption** absent in most ordinal variables. + +Each intercept implies a log-cumulative-odds for each $k \in K$. +We also need to **undo the cumulative nature of the $K-1$ intercepts**. +We can accomplish this by first converting a **log-cumulative-odds back to a cumulative probability**. +This is done by undoing the logit transformation and applying the logistic function: + +$$\mathrm{logit}^{-1}(x) = \mathrm{logistic}(x) = \frac{1}{1 + e^{-x}}$$ + +Then, finally, we can remove the **cumulative from "cumulative probability"** by +**subtraction of each of the $k$ cut points by their previous $k-1$ cut point**: + +$$P(Y=k) = P(Y \leq k) - P(Y \leq k-1)$$ + +where $Y$ is the dependent variable and $k \in K$ are the cut points for each intercept. + +Let me show you an example with some synthetic data. + +````julia:ex1 +using DataFrames +using CairoMakie +using AlgebraOfGraphics +using Distributions +using StatsFuns: logit +```` + +Here we have a discrete variable `x` with 6 possible ordered values as response. +The values range from 1 to 6 having probability, respectively: +10%, 15%, 33%, 25%, 10%, and 7%; +represented with the `probs` vector. +The underlying distribution is represented by a +`Categorical` distribution from `Distributions.jl`, +which takes a vector of probabilities as parameters (`probs`). + +For each value we are calculating: + +1. **P**robability **M**ass **F**unction with the `pdf` function +2. **C**umulative **D**istribution **F**unction with the `cdf` function +3. **Log-cumulative-odds** with the `logit` transformation of the CDF + +In the plot below there are 3 subplots: +- Upper corner: histogram of `x` +- Left lower corner: CDF of `x` +- Right lower corner: log-cumulative-odds of `x` + +````julia:ex2 +let + probs = [0.10, 0.15, 0.33, 0.25, 0.10, 0.07] + dist = Categorical(probs) + x = 1:length(probs) + x_pmf = pdf.(dist, x) + x_cdf = cdf.(dist, x) + x_logodds_cdf = logit.(x_cdf) + df = DataFrame(; x, x_pmf, x_cdf, x_logodds_cdf) + labels = ["CDF", "Log-cumulative-odds"] + f = Figure() + plt1 = data(df) * mapping(:x, :x_pmf) * visual(BarPlot) + plt2 = + data(df) * + mapping(:x, [:x_cdf, :x_logodds_cdf]; col=dims(1) => renamer(labels)) * + visual(ScatterLines) + axis = (; xticks=1:6) + draw!(f[1, 2:3], plt1; axis) + draw!(f[2, 1:4], plt2; axis, facet=(; linkyaxes=:none)) + f + save(joinpath(@OUTPUT, "logodds.svg"), f) # hide +end +```` + +\fig{logodds} +\center{*Ordinal Dependent Variable*} \\ + +As we can see, we have $K-1$ (in our case $6-1=5$) intercept values in log-cumulative-odds. +You can carly see that these values they violate the **equidistant assumption** +for metric response values. +The spacing between the cut points are not the same, they vary. + +## Adding Coefficients $\boldsymbol{\beta}$ + +Ok, the $K-1$ intercepts $\boldsymbol{\alpha}$ are done. +Now let's add coefficients to act as covariate effects in our ordinal regression model. + +We transformed everything into log-odds scale so that we can add effects +(coefficients multiplying a covariate) or basal rates (intercepts) together. +For each $k \in K-1$, we calculate: + +$$\phi_k = \alpha_k + \beta_i x_i$$ + +where $\alpha_k$ is the log-cumulative-odds for the $k \in K-1$ intercepts, +$\beta_i$ is the coefficient for the $i$th covariate $x$. +Finally, $\phi_k$ represents the linear predictor for the $k$th intercept. + +Observe that the coefficient $\beta$ is being added to a log-cumulative-odds, +such that, it will be expressed in a log-cumulative-odds also. + +We can express it in matrix form: + +$$\boldsymbol{\phi} = \boldsymbol{\alpha} + \mathbf{X} \cdot \boldsymbol{\beta}$$ + +where $\boldsymbol{\phi}$, $\boldsymbol{\alpha}$ and $\boldsymbol{\beta}$ are vectors +and $\mathbf{X}$ is the data matrix where each row is an observation and each column a covariate. + +This still obeys the ordered constraint on the dependent variable possible values. + +#### How to Interpret Coefficient $\boldsymbol{\beta}$? + +Now, suppose we have found our ideal value for our $\boldsymbol{\beta}$. +**How we would interpret our $\boldsymbol{\beta}$ estimated values?** + +First, to recap, $\boldsymbol{\beta}$ measures the strength of association between +the covariate $\mathbf{x}$ and dependent variable $\mathbf{y}$. +But, $\boldsymbol{\beta}$ is nested inside a non-linear transformation called +logistic function: + +$$\mathrm{logistic}(\boldsymbol{\beta}) = \frac{1}{1 + e^{-\boldsymbol{\beta}}}$$ + +So, our first step is to **undo the logistic function**. +There is a function that is called **logit function** that is the **inverse** +of the logistic function: + +$$\mathrm{logistic}^{-1}(x) = \mathrm{logit}(x) = \ln\left(\frac{x}{1 -x}\right)$$ + +where $\ln$ is the natural logarithm function. + +If we analyze closely the logit function we can find that +inside the $\ln$ there is a disguised odds in the $\frac{x}{1 -x}$. +Hence, our $\boldsymbol{\beta}$ can be interpreted as the +**logarithm of the odds**, or in short form: the **log-odds**. + +We already saw how odds, log-odds, and probability are related +in the previous logistic regression tutorial. +So you might want to go back there to get the full explanation. + +The log-odds are the key to interpret coefficient $\boldsymbol{\beta}$**. +Any positive value of $\beta$ means that there exists a positive association between $x$ and $y$, while any negative values indicate a negative association between $x$ and $y$. +Values close to 0 demonstrates the lack of association between $x$ and $y$. + +## Likelihood + +We have almost everything we need for our ordinal regression. +We are only missing a final touch. +Currently our **logistic function outputs a vector of probabilities** that sums to 1. + +All of the intercepts $\alpha_k$ along with the coefficients $\beta_i$ are in +log-cumulative-odds scale. +If we apply the logistic function to the linear predictors $\phi_k$ we get $K-1$ +probabilities: one for each $\phi_k$ + +We need a **likelihood that can handle a vector of probabilities and outputs a single +discrete value**. +The **categorical distribution is the perfect candidate**. + +## Bayesian Ordinal Regression + +Now we have all the components for our Bayesian ordinal regression specification: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Categorical}(\mathbf{p}) \\ +\mathbf{p} &= \text{Logistic}(\boldsymbol{\phi}) \\ +\boldsymbol{\phi} &= \boldsymbol{\alpha} + \mathbf{X} \cdot \boldsymbol{\beta} \\ +\alpha_1 &= \text{CDF}(y_1) \\ +\alpha_k &= \text{CDF}(y_k) - \text{CDF}(y_{k-1}) \quad \text{for} \quad 1 < k < K-1 \\ +\alpha_{K-1} &= 1 - \text{CDF}(y_{K-1}) +\end{aligned} +$$ + +where: + +* $\mathbf{y}$ -- ordered discrete dependent variable. +* $\mathbf{p}$ -- probability vector of length $K$. +* $K$ -- number of possible values $\mathbf{y}$ can take, i.e. number of ordered discrete values. +* $\boldsymbol{\phi}$ -- log-cumulative-odds, i.e. cut points considering the intercepts and covariates effect. +* $\alpha_k$ -- intercept in log-cumulative-odds for each $k \in K-1$. +* $\mathbf{X}$ -- covariate data matrix. +* $\boldsymbol{\beta}$ -- coefficient vector of the same length as the number of columns in $\mathbf{X}$. +* $\mathrm{logistic}$ -- logistic function. +* $\mathrm{CDF}$ -- **c**umulative **d**istribution **f**unction. + +What remains is to specify the model parameters' prior distributions: + +* Prior Distribution of $\boldsymbol{\alpha}$ -- Knowledge we possess regarding the model's intercepts. +* Prior Distribution of $\boldsymbol{\beta}$ -- Knowledge we possess regarding the model's independent variables' coefficients. + +Our goal is to instantiate an ordinal regression with the observed data ($\mathbf{y}$ and $\mathbf{X}$) and find the posterior +distribution of our model's parameters of interest ($\boldsymbol{\alpha}$ and $\boldsymbol{\beta}$). This means to find the full posterior +distribution of: + +$$ P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\boldsymbol{\alpha}, \boldsymbol{\beta} \mid \mathbf{y}) $$ + +Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error +parameter $\sigma$ in our ordinal regression. This is due to the Categorical distribution not having +a "scale" parameter such as the $\sigma$ parameter in the Gaussian/normal distribution. + +This is easily accomplished with Turing: + +````julia:ex3 +using Turing +using Bijectors +using LazyArrays +using LinearAlgebra +using Random: seed! +using Bijectors: transformed, OrderedBijector +seed!(123) +setprogress!(false) # hide + +@model function ordreg(X, y; predictors=size(X, 2), ncateg=maximum(y)) + #priors + cutpoints ~ transformed(filldist(TDist(3) * 5, ncateg - 1), OrderedBijector()) + β ~ filldist(TDist(3) * 2.5, predictors) + + #likelihood + return y ~ arraydist([OrderedLogistic(X[i, :] ⋅ β, cutpoints) for i in 1:length(y)]) +end; +```` + +First, let's deal with the new stuff in our model: the **`Bijectors.jl`'s `transformed` and `OrderedBijector`**. +As I've said in the [4. **How to use Turing**](/pages/04_Turing/), +Turing has a rich ecosystem of packages. +Bijectors implements a set of functions for transforming constrained random variables +(e.g. simplexes, intervals) to Euclidean space. +Here we are defining `cutpoints` as a `ncateg - 1` vector of Student-$t$ distributions +with mean 0, standard deviation 5 and degrees of freedom $\nu = 3$. +Remember that we only need $K-1$ cutpoints for all of our $K$ intercepts. +And we are also constraining it to be an ordered vector with `transformed(d, OrderedBijector)`, +such that for all cutpoints $c_i$ we have $c_1 < c_2 < ... c_{k-1}$. + +As before, we are giving $\boldsymbol{\beta}$ a very weakly informative priors of a +Student-$t$ distribution centered on 0 with variance 1 and degrees of freedom $\nu = 3$. +That wide-tailed $t$ distribution will cover all possible values for our coefficients. +Remember the Student-$t$ also has support over all the real number line $\in (-\infty, +\infty)$. Also the `filldist()` is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution. + +Finally, in the likelihood, +Turing's `arraydist()` function wraps an array of distributions returning a new distribution sampling from the individual +distributions. +And we use some indexing inside an array literal. + +## Example - Esoph + +For our example, I will use a famous dataset called `esoph` (Breslow & Day, 1980), +which is data from a case-control study of (o)esophageal cancer in Ille-et-Vilaine, France. +It has records for 88 age/alcohol/tobacco combinations: + +* `agegp`: Age group + * `1`: 25-34 years + * `2`: 35-44 years + * `3`: 45-54 years + * `4`: 55-64 years + * `5`: 65-74 years + * `6`: 75+ years +* `alcgp`: Alcohol consumption + * `1`: 0-39 g/day + * `2`: 40-79 g/day + * `3`: 80-119 g/day + * `4`: 120+ g/day +* `tobgp`: Tobacco consumption + * `1`: 0-9 g/day + * `2`: 10-19 g/day + * `3`: 20-29 g/day + * `4`: 30+ g/day +* `ncases`: Number of cases +* `ncontrols`: Number of controls + +Ok let's read our data with `CSV.jl` and output into a `DataFrame` from `DataFrames.jl`: + +````julia:ex4 +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/esoph.csv" +esoph = CSV.read(HTTP.get(url).body, DataFrame) +```` + +Now let's us instantiate our model with the data. +But here I need to do some data wrangling to create the data matrix `X`. +Specifically, I need to convert all of the categorical variables to integer values, +representing the ordinal values of both our independent and also dependent variables: + +````julia:ex5 +using CategoricalArrays + +DataFrames.transform!( + esoph, + :agegp => + x -> categorical( + x; levels=["25-34", "35-44", "45-54", "55-64", "65-74", "75+"], ordered=true + ), + :alcgp => + x -> categorical(x; levels=["0-39g/day", "40-79", "80-119", "120+"], ordered=true), + :tobgp => + x -> categorical(x; levels=["0-9g/day", "10-19", "20-29", "30+"], ordered=true); + renamecols=false, +) +DataFrames.transform!( + esoph, [:agegp, :alcgp, :tobgp] .=> ByRow(levelcode); renamecols=false +) + +X = Matrix(select(esoph, [:agegp, :alcgp])) +y = esoph[:, :tobgp] +model = ordreg(X, y); +```` + +And, finally, we will sample from the Turing model. We will be using the default `NUTS()` sampler with `1_000` samples, with +4 Markov chains using multiple threads `MCMCThreads()`: + +````julia:ex6 +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) +```` + +We had no problem with the Markov chains as all the `rhat` are well below `1.01` (or above `0.99`). +Note that the coefficients are in log-odds scale. They are the natural log of the odds[^logit], and odds is +defined as: + +$$ \text{odds} = \frac{p}{1-p} $$ + +where $p$ is a probability. So log-odds is defined as: + +$$ \log(\text{odds}) = \log \left( \frac{p}{1-x} \right) $$ + +So in order to get odds from a log-odds we must undo the log operation with a exponentiation. +This translates to: + +$$ \text{odds} = \exp ( \log ( \text{odds} )) $$ + +We can do this with a transformation in a `DataFrame` constructed from a `Chains` object: + +````julia:ex7 +using Chain + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end +```` + +Our interpretation of odds is the same as in betting games. Anything below 1 signals a unlikely probability that $y$ will be $1$. +And anything above 1 increases the probability of $y$ being $1$, while 1 itself is a neutral odds for $y$ being either $1$ or $0$. +Since I am not a gambling man, let's talk about probabilities. So I will create a function called `logodds2prob()` that converts +log-odds to probabilities: + +````julia:ex8 +function logodds2prob(logodds::Float64) + return exp(logodds) / (1 + exp(logodds)) +end + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false) +end +```` + +There you go, much better now. Let's analyze our results. +The `cutpoints` is the basal rate of the probability of our dependent variable +having values less than a certain value. +For example the cutpoint for having values less than `2` which its code represents +the tobacco consumption of 10-19 g/day has a median value of 20%. + +Now let's take a look at our coefficients +All coefficients whose 95% credible intervals captures the value $\frac{1}{2} = 0.5$ tells +that the effect on the propensity of tobacco consumption is inconclusive. +It is pretty much similar to a 95% credible interval that captures the 0 in +the linear regression coefficients. + +That's how you interpret 95% credible intervals from a `quantile()` output of a ordinal regression `Chains` +object converted from log-odds to probability. + +## Footnotes + +[^logit]: actually the [logit](https://en.wikipedia.org/wiki/Logit) function or the log-odds is the logarithm of the odds $\frac{p}{1-p}$ where $p$ is a probability. + +## References + +Breslow, N. E. & Day, N. E. (1980). **Statistical Methods in Cancer Research. Volume 1: The Analysis of Case-Control Studies**. IARC Lyon / Oxford University Press. + diff --git a/assets/literate/08_ordinal_reg_script.jl b/assets/literate/08_ordinal_reg_script.jl new file mode 100644 index 00000000..d913172f --- /dev/null +++ b/assets/literate/08_ordinal_reg_script.jl @@ -0,0 +1,95 @@ +# This file was generated, do not modify it. + +using DataFrames +using CairoMakie +using AlgebraOfGraphics +using Distributions +using StatsFuns: logit + +let + probs = [0.10, 0.15, 0.33, 0.25, 0.10, 0.07] + dist = Categorical(probs) + x = 1:length(probs) + x_pmf = pdf.(dist, x) + x_cdf = cdf.(dist, x) + x_logodds_cdf = logit.(x_cdf) + df = DataFrame(; x, x_pmf, x_cdf, x_logodds_cdf) + labels = ["CDF", "Log-cumulative-odds"] + f = Figure() + plt1 = data(df) * mapping(:x, :x_pmf) * visual(BarPlot) + plt2 = + data(df) * + mapping(:x, [:x_cdf, :x_logodds_cdf]; col=dims(1) => renamer(labels)) * + visual(ScatterLines) + axis = (; xticks=1:6) + draw!(f[1, 2:3], plt1; axis) + draw!(f[2, 1:4], plt2; axis, facet=(; linkyaxes=:none)) + f + save(joinpath(@OUTPUT, "logodds.svg"), f) # hide +end + +using Turing +using Bijectors +using LazyArrays +using LinearAlgebra +using Random: seed! +using Bijectors: transformed, OrderedBijector +seed!(123) +setprogress!(false) # hide + +@model function ordreg(X, y; predictors=size(X, 2), ncateg=maximum(y)) + #priors + cutpoints ~ transformed(filldist(TDist(3) * 5, ncateg - 1), OrderedBijector()) + β ~ filldist(TDist(3) * 2.5, predictors) + + #likelihood + return y ~ arraydist([OrderedLogistic(X[i, :] ⋅ β, cutpoints) for i in 1:length(y)]) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/esoph.csv" +esoph = CSV.read(HTTP.get(url).body, DataFrame) + +using CategoricalArrays + +DataFrames.transform!( + esoph, + :agegp => + x -> categorical( + x; levels=["25-34", "35-44", "45-54", "55-64", "65-74", "75+"], ordered=true + ), + :alcgp => + x -> categorical(x; levels=["0-39g/day", "40-79", "80-119", "120+"], ordered=true), + :tobgp => + x -> categorical(x; levels=["0-9g/day", "10-19", "20-29", "30+"], ordered=true); + renamecols=false, +) +DataFrames.transform!( + esoph, [:agegp, :alcgp, :tobgp] .=> ByRow(levelcode); renamecols=false +) + +X = Matrix(select(esoph, [:agegp, :alcgp])) +y = esoph[:, :tobgp] +model = ordreg(X, y); + +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) + +using Chain + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end + +function logodds2prob(logodds::Float64) + return exp(logodds) / (1 + exp(logodds)) +end + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false) +end diff --git a/assets/literate/09_count_reg.md b/assets/literate/09_count_reg.md new file mode 100644 index 00000000..7db81ff1 --- /dev/null +++ b/assets/literate/09_count_reg.md @@ -0,0 +1,402 @@ + +# Bayesian Regression with Count Data + +Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The third of these is +**regression with count data** (also called Poisson regression). + +A regression with count data behaves exactly like a linear model: it makes a prediction simply by computing a weighted +sum of the independent variables $\mathbf{X}$ by the estimated coefficients $\boldsymbol{\beta}$, plus an intercept +$\alpha$. However, instead of returning a continuous value $y$, such as linear regression, it returns the **natural +log** of $y$. + +We use regression with count data when our dependent variable is restricted to positive integers, *i.e.* $y \in \mathbb{Z}^+$. +See the figure below for a graphical intuition of the exponential function: + +````julia:ex1 +using CairoMakie + +f, ax, l = lines(-6 .. 6, exp; axis=(xlabel=L"x", ylabel=L"e^x")) +save(joinpath(@OUTPUT, "exponential.svg"), f); # hide +```` + +\fig{exponential} +\center{*Exponential Function*} \\ + +As we can see, the exponential function is basically a mapping of any real number to a +positive real number in the range between 0 and $+\infty$ (non-inclusive): + +$$ \text{Exponential}(x) = \{ \mathbb{R} \in [- \infty , + \infty] \} \to \{ \mathbb{R} \in [0, + \infty] \} $$ + +That is, the exponential function is the ideal candidate for when we need to convert something continuous without restrictions +to something continuous restricted to taking positive values only. That is why it is used when we need a model to have a +positive-only dependent variable. This is the case of a dependent variable for count data. + +## Comparison with Linear Regression + +Linear regression follows the following mathematical formulation: + +$$ \text{Linear} = \theta_0 + \theta_1 x_1 + \theta_2 x_2 + \ldots + \theta_n x_n $$ + +* $\theta$ - model parameters + * $\theta_0$ - intercept + * $\theta_1, \theta_2, \dots$ - independent variables $x_1, x_2, \dots$ coefficients +* $n$ - total number of independent variables + +Regression with count data would add the exponential function to the linear term: + +$$ \log(y) = \theta_0 \cdot \theta_1 x_1 \cdot \theta_2 x_2 \cdot \dots \cdot \theta_n x_n $$ + +which is the same as: + +$$ y = e^{(\theta_0 + \theta_1 x_1 + \theta_2 x_2 + \ldots + \theta_n x_n)} $$ + +## Bayesian Regression with Count Data + +We can model regression with count data in two ways. The first option with a **Poisson likelihood** function +and the second option with a **negative binomial likelihood** function. + +With the **Poisson likelihood** we model a discrete and positive dependent variable $y$ by assuming that a given number of +independent $y$ events will occur with a known constant average rate. + +In a **negative binomial likelihood**, model a discrete and positive dependent variable $y$ by assuming that a given number $n$ of +independent $y$ events will occur by asking a yes-no question for each $n$ with probability $p$ until $k$ success(es) is obtained. +Note that it becomes identical to the Poisson likelihood when at the limit of $k \to \infty$. This makes the negative binomial a +**robust option to replace a Poisson likelihood** to model phenomena with a *overdispersion* (excess expected variation in data). +This occurs due to the Poisson likelihood making an assumption that the dependent variable $y$ has the same mean and variance, +while in the negative binomial likelihood the mean and the variance do not need to be equal. + +### Using Poisson Likelihood + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Poisson}\left( e^{(\alpha + \mathbf{X} \cdot \boldsymbol{\beta})} \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) +\end{aligned} +$$ + +where: + +* $\mathbf{y}$ -- discrete and positive dependent variable. +* $e$ -- exponential. +* $\alpha$ -- intercept. +* $\boldsymbol{\beta}$ -- coefficient vector. +* $\mathbf{X}$ -- data matrix. + +As we can see, the linear predictor $\alpha + \mathbf{X} \cdot \boldsymbol{\beta}$ is the logarithm of the value of +$y$. So we need to apply the exponential function the values of the linear predictor: + +$$ +\begin{aligned} +\log(\mathbf{y}) &= \alpha + \mathbf{X} \cdot \boldsymbol{\beta} \\ +\mathbf{y} &= e^{\alpha \mathbf{X} \cdot \boldsymbol{\beta}} \\ +\mathbf{y} &= e^{\alpha} \cdot e^{\left( \mathbf{X} \cdot \boldsymbol{\beta} \right) } +\end{aligned} +$$ + +The intercept $\alpha$ and coefficients $\boldsymbol{\beta}$ are only interpretable after exponentiation. + +What remains is to specify the model parameters' prior distributions: + +* Prior Distribution of $\alpha$ -- Knowledge we possess regarding the model's intercept. +* Prior Distribution of $\boldsymbol{\beta}$ -- Knowledge we possess regarding the model's independent variables' coefficients. + +Our goal is to instantiate a regression with count data using the observed data ($\mathbf{y}$ and $\mathbf{X}$) and find the posterior +distribution of our model's parameters of interest ($\alpha$ and $\boldsymbol{\beta}$). This means to find the full posterior +distribution of: + +$$ P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta} \mid \mathbf{y}) $$ + +Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error +parameter $\sigma$ in our regression with count data. This is due to the Poisson not having +a "scale" parameter such as the $\sigma$ parameter in the Gaussian/normal distribution. + +This is easily accomplished with Turing: + +````julia:ex2 +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function poissonreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + + #likelihood + return y ~ arraydist(LazyArray(@~ LogPoisson.(α .+ X * β))) +end; +```` + +Here I am specifying very weakly informative priors: + +* $\alpha \sim \text{Normal}(0, 2.5)$ -- This means a normal distribution centered on 0 with a standard deviation of 2.5. That prior should with ease cover all possible values of $\alpha$. Remember that the normal distribution has support over all the real number line $\in (-\infty, +\infty)$. +* $\boldsymbol{\beta} \sim \text{Student-}t(0,1,3)$ -- The predictors all have a prior distribution of a Student-$t$ distribution centered on 0 with variance 1 and degrees of freedom $\nu = 3$. That wide-tailed $t$ distribution will cover all possible values for our coefficients. Remember the Student-$t$ also has support over all the real number line $\in (-\infty, +\infty)$. Also the `filldist()` is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution. + +Turing's `arraydist()` function wraps an array of distributions returning a new distribution sampling from the individual +distributions. And the LazyArrays' `LazyArray()` constructor wrap a lazy object that wraps a computation producing an array +to an array. Last, but not least, the macro `@~` creates a broadcast and is a nice short hand for the familiar dot `.` +broadcasting operator in Julia. This is an efficient way to tell Turing that our `y` vector is distributed lazily as a +`LogPoisson` broadcasted to `α` added to the product of the data matrix `X` and `β` coefficient vector. `LogPoisson` is +Turing's efficient distribution that already apply exponentiation to all the linear predictors. + +### Using Negative Binomial Likelihood + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Negative Binomial}\left( e^{(\alpha + \mathbf{X} \cdot \boldsymbol{\beta})}, \phi \right) \\ +\phi &\sim \text{Gamma}(0.01, 0.01) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) +\end{aligned} +$$ + +where: + +* $\mathbf{y}$ -- discrete and positive dependent variable. +* $e$ -- exponential. +* $\phi$ -- dispersion. +* $\phi^-$ -- reciprocal dispersion. +* $\alpha$ -- intercept. +* $\boldsymbol{\beta}$ -- coefficient vector. +* $\mathbf{X}$ -- data matrix. + +Note that when we compare with the Poisson model, we have a new parameter $\phi$ that parameterizes the negative binomial +likelihood. This parameter is the probability of successes $p$ of the negative binomial distribution and we generally use +a Gamma distribution as prior so that the inverse of $\phi$ which is $\phi^-$ fulfills the function of a "reciprocal dispersion" +parameter. Most of the time we use a weakly informative prior of the parameters shape $\alpha = 0.01$ and scale $\theta = 0.01$ +(Gelman et al., 2013; 2020). But you can also use $\phi^- \sim \text{Exponential}(1)$ as prior (McElreath, 2020). + +Here is what a $\text{Gamma}(0.01, 0.01)$ looks like: + +````julia:ex3 +using Distributions +f, ax, l = lines( + Gamma(0.01, 0.01); + linewidth=2, + axis=(xlabel=L"\phi", ylabel="Density", limits=(0, 0.03, nothing, nothing)), +) +save(joinpath(@OUTPUT, "gamma.svg"), f); # hide +```` + +\fig{gamma} +\center{*Gamma Distribution with $\alpha = 0.01$ and $\theta = 0.01$*} \\ + +In both likelihood options, what remains is to specify the model parameters' prior distributions: + +* Prior Distribution of $\alpha$ -- Knowledge we possess regarding the model's intercept. +* Prior Distribution of $\boldsymbol{\beta}$ -- Knowledge we possess regarding the model's independent variables' coefficients. + +Our goal is to instantiate a regression with count data using the observed data ($\mathbf{y}$ and $\mathbf{X}$) and find the posterior +distribution of our model's parameters of interest ($\alpha$ and $\boldsymbol{\beta}$). This means to find the full posterior +distribution of: + +$$ P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta} \mid \mathbf{y}) $$ + +Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error +parameter $\sigma$ in our regression with count data. This is due to neither the Poisson nor negative binomial distributions having +a "scale" parameter such as the $\sigma$ parameter in the Gaussian/normal distribution. + +#### Alternative Negative Binomial Parameterization + +One last thing before we get into the details of the negative binomial distribution is to consider an alternative parameterization. +Julia's `Distributions.jl` and, consequently, Turing's parameterization of the negative binomial distribution follows the following the [Wolfram reference](https://reference.wolfram.com/language/ref/NegativeBinomialDistribution.html): + +$$ +\text{Negative-Binomial}(k \mid r, p) \sim \frac{\Gamma(k+r)}{k! \Gamma(r)} p^r (1 - p)^k, \quad \text{for } k = 0, 1, 2, \ldots +$$ + +where: +* $k$ -- number of failures before the $r$th success in a sequence of independent Bernoulli trials +* $r$ -- number of successes +* $p$ -- probability of success in an individual Bernoulli trial + +This is not ideal for most of the modeling situations that we would employ the negative binomial distribution. +In particular, we want to have a parameterization that is more appropriate for count data. +What we need is the familiar **mean** (or location) and **variance** (or scale) parameterization. +If we look in [Stan's documentation for the `neg_binomial_2` function](https://mc-stan.org/docs/functions-reference/nbalt.html), we have the following two equations: + +$$ +\begin{aligned} +\mu &= \frac{r (1 - p)}{p} \label{negbin_mean} \\ +\mu + \frac{\mu^2}{\phi} &= \frac{r (1 - p)}{p^2} +\end{aligned} +$$ + +With a little bit of algebra, we can substitute the first equation of \eqref{negbin_mean} into the right hand side of the second equation and get the following: + +$$ +\begin{aligned} +\mu + \frac{\mu^2}{\phi} &= \frac{μ}{p} \\ +1 + \frac{\mu}{\phi} &= \frac{1}{p} \\ +p &= \frac{1}{\frac{1 + \mu}{\phi}} +\end{aligned} +$$ + +Then in \eqref{negbin_mean} we have: + +$$ +\begin{aligned} +\mu &= r \left(1 - \left( \frac{1}{\frac{1 + \mu}{\phi}} \right) \right) \cdot \left(1 + \frac{\mu}{\phi} \right) \\ +\mu &= r \left( \left(1 + \frac{\mu}{\phi} \right) - 1 \right) \\ +r &= \phi +\end{aligned} +$$ + +Hence, the resulting map is $\text{Negative-Binomial}(\mu, \phi) \equiv \text{Negative-Binomial} \left( r = \phi, p = \frac{1}{\frac{1 + \mu}{\phi}} \right)$. +I would like to point out that this implementation was done by [Tor Fjelde](https://github.com/torfjelde) in a [COVID-19 model with the code available in GitHub](https://github.com/cambridge-mlg/Covid19). +So we can use this parameterization in our negative binomial regression model. +But first, we need to define an alternative negative binomial distribution function: + +````julia:ex4 +function NegativeBinomial2(μ, ϕ) + p = 1 / (1 + μ / ϕ) + p = p > 0 ? p : 1e-4 # numerical stability + r = ϕ + + return NegativeBinomial(r, p) +end +```` + +Now we create our Turing model with the alternative `NegBinomial2` parameterization: + +````julia:ex5 +@model function negbinreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + ϕ⁻ ~ Gamma(0.01, 0.01) + ϕ = 1 / ϕ⁻ + + #likelihood + return y ~ arraydist(LazyArray(@~ NegativeBinomial2.(exp.(α .+ X * β), ϕ))) +end; +```` + +Here I am also specifying very weakly informative priors: + +* $\alpha \sim \text{Normal}(0, 2.5)$ -- This means a normal distribution centered on 0 with a standard deviation of 2.5. That prior should with ease cover all possible values of $\alpha$. Remember that the normal distribution has support over all the real number line $\in (-\infty, +\infty)$. +* $\boldsymbol{\beta} \sim \text{Student-}t(0,1,3)$ -- The predictors all have a prior distribution of a Student-$t$ distribution centered on 0 with variance 1 and degrees of freedom $\nu = 3$. That wide-tailed $t$ distribution will cover all possible values for our coefficients. Remember the Student-$t$ also has support over all the real number line $\in (-\infty, +\infty)$. Also the `filldist()` is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution. +* $\phi \sim \text{Exponential}(1)$ -- overdispersion parameter of the negative binomial distribution. + +Turing's `arraydist()` function wraps an array of distributions returning a new distribution sampling from the individual +distributions. And the LazyArrays' `LazyArray()` constructor wrap a lazy object that wraps a computation producing an array +to an array. Last, but not least, the macro `@~` creates a broadcast and is a nice short hand for the familiar dot `.` +broadcasting operator in Julia. This is an efficient way to tell Turing that our `y` vector is distributed lazily as a +`NegativeBinomial2` broadcasted to `α` added to the product of the data matrix `X` and `β` coefficient vector. +Note that `NegativeBinomial2` does not apply exponentiation so we had to include the `exp.()` broadcasted function to all the linear predictors. + +## Example - Roaches Extermination + +For our example, I will use a famous dataset called `roaches` (Gelman & Hill, 2007), which is data on the efficacy of a +pest management system at reducing the number of roaches in urban apartments. +It has 262 observations and the following variables: + +* `y` -- number of roaches caught. +* `roach1` -- pretreatment number of roaches. +* `treatment` -- binary/dummy (0 or 1) for treatment indicator. +* `senior` -- binary/dummy (0 or 1) for only elderly residents in building. +* `exposure2` -- number of days for which the roach traps were used + +Ok let's read our data with `CSV.jl` and output into a `DataFrame` from `DataFrames.jl`: + +````julia:ex6 +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/roaches.csv" +roaches = CSV.read(HTTP.get(url).body, DataFrame) +describe(roaches) +```` + +As you can see from the `describe()` output the average number of roaches caught by the pest management system is around 26 roaches. +The average number of roaches pretreatment is around 42 roaches (oh boy...). +30% of the buildings has only elderly residents and 60% of the buildings received a treatment by the pest management system. +Also note that the traps were set in general for only 1 day and it ranges from 0.2 days (almost 5 hours) to 4.3 days +(which is approximate 4 days and 7 hours). + +### Poisson Regression + +Let's first run the Poisson regression. +First, we instantiate our model with the data: + +````julia:ex7 +X = Matrix(select(roaches, Not(:y))) +y = roaches[:, :y] +model_poisson = poissonreg(X, y); +```` + +And, finally, we will sample from the Turing model. We will be using the default `NUTS()` sampler with `1_000` samples, with +4 Markov chains using multiple threads `MCMCThreads()`: + +````julia:ex8 +chain_poisson = sample(model_poisson, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain_poisson) +```` + +We had no problem with the Markov chains as all the `rhat` are well below `1.01` (or above `0.99`). +Note that the coefficients are in log scale. So we need to apply the exponential function to them. +We can do this with a transformation in a `DataFrame` constructed from a `Chains` object: + +````julia:ex9 +using Chain + +@chain quantile(chain_poisson) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end +```` + +Let's analyze our results. The intercept `α` is the basal number of roaches caught `y` and has +a median value of 19.4 roaches caught. The remaining 95% credible intervals for the `β`s can be interpreted as follows: + +* `β[1]` -- first column of `X`, `roach1`, has 95% credible interval 1.01 to 1.01. This means that each **increase** in one unit of `roach1` is related to an increase of 1% more roaches caught. +* `β[2]` -- second column of `X`, `treatment`, has 95% credible interval 0.57 to 0.63. This means that if an apartment was treated with the pest management system then we expect an **decrease** of around 40% roaches caught. +* `β[3]` -- third column of `X`, `senior`, has a 95% credible interval from 0.64 to 0.73. This means that if an apartment building has only elderly residents then we expect an **decrease** of around 30% roaches caught. +* `β[4]` -- fourth column of `X`, `exposure2`, has a 95% credible interval from 1.09 to 1.26. Each increase in one day for the exposure of traps in an apartment we expect an **increase** of between 9% to 26% roaches caught. + +That's how you interpret 95% credible intervals from a `quantile()` output of a regression with count data `Chains` +object converted from a log scale. + +### Negative Binomial Regression + +Let's now run the negative binomial regression. + +````julia:ex10 +model_negbin = negbinreg(X, y); +```` + +We will also default `NUTS()` sampler with `1_000` samples, with 4 Markov chains using multiple threads `MCMCThreads()`: + +````julia:ex11 +seed!(111) # hide +chain_negbin = sample(model_negbin, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain_negbin) +```` + +We had no problem with the Markov chains as all the `rhat` are well below `1.01` (or above `0.99`). +Note that the coefficients are in log scale. So we need to also apply the exponential function as we did before. + +````julia:ex12 +@chain quantile(chain_negbin) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end +```` + +Our results show much more uncertainty in the coefficients than in the Poisson regression. +So it might be best to use the Poisson regression in the `roaches` dataset. + +## References + +Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press. + +Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian Data Analysis. Chapman and Hall/CRC. + +Gelman, A., Hill, J., & Vehtari, A. (2020). Regression and other stories. Cambridge University Press. + +McElreath, R. (2020). *Statistical rethinking: A Bayesian course with examples in R and Stan*. CRC press. + diff --git a/assets/literate/09_count_reg_script.jl b/assets/literate/09_count_reg_script.jl new file mode 100644 index 00000000..af55b5f4 --- /dev/null +++ b/assets/literate/09_count_reg_script.jl @@ -0,0 +1,81 @@ +# This file was generated, do not modify it. + +using CairoMakie + +f, ax, l = lines(-6 .. 6, exp; axis=(xlabel=L"x", ylabel=L"e^x")) +save(joinpath(@OUTPUT, "exponential.svg"), f); # hide + +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function poissonreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + + #likelihood + return y ~ arraydist(LazyArray(@~ LogPoisson.(α .+ X * β))) +end; + +using Distributions +f, ax, l = lines( + Gamma(0.01, 0.01); + linewidth=2, + axis=(xlabel=L"\phi", ylabel="Density", limits=(0, 0.03, nothing, nothing)), +) +save(joinpath(@OUTPUT, "gamma.svg"), f); # hide + +function NegativeBinomial2(μ, ϕ) + p = 1 / (1 + μ / ϕ) + p = p > 0 ? p : 1e-4 # numerical stability + r = ϕ + + return NegativeBinomial(r, p) +end + +@model function negbinreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + ϕ⁻ ~ Gamma(0.01, 0.01) + ϕ = 1 / ϕ⁻ + + #likelihood + return y ~ arraydist(LazyArray(@~ NegativeBinomial2.(exp.(α .+ X * β), ϕ))) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/roaches.csv" +roaches = CSV.read(HTTP.get(url).body, DataFrame) +describe(roaches) + +X = Matrix(select(roaches, Not(:y))) +y = roaches[:, :y] +model_poisson = poissonreg(X, y); + +chain_poisson = sample(model_poisson, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain_poisson) + +using Chain + +@chain quantile(chain_poisson) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end + +model_negbin = negbinreg(X, y); + +seed!(111) # hide +chain_negbin = sample(model_negbin, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain_negbin) + +@chain quantile(chain_negbin) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end diff --git a/assets/literate/10_robust_reg.md b/assets/literate/10_robust_reg.md new file mode 100644 index 00000000..e8adeca3 --- /dev/null +++ b/assets/literate/10_robust_reg.md @@ -0,0 +1,244 @@ + +# Robust Bayesian Regression + +Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The fourth of these is +**robust regression**. + +A regression with count data behaves exactly like a linear model: it makes a prediction simply by computing a weighted +sum of the independent variables $\mathbf{X}$ by the estimated coefficients $\boldsymbol{\beta}$, plus an intercept +$\alpha$. However, instead of using a Gaussian/normal likelihood function, it uses a **Student-$t$ likelihood function**. + +We use robust regression in the same context as linear regression: our dependent variable is continuous. But robust regression +allows us to **better handle outliers** in our data. + +Before we dive in the nuts and bolts of robust regression let's remember the Gaussian/normal curve that has a bell shape +(figure below). It does not have a "fat tail" (or sometimes known as "long tail"). In other words, the observations are +not far from the mean. When we use this distribution as a likelihood function in the Bayesian models, we force that all +estimates must be conditioned into a normal distribution of the dependent variable. If there are many outliers in the +data (observations quite far from the mean), this causes the estimates of the independent variables' coefficients to +be unstable. This is because the normal distribution cannot contemplate observations that are very spread away from the +mean without having to change the mean's position (or location). In other words, the bell curve needs to "shift" to be able +to contemplate outliers, thus making the inference unstable. + +````julia:ex1 +using CairoMakie +using Distributions + +f, ax, l = lines(-4 .. 4, Normal(0, 1); linewidth=5, axis=(; xlabel=L"x", ylabel="Density")) +save(joinpath(@OUTPUT, "normal.svg"), f); # hide +```` + +\fig{normal} +\center{*Normal with $\mu=0$ and $\sigma = 1$*} \\ + +So we need a more "malleable" distribution as a likelihood function. A distribution that is more robust to outliers. +A distribution similar to Normal but that has "fatter" (or "longer") tails to precisely contemplate observations very +far from the average without having to "shift" the mean's position (or location). For that we have the Student-$t$ distribution. +See the figure below to remember its shape. + +````julia:ex2 +f, ax, l = lines(-4 .. 4, TDist(2); linewidth=5, axis=(xlabel=L"x", ylabel="Density")) +save(joinpath(@OUTPUT, "tdist.svg"), f); # hide +```` + +\fig{tdist} +\center{*Student-$t$ with $\nu = 2$*} \\ + +## Comparison Between Normal vs Student-$t$ + +Take a look at the tails in the comparison below: + +````julia:ex3 +f, ax, l = lines( + -4 .. 4, + Normal(0, 1); + linewidth=5, + label="Normal", + axis=(; xlabel=L"x", ylabel="Density"), +) +lines!(ax, -4 .. 4, TDist(2); linewidth=5, label="Student") +axislegend(ax) +save(joinpath(@OUTPUT, "comparison_normal_student.svg"), f); # hide +```` + +\fig{comparison_normal_student} +\center{*Comparison between Normal and Student-$t$ Distributions*} \\ + +## Bayesian Robust Regression + +The standard approach for modeling a continuous dependent variable is with a Gaussian/normal likelihood function. +This implies that the model error, $\sigma$ of the Gaussian/normal likelihood function is distributed as a normal distribution: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Normal}\left( \alpha + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +From a Bayesian point of view, there is nothing special about Gaussian/normal likelihood function +It is just a probabilistic distribution specified in a model. We can make the model more robust +by using a Student-$t$ distribution as a likelihood function. This implies that the model error, +$\sigma$ does not follow a normal distribution, instead it follows a Student-$t$ distribution: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Student}\left( \nu, \alpha + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ +\nu &\sim \text{Log-Normal}(2, 1) \\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +Here are some differences: + +1. Student-$t$ likelihood function requires one additional parameter: $\nu$, degrees of freedom. These control how "fat" (or "long") the tails will be. Values of $\nu> 20$ forces the Student-$t$ distribution to practically become a normal distribution. +2. There is nothing special about $\nu$. It is just another parameter for the model to estimate. So just specify a prior on it. In this case, I am using a Log-Normal distribution with mean 2 and standard deviation 1. + +Note that there is also nothing special about the priors of the $\boldsymbol{\beta}$ coefficients or the intercept $\alpha$. +We could very well also specify other distributions as priors or even make the model even more robust to outliers by +specifying priors as Student-$t$ distributions with degrees of freedom $\nu = 3$: + +$$ +\begin{aligned} +\alpha &\sim \text{Student}(\nu_\alpha = 3, \mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Student}(\nu_{\boldsymbol{\beta}} = 3, \mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) +\end{aligned} +$$ + +Our goal is to instantiate a regression with count data using the observed data ($\mathbf{y}$ and $\mathbf{X}$) and find the posterior +distribution of our model's parameters of interest ($\alpha$ and $\boldsymbol{\beta}$). This means to find the full posterior +distribution of: + +$$ P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta}, \sigma \mid \mathbf{y}) $$ + +This is easily accomplished with Turing: + +````julia:ex4 +using Turing +using Statistics: mean, std +using StatsBase: mad +using Random: seed! +seed!(123) +seed!(456) # hide +setprogress!(false) # hide + +@model function robustreg(X, y; predictors=size(X, 2)) + #priors + α ~ LocationScale(median(y), 2.5 * mad(y), TDist(3)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + ν ~ LogNormal(2, 1) + + #likelihood + return y ~ arraydist(LocationScale.(α .+ X * β, σ, TDist.(ν))) +end; +```` + +Here I am specifying very weakly informative priors: + +* $\alpha \sim \text{Student-}t(\operatorname{median}(\mathbf{y}), 2.5 \cdot \operatorname{MAD}(\mathbf{y}), \nu_{\alpha} = 3)$ -- This means a Student-$t$ distribution with degrees of freedom `ν = 3` centered on `y`'s median with variance 2.5 times the mean absolute deviation (MAD) of `y`. That prior should with ease cover all possible values of $\alpha$. Remember that the Student-$t$ distribution has support over all the real number line $\in (-\infty, +\infty)$. The `LocationScale()` Turing's function adds location and scale parameters to distributions that doesn't have it. This is the case with the `TDist()` distribution which only takes the `ν` degrees of of freedom as parameter. +* $\boldsymbol{\beta} \sim \text{Student-}t(0,1,\nu_{\boldsymbol{\beta}})$ -- The predictors all have a prior distribution of a Student-$t$ distribution with degrees of freedom `ν = 3` centered on 0 with variance 1 and degrees of freedom $\nu_{\boldsymbol{\beta}}$. That wide-tailed $t$ distribution will cover all possible values for our coefficients. Remember the Student-$t$ also has support over all the real number line $\in (-\infty, +\infty)$. Also the `filldist()` is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution. +* $\sigma \sim \text{Exponential}(1)$ -- A wide-tailed-positive-only distribution perfectly suited for our model's error. + +Turing's `arraydist()` function wraps an array of distributions returning a new distribution sampling from the individual +distributions. It creates a broadcast and is a nice short hand for the familiar dot `.` broadcasting operator in Julia. +By specifying that `y` vector is "broadcasted distributed" as a `LocationScale` broadcasted to mean (location parameter) +`α` added to the product of the data matrix `X` and `β` coefficient vector along with a variance (scale parameter) `σ`. +To conclude, we place inside the `LocationScale` a broadcasted `TDist` with `ν` degrees of freedom parameter. + +## Example - Duncan's Prestige + +For our example, I will use a famous dataset called `duncan` (Duncan, 1961), which is data from occupation's prestige filled with +outliers. +It has 45 observations and the following variables: + +* `profession`: name of the profession. +* `type`: type of occupation. A qualitative variable: + * `prof` - professional or management. + * `wc` - white-collar. + * `bc` - blue-collar. +* `income`: percentage of people in the occupation earning over U\$ 3,500 per year in 1950 (more or less U\$ 36,000 in 2017). +* `education`: percentage of people in the occupation who had a high school diploma in 1949 (which, being cynical, we can say is somewhat equivalent to a PhD degree in 2017). +* `prestige`: percentage of respondents in the survey who classified their occupation as at least "good" with respect to prestige. + +Ok let's read our data with `CSV.jl` and output into a `DataFrame` from `DataFrames.jl`: + +````julia:ex5 +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/duncan.csv" +duncan = CSV.read(HTTP.get(url).body, DataFrame) +describe(duncan) +```` + +As you can see from the `describe()` output the average occupation's percentage of respondents who classified their occupation +as at least "good" with respect to prestige is around 41%. But `prestige` variable is very dispersed and actually has a bimodal +distribution: + +````julia:ex6 +f = Figure() +plt = data(duncan) * mapping(:prestige) * AlgebraOfGraphics.density() +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "prestige_density.svg"), f); # hide +```` + +\fig{prestige_density} +\center{*Density Plot of `prestige`*} \\ + +Besides that, the mean `prestige` per `type` shows us where the source of variation might come from: + +````julia:ex7 +gdf = groupby(duncan, :type) +f = Figure() +plt = + data(combine(gdf, :prestige => mean)) * mapping(:type, :prestige_mean) * visual(BarPlot) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "prestige_per_type.svg"), f); # hide +```` + +\fig{prestige_per_type} +\center{*Mean `prestige` per `type`*} \\ + +Now let's us instantiate our model with the data: + +````julia:ex8 +X = Matrix(select(duncan, [:income, :education])) +y = duncan[:, :prestige] +model = robustreg(X, y); +```` + +And, finally, we will sample from the Turing model. We will be using the default `NUTS()` sampler with `1_000` samples, with +4 Markov chains using multiple threads `MCMCThreads()`: + +````julia:ex9 +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) +```` + +We had no problem with the Markov chains as all the `rhat` are well below `1.01` (or above `0.99`). +Also note that all degrees of freedom parameters, the `ν` stuff, have been estimated with mean around 3 to 5, +which indeed signals that our model needed fat tails to make a robust inference. +Our model has an error `σ` of around 7. So it estimates occupation's prestige ±7. The intercept `α` is the basal occupation's +prestige value. So each occupation has -7±7 prestige before we add the coefficients multiplied by the occupations' independent variables. +And from our coefficients $\boldsymbol{\beta}$, we can see that the `quantile()` tells us the uncertainty around their +estimates: + +````julia:ex10 +quantile(chain) +```` + +* `β[1]` -- first column of `X`, `income`, has 95% credible interval from 0.55 to 0.96. This means that an increase of U\$ 1,000 in occupations' annual income is associated with an increase in roughly 0.5 to 1.0 in occupation's prestige. +* `β[2]` -- second column of `X`, `education`, has a 95% credible interval from 0.29 to 0.61. So we expect that an increase of 1% in occupations' percentage of respondents who had a high school diploma increases occupations' prestige roughly 0.3 to 0.6. + +That's how you interpret 95% credible intervals from a `quantile()` output of a robust regression `Chains` object. + +## References + +Duncan, O. D. (1961). A socioeconomic index for all occupations. Class: Critical Concepts, 1, 388–426. + diff --git a/assets/literate/10_robust_reg_script.jl b/assets/literate/10_robust_reg_script.jl new file mode 100644 index 00000000..58a06afa --- /dev/null +++ b/assets/literate/10_robust_reg_script.jl @@ -0,0 +1,69 @@ +# This file was generated, do not modify it. + +using CairoMakie +using Distributions + +f, ax, l = lines(-4 .. 4, Normal(0, 1); linewidth=5, axis=(; xlabel=L"x", ylabel="Density")) +save(joinpath(@OUTPUT, "normal.svg"), f); # hide + +f, ax, l = lines(-4 .. 4, TDist(2); linewidth=5, axis=(xlabel=L"x", ylabel="Density")) +save(joinpath(@OUTPUT, "tdist.svg"), f); # hide + +f, ax, l = lines( + -4 .. 4, + Normal(0, 1); + linewidth=5, + label="Normal", + axis=(; xlabel=L"x", ylabel="Density"), +) +lines!(ax, -4 .. 4, TDist(2); linewidth=5, label="Student") +axislegend(ax) +save(joinpath(@OUTPUT, "comparison_normal_student.svg"), f); # hide + +using Turing +using Statistics: mean, std +using StatsBase: mad +using Random: seed! +seed!(123) +seed!(456) # hide +setprogress!(false) # hide + +@model function robustreg(X, y; predictors=size(X, 2)) + #priors + α ~ LocationScale(median(y), 2.5 * mad(y), TDist(3)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + ν ~ LogNormal(2, 1) + + #likelihood + return y ~ arraydist(LocationScale.(α .+ X * β, σ, TDist.(ν))) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/duncan.csv" +duncan = CSV.read(HTTP.get(url).body, DataFrame) +describe(duncan) + +f = Figure() +plt = data(duncan) * mapping(:prestige) * AlgebraOfGraphics.density() +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "prestige_density.svg"), f); # hide + +gdf = groupby(duncan, :type) +f = Figure() +plt = + data(combine(gdf, :prestige => mean)) * mapping(:type, :prestige_mean) * visual(BarPlot) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "prestige_per_type.svg"), f); # hide + +X = Matrix(select(duncan, [:income, :education])) +y = duncan[:, :prestige] +model = robustreg(X, y); + +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) + +quantile(chain) diff --git a/assets/literate/11_multilevel_models.md b/assets/literate/11_multilevel_models.md new file mode 100644 index 00000000..3b154ace --- /dev/null +++ b/assets/literate/11_multilevel_models.md @@ -0,0 +1,424 @@ + +# Multilevel Models (a.k.a. Hierarchical Models) + +Bayesian hierarchical models (also called multilevel models) are a statistical model written at *multiple* levels +(hierarchical form) that estimates the parameters of the posterior distribution using the Bayesian approach. +The sub-models combine to form the hierarchical model, and Bayes' theorem is used to integrate them with the +observed data and to account for all the uncertainty that is present. The result of this integration is the +posterior distribution, also known as an updated probability estimate, as additional evidence of the likelihood +function is integrated together with the prior distribution of the parameters. + +Hierarchical modeling is used when information is available at several different levels of observation units. +The hierarchical form of analysis and organization helps to understand multiparameter problems and also plays +an important role in the development of computational strategies. + +Hierarchical models are mathematical statements that involve several parameters, so that the estimates of some parameters +depend significantly on the values of other parameters. The figure below shows a hierarchical model in which there is a +$\phi$ hyperparameter that parameterizes the parameters $\theta_1, \theta_2, \dots, \theta_N$ that are finally used to +infer the posterior density of some variable of interest $\mathbf{y} = y_1, y_2, \dots, y_N$. + +![Bayesian Workflow](/pages/images/hierarchical.png) + +\center{*Hierarchical Model*} \\ + +## When to use Multilevel Models? + +Multilevel models are particularly suitable for research projects where participant data is organized at more than one level, *i.e.* nested data. +Units of analysis are usually individuals (at a lower level) that are nested in contextual/aggregate units (at a higher level). +An example is when we are measuring the performance of individuals and we have additional information about belonging to different +groups such as sex, age group, hierarchical level, educational level or housing status. + +There is a main assumption that cannot be violated in multilevel models which is **exchangeability** (de Finetti, 1974; Nau, 2001). +Yes, this is the same assumption that we discussed in [2. **What is Bayesian Statistics?**](/pages/2_bayes_stats/). +This assumption assumes that groups are exchangeable. The figure below shows a graphical representation of the exchangeability. +The groups shown as "cups" that contain observations shown as "balls". If in the model's inferences, this assumption is violated, +then multilevel models are not appropriate. This means that, since there is no theoretical justification to support exchangeability, +the inferences of the multilevel model are not robust and the model can suffer from several pathologies and should not be used for any +scientific or applied analysis. + +![Bayesian Workflow](/pages/images/exchangeability-1.png) +![Bayesian Workflow](/pages/images/exchangeability-2.png) + +\center{*Exchangeability -- Images from [Michael Betancourt](https://betanalpha.github.io/)*} \\ + +## Hyperpriors + +As the priors of the parameters are sampled from another prior of the hyperparameter (upper-level's parameter), +which are called **hyperpriors**. This makes one group's estimates help the model to better estimate the other groups +by providing more **robust and stable estimates**. + +We call the global parameters as **population effects** (or population-level effects, also sometimes called fixed effects) +and the parameters of each group as **group effects** (or group-level effects, also sometimes called random effects). +That is why multilevel models are also known as mixed models in which we have both fixed effects and random effects. + +## Three Approaches to Multilevel Models + +Multilevel models generally fall into three approaches: + +1. **Random-intercept model**: each group receives a **different intercept** in addition to the global intercept. +2. **Random-slope model**: each group receives **different coefficients** for each (or a subset of) independent variable(s) in addition to a global intercept. +3. **Random-intercept-slope model**: each group receives **both a different intercept and different coefficients** for each independent variable in addition to a global intercept. +4. **Correlated-Random-intercept-slope model**: each group receives **both a different intercept and different coefficients** for each independent variable in addition to a global intercept; while also taking into account the **correlation/covariance amongst intercept/coefficients**. + +### Random-Intercept Model + +The first approach is the **random-intercept model** in which we specify a different intercept for each group, +in addition to the global intercept. These group-level intercepts are sampled from a hyperprior. + +To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. +Mathematically a Bayesian multilevel random-slope linear regression model is: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Normal}\left( \alpha + \alpha_j + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\alpha_j &\sim \text{Normal}(0, \tau) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ +\tau &\sim \text{Cauchy}^+(0, \psi_{\alpha})\\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +The priors on the global intercept $\alpha$, global coefficients $\boldsymbol{\beta}$ and error $\sigma$, along with +the Gaussian/normal likelihood on $\mathbf{y}$ are the same as in the linear regression model. +But now we have **new parameters**. The first are the **group intercepts** prior $\alpha_j$ that denotes that every group +$1, 2, \dots, J$ has its own intercept sampled from a normal distribution centered on 0 with a standard deviation $\psi_\alpha$. +This group intercept is added to the linear predictor inside the Gaussian/normal likelihood function. The **group intercepts' standard +deviation** $\tau$ have a hyperprior (being a prior of a prior) which is sampled from a positive-constrained Cauchy distribution (a special +case of the Student-$t$ distribution with degrees of freedom $\nu = 1$) with mean 0 and standard deviation $\sigma_\alpha$. +This makes the group-level intercept's dispersions being sampled from the same parameter $\tau$ which allows the model +to use information from one group intercept to infer robust information regarding another group's intercept dispersion and so on. + +This is easily accomplished with Turing: + +````julia:ex1 +using Turing +using LinearAlgebra +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function varying_intercept( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + αⱼ ~ filldist(Normal(0, τ), n_gr) # group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ αⱼ[idx] + return y ~ MvNormal(ŷ, σ^2 * I) +end; +```` + +### Random-Slope Model + +The second approach is the **random-slope model** in which we specify a different slope for each group, +in addition to the global intercept. These group-level slopes are sampled from a hyperprior. + +To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. +Mathematically a Bayesian multilevel random-slope linear regression model is: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Normal}\left( \alpha + \mathbf{X} \cdot \boldsymbol{\beta}_j \cdot \boldsymbol{\tau}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta}_j &\sim \text{Normal}(0, 1) \\ +\boldsymbol{\tau} &\sim \text{Cauchy}^+(0, \psi_{\boldsymbol{\beta}})\\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +Here we have a similar situation from before with the same hyperprior, but now it is a hyperprior for the the group coefficients' +standard deviation prior: $\boldsymbol{\beta}_j$. +This makes the group-level coefficients's dispersions being sampled from the same parameter $\tau$ which allows the model +to use information from one group coefficients to infer robust information regarding another group's coefficients dispersion and so on. + +In Turing we can accomplish this as: + +````julia:ex2 +@model function varying_slope(X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random slopes + #usually requires thoughtful specification + τ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs + βⱼ ~ filldist(Normal(0, 1), predictors, n_gr) # group-level standard normal slopes + + #likelihood + ŷ = α .+ X * βⱼ * τ + return y ~ MvNormal(ŷ, σ^2 * I) +end; +```` + +### Random-Intercept-Slope Model + +The third approach is the **random-intercept-slope model** in which we specify a different intercept +and slope for each group, in addition to the global intercept. +These group-level intercepts and slopes are sampled from hyperpriors. + +To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. +Mathematically a Bayesian multilevel random-intercept-slope linear regression model is: + +$$ +\begin{aligned} +\mathbf{y} &\sim \text{Normal}\left( \alpha + \alpha_j + \mathbf{X} \cdot \boldsymbol{\beta}_j \cdot \boldsymbol{\tau}_{\boldsymbol{\beta}}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\alpha_j &\sim \text{Normal}(0, \tau_{\alpha}) \\ +\boldsymbol{\beta}_j &\sim \text{Normal}(0, 1) \\ +\tau_{\alpha} &\sim \text{Cauchy}^+(0, \psi_{\alpha})\\ +\boldsymbol{\tau}_{\boldsymbol{\beta}} &\sim \text{Cauchy}^+(0, \psi_{\boldsymbol{\beta}})\\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +Here we have a similar situation from before with the same hyperpriors, but now we fused both random-intercept +and random-slope together. + +In Turing we can accomplish this as: + +````julia:ex3 +@model function varying_intercept_slope( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts and slopes + #usually requires thoughtful specification + τₐ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + τᵦ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs + αⱼ ~ filldist(Normal(0, τₐ), n_gr) # group-level intercepts + βⱼ ~ filldist(Normal(0, 1), predictors, n_gr) # group-level standard normal slopes + + #likelihood + ŷ = α .+ αⱼ[idx] .+ X * βⱼ * τᵦ + return y ~ MvNormal(ŷ, σ^2 * I) +end; +```` + +In all of the models so far, +we are using the `MvNormal` construction where we specify both +a vector of means (first positional argument) +and a covariance matrix (second positional argument). +Regarding the covariance matrix `σ^2 * I`, +it uses the model's errors `σ`, here parameterized as a standard deviation, +squares it to produce a variance parameterization, +and multiplies by `I`, which is Julia's `LinearAlgebra` standard module implementation +to represent an identity matrix of any size. + +### Correlated-Random-Intercept-Slope Model + +The third approach is the **correlated-random-intercept-slope model** in which we specify a different intercept +and slope for each group, in addition to the global intercept. +These group-level intercepts and slopes are sampled from hyperpriors. +Finally, we also model the correlation/covariance amongst the intercepts and slopes. +We assume that they are not independent anymore, rather they are correlated. + +In order to model the correlation between parameters, +we need a prior on correlation/covariance matrices. +There are several ways to define priors on covariance matrices. +In some ancient time, "Bayesian elders" used distributions such as +the [Wishart distribution](https://en.wikipedia.org/wiki/Wishart_distribution) +and the [Inverse Wishart distribution](https://en.wikipedia.org/wiki/Inverse-Wishart_distribution). +However, priors on covariance matrices are not that intuitive, +and can be hard to translate into real-world scenarios. +That's why I much prefer to construct my covariances matrices from a +correlation matrix and a vector of standard deviations. +Remember that every covariance matrix $\boldsymbol{\Sigma}$ can be decomposed into: + +$$\boldsymbol{\Sigma}=\text{diag}_\text{matrix}(\boldsymbol{\tau}) \cdot \boldsymbol{\Omega} \cdot \text{diag}_\text{matrix}(\boldsymbol{\tau})$$ + +where $\boldsymbol{\Omega}$ is a correlation matrix with +$1$s in the diagonal and the off-diagonal elements between -1 e 1 $\rho \in (-1, 1)$. +$\boldsymbol{\tau}$ is a vector composed of the variables' standard deviation from +$\boldsymbol{\Sigma}$ (is is the $\boldsymbol{\Sigma}$'s diagonal). + +These are much more intuitive and easy to work with, +specially if you are not working together with some "Bayesian elders". +This approach leaves us with the need to specify a prior on correlation matrices. +Luckily, we have the [LKJ distribution](https://en.wikipedia.org/wiki/Lewandowski-Kurowicka-Joe_distribution): +distribution for positive definite matrices with unit diagonals +(exactly what a correlation matrix is). + +To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. +Mathematically a Bayesian multilevel correlated-random-intercept-slope linear regression model is: + +$$ +\begin{aligned} +\mathbf{y} & \sim \text{Normal}(\mathbf{X} \boldsymbol{\beta}_{j}, \sigma) \\ +\boldsymbol{\beta}_j & \sim \text{Multivariate Normal}(\boldsymbol{\mu}_j, \boldsymbol{\Sigma}) +\quad \text{for}\quad j \in \{ 1, \dots, J \} \\ +\boldsymbol{\Sigma} & \sim \text{LKJ}(\eta) \\ +\sigma & \sim \text{Exponential}(\lambda_\sigma) +\end{aligned} +$$ + +We don't have any $\alpha$s. +If we want a varying intercept, we just insert a column filled with $1$s in the data matrix $\mathbf{X}$. +Mathematically, this makes the column behave like an "identity" variable +(because the number $1$ in the multiplication operation $1 \cdot \beta$ is the identity element. +It maps $x \to x$ keeping the value of $x$ intact) and, consequently, +we can interpret the column's coefficient as the model's intercept. + +In Turing we can accomplish this as: + +````julia:ex4 +using PDMats + +@model function correlated_varying_intercept_slope( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + Ω ~ LKJCholesky(predictors, 2.0) # Cholesky decomposition correlation matrix + σ ~ Exponential(std(y)) + + #prior for variance of random correlated intercepts and slopes + #usually requires thoughtful specification + τ ~ filldist(truncated(Cauchy(0, 2); lower=0), predictors) # group-level SDs + γ ~ filldist(Normal(0, 5), predictors, n_gr) # matrix of group coefficients + + #reconstruct Σ from Ω and τ + Σ_L = Diagonal(τ) * Ω.L + Σ = PDMat(Cholesky(Σ_L + 1e-6 * I)) # numerical instability + #reconstruct β from Σ and γ + β = Σ * γ + + #likelihood + return y ~ arraydist([Normal(X[i, :] ⋅ β[:, idx[i]], σ) for i in 1:length(y)]) +end; +```` + +In the `correlated_varying_intercept_slope` model, +we are using the efficient [Cholesky decomposition](https://en.wikipedia.org/wiki/Cholesky_decomposition) +version of the LKJ prior with `LKJCholesky`. +We put priors on all group coefficients `γ` and reconstruct both the +covariance matrix `Σ` and the coefficient matrix `β`. + +## Example - Cheese Ratings + +For our example, I will use a famous dataset called `cheese` (Boatwright, McCulloch & Rossi, 1999), which is data from +cheese ratings. A group of 10 rural and 10 urban raters rated 4 types of different cheeses (A, B, C and D) in two samples. +So we have $4 \cdot 20 \cdot 2 = 160$ observations and 4 variables: + +* `cheese`: type of cheese from `A` to `D` +* `rater`: id of the rater from `1` to `10` +* `background`: type of rater, either `rural` or `urban` +* `y`: rating of the cheese + +OK, let's read our data with `CSV.jl` and output into a `DataFrame` from `DataFrames.jl`: + +````julia:ex5 +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv" +cheese = CSV.read(HTTP.get(url).body, DataFrame) +describe(cheese) +```` + +As you can see from the `describe()` output, the mean cheese ratings is around 70 ranging from 33 to 91. + +In order to prepare the data for Turing, I will convert the `String`s in variables `cheese` and `background` +to `Int`s. Regarding `cheese`, I will create 4 dummy variables one for each cheese type; and `background` will be +converted to integer data taking two values: one for each background type. My intent is to model `background` +as a group both for intercept and coefficients. +Take a look at how the data will look like for the first 5 observations: + +````julia:ex6 +for c in unique(cheese[:, :cheese]) + cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0) +end + +cheese[:, :background_int] = map(cheese[:, :background]) do b + if b == "rural" + 1 + elseif b == "urban" + 2 + else + missing + end +end + +first(cheese, 5) +```` + +Now let's us instantiate our model with the data. +Here, I will specify a vector of `Int`s named `idx` to represent the different observations' +group memberships. This will be used by Turing when we index a parameter with the `idx`, +*e.g.* `αⱼ[idx]`. + +````julia:ex7 +X = Matrix(select(cheese, Between(:cheese_A, :cheese_D))); +y = cheese[:, :y]; +idx = cheese[:, :background_int]; +```` + +The first model is the `varying_intercept`: + +````julia:ex8 +model_intercept = varying_intercept(X, idx, y) +chain_intercept = sample(model_intercept, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_intercept))) +```` + +Here we can see that the model has a population-level intercept `α` along with population-level coefficients `β`s for each `cheese` +dummy variable. But notice that we have also group-level intercepts for each of the groups `αⱼ`s. +Specifically, `αⱼ[1]` are the rural raters and `αⱼ[2]` are the urban raters. + +Now let's go to the second model, `varying_slope`: + +````julia:ex9 +model_slope = varying_slope(X, idx, y) +chain_slope = sample(model_slope, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_slope))) +```` + +Here we can see that the model has still a population-level intercept `α`. But now our population-level +coefficients `β`s are replaced by group-level coefficients `βⱼ`s along with their standard deviation `τᵦ`s. +Specifically `βⱼ`'s first index denotes the 4 dummy `cheese` variables' and the second index are the group +membership. So, for example `βⱼ[1,1]` is the coefficient for `cheese_A` and rural raters (group 1). + +Now let's go to the third model, `varying_intercept_slope`: + +````julia:ex10 +model_intercept_slope = varying_intercept_slope(X, idx, y) +chain_intercept_slope = sample(model_intercept_slope, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_intercept_slope))) +```` + +Now we have fused the previous model in one. We still have a population-level intercept `α`. But now +we have in the same model group-level intercepts for each of the groups `αⱼ`s and group-level along with their standard +deviation `τₐ`. We also have the coefficients `βⱼ`s with their standard deviation `τᵦ`s. +The parameters are interpreted exactly as the previous cases. + +Now let's go to the fourth model, `varying_intercept_slope`. +Here we are going to add a columns of into the data matrix `X`: + +````julia:ex11 +X_correlated = hcat(fill(1, size(X, 1)), X) +model_correlated = correlated_varying_intercept_slope(X_correlated, idx, y) +chain_correlated = sample(model_correlated, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_correlated))) +```` + +We can see that we also get all of the Cholesky factors of the correlation +matrix `Ω` which we can transform back into a correlation matrix by doing +`Ω = Ω.L * Ω.L'`. + +## References + +Boatwright, P., McCulloch, R., & Rossi, P. (1999). Account-level modeling for trade promotion: An application of a constrained parameter hierarchical model. Journal of the American Statistical Association, 94(448), 1063–1073. + +de Finetti, B. (1974). Theory of Probability (Volume 1). New York: John Wiley & Sons. + +Nau, R. F. (2001). De Finetti was Right: Probability Does Not Exist. Theory and Decision, 51(2), 89–124. https://doi.org/10.1023/A:1015525808214 + diff --git a/assets/literate/11_multilevel_models_script.jl b/assets/literate/11_multilevel_models_script.jl new file mode 100644 index 00000000..7056344a --- /dev/null +++ b/assets/literate/11_multilevel_models_script.jl @@ -0,0 +1,126 @@ +# This file was generated, do not modify it. + +using Turing +using LinearAlgebra +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function varying_intercept( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + αⱼ ~ filldist(Normal(0, τ), n_gr) # group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ αⱼ[idx] + return y ~ MvNormal(ŷ, σ^2 * I) +end; + +@model function varying_slope(X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random slopes + #usually requires thoughtful specification + τ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs + βⱼ ~ filldist(Normal(0, 1), predictors, n_gr) # group-level standard normal slopes + + #likelihood + ŷ = α .+ X * βⱼ * τ + return y ~ MvNormal(ŷ, σ^2 * I) +end; + +@model function varying_intercept_slope( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts and slopes + #usually requires thoughtful specification + τₐ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + τᵦ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs + αⱼ ~ filldist(Normal(0, τₐ), n_gr) # group-level intercepts + βⱼ ~ filldist(Normal(0, 1), predictors, n_gr) # group-level standard normal slopes + + #likelihood + ŷ = α .+ αⱼ[idx] .+ X * βⱼ * τᵦ + return y ~ MvNormal(ŷ, σ^2 * I) +end; + +using PDMats + +@model function correlated_varying_intercept_slope( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + Ω ~ LKJCholesky(predictors, 2.0) # Cholesky decomposition correlation matrix + σ ~ Exponential(std(y)) + + #prior for variance of random correlated intercepts and slopes + #usually requires thoughtful specification + τ ~ filldist(truncated(Cauchy(0, 2); lower=0), predictors) # group-level SDs + γ ~ filldist(Normal(0, 5), predictors, n_gr) # matrix of group coefficients + + #reconstruct Σ from Ω and τ + Σ_L = Diagonal(τ) * Ω.L + Σ = PDMat(Cholesky(Σ_L + 1e-6 * I)) # numerical instability + #reconstruct β from Σ and γ + β = Σ * γ + + #likelihood + return y ~ arraydist([Normal(X[i, :] ⋅ β[:, idx[i]], σ) for i in 1:length(y)]) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv" +cheese = CSV.read(HTTP.get(url).body, DataFrame) +describe(cheese) + +for c in unique(cheese[:, :cheese]) + cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0) +end + +cheese[:, :background_int] = map(cheese[:, :background]) do b + if b == "rural" + 1 + elseif b == "urban" + 2 + else + missing + end +end + +first(cheese, 5) + +X = Matrix(select(cheese, Between(:cheese_A, :cheese_D))); +y = cheese[:, :y]; +idx = cheese[:, :background_int]; + +model_intercept = varying_intercept(X, idx, y) +chain_intercept = sample(model_intercept, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_intercept))) + +model_slope = varying_slope(X, idx, y) +chain_slope = sample(model_slope, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_slope))) + +model_intercept_slope = varying_intercept_slope(X, idx, y) +chain_intercept_slope = sample(model_intercept_slope, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_intercept_slope))) + +X_correlated = hcat(fill(1, size(X, 1)), X) +model_correlated = correlated_varying_intercept_slope(X_correlated, idx, y) +chain_correlated = sample(model_correlated, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_correlated))) diff --git a/assets/literate/12_Turing_tricks.md b/assets/literate/12_Turing_tricks.md new file mode 100644 index 00000000..2e817895 --- /dev/null +++ b/assets/literate/12_Turing_tricks.md @@ -0,0 +1,311 @@ + +# Computational Tricks with Turing \\ (Non-Centered Parametrization \\ and QR Decomposition) + +There are some computational tricks that we can employ with Turing. +I will cover here two computational tricks: + +1. **QR Decomposition** +2. **Non-Centered Parametrization** + +## QR Decomposition + +Back in "Linear Algebra 101" we've learned that any matrix (even rectangular ones) can be factored +into the product of two matrices: + +* $\mathbf{Q}$: an orthogonal matrix (its columns are orthogonal unit vectors meaning $\mathbf{Q}^T = \mathbf{Q}^{-1})$. +* $\mathbf{R}$: an upper triangular matrix. + +This is commonly known as the [**QR Decomposition**](https://en.wikipedia.org/wiki/QR_decomposition): + +$$ \mathbf{A} = \mathbf{Q} \cdot \mathbf{R} $$ + +Let me show you an example with a random matrix $\mathbf{A} \in \mathbb{R}^{3 \times 2}$: + +````julia:ex1 +A = rand(3, 2) +```` + +Now let's factor `A` using `LinearAlgebra`'s `qr()` function: + +````julia:ex2 +using LinearAlgebra: qr, I +Q, R = qr(A) +```` + +Notice that `qr()` produced a tuple containing two matrices `Q` and `R`. `Q` is a 3x3 orthogonal matrix. +And `R` is a 2x2 upper triangular matrix. +So that $\mathbf{Q}^T = \mathbf{Q}^{-1}$ (the transpose is equal the inverse): + +````julia:ex3 +Matrix(Q') ≈ Matrix(Q^-1) +```` + +Also note that $\mathbf{Q}^T \cdot \mathbf{Q}^{-1} = \mathbf{I}$ (identity matrix): + +````julia:ex4 +Q' * Q ≈ I(3) +```` + +This is nice. But what can we do with QR decomposition? It can speed up Turing's sampling by +a huge factor while also decorrelating the columns of $\mathbf{X}$, *i.e.* the independent variables. +The orthogonal nature of QR decomposition alters the posterior's topology and makes it easier +for HMC or other MCMC samplers to explore it. Let's see how fast we can get with QR decomposition. +First, let's go back to the `kidiq` example in [6. **Bayesian Linear Regression**](/pages/6_linear_reg/): + +````julia:ex5 +using Turing +using LinearAlgebra: I +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function linreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + + #likelihood + return y ~ MvNormal(α .+ X * β, σ^2 * I) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv" +kidiq = CSV.read(HTTP.get(url).body, DataFrame) +X = Matrix(select(kidiq, Not(:kid_score))) +y = kidiq[:, :kid_score] +model = linreg(X, y) +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +```` + +See the wall duration in Turing's `chain`: for me it took around 24 seconds. + +Now let's us incorporate QR decomposition in the linear regression model. +Here, I will use the "thin" instead of the "fat" QR, which scales the $\mathbf{Q}$ and $\mathbf{R}$ +matrices by a factor of $\sqrt{n-1}$ where $n$ is the number of rows of $\mathbf{X}$. +In practice it is better to implement the thin QR decomposition, which is to be preferred to the fat QR decomposition. +It is numerically more stable. Mathematically, the thin QR decomposition is: + +$$ +\begin{aligned} +x &= \mathbf{Q}^* \mathbf{R}^* \\ +\mathbf{Q}^* &= \mathbf{Q} \cdot \sqrt{n - 1} \\ +\mathbf{R}^* &= \frac{1}{\sqrt{n - 1}} \cdot \mathbf{R}\\ +\boldsymbol{\mu} +&= \alpha + \mathbf{X} \cdot \boldsymbol{\beta} + \sigma +\\ +&= \alpha + \mathbf{Q}^* \cdot \mathbf{R}^* \cdot \boldsymbol{\beta} + \sigma +\\ +&= \alpha + \mathbf{Q}^* \cdot (\mathbf{R}^* \cdot \boldsymbol{\beta}) + \sigma +\\ +&= \alpha + \mathbf{Q}^* \cdot \widetilde{\boldsymbol{\beta}} + \sigma +\\ +\end{aligned} +$$ + +Then we can recover the original $\boldsymbol{\beta}$ with: + +$$ \boldsymbol{\beta} = \mathbf{R}^{*-1} \cdot \widetilde{\boldsymbol{\beta}} $$ + +In Turing, a model with QR decomposition would be the same `linreg` but with a +different `X` matrix supplied, since it is a data transformation. First, we +decompose your model data `X` into `Q` and `R`: + +````julia:ex6 +Q, R = qr(X) +Q_ast = Matrix(Q) * sqrt(size(X, 1) - 1) +R_ast = R / sqrt(size(X, 1) - 1); +```` + +Then, we instantiate a model with `Q` instead of `X` and sample as you would: + +````julia:ex7 +model_qr = linreg(Q_ast, y) +chain_qr = sample(model_qr, NUTS(1_000, 0.65), MCMCThreads(), 1_000, 4) +```` + +See the wall duration in Turing's `chain_qr`: for me it took around 5 seconds. Much faster than +the regular `linreg`. +Now we have to reconstruct our $\boldsymbol{\beta}$s: + +````julia:ex8 +betas = mapslices( + x -> R_ast^-1 * x, chain_qr[:, namesingroup(chain_qr, :β), :].value.data; dims=[2] +) +chain_beta = setrange( + Chains(betas, ["real_β[$i]" for i in 1:size(Q_ast, 2)]), 1_001:1:2_000 +) +chain_qr_reconstructed = hcat(chain_beta, chain_qr) +```` + +## Non-Centered Parametrization + +Now let's us explore **Non-Centered Parametrization** (NCP). This is useful when the posterior's +topology is very difficult to explore as has regions where HMC sampler has to +change the step size $L$ and the $\epsilon$ factor. This is I've showed one of the most infamous +case in [5. **Markov Chain Monte Carlo (MCMC)**](/pages/5_MCMC/): Neal's Funnel (Neal, 2003): + +````julia:ex9 +using CairoMakie +using Distributions +funnel_y = rand(Normal(0, 3), 10_000) +funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2) + +f, ax, s = scatter( + funnel_x, + funnel_y; + color=(:steelblue, 0.3), + axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)), +) +save(joinpath(@OUTPUT, "funnel.svg"), f); # hide +```` + +\fig{funnel} +\center{*Neal's Funnel*} \\ + +Here we see that in upper part of the funnel HMC has to take few steps $L$ and be more liberal with +the $\epsilon$ factor. But, the opposite is in the lower part of the funnel: way more steps $L$ and be +more conservative with the $\epsilon$ factor. + +To see the devil's funnel (how it is known in some Bayesian circles) in action, let's code it in Turing and then sample: + +````julia:ex10 +@model function funnel() + y ~ Normal(0, 3) + return x ~ Normal(0, exp(y / 2)) +end + +chain_funnel = sample(funnel(), NUTS(), MCMCThreads(), 1_000, 4) +```` + +Wow, take a look at those `rhat` values... That sucks: all are above `1.01` even with 4 parallel chains with 1,000 +iterations! + +How do we deal with that? We **reparametrize**! Note that we can add two normal distributions in the following manner: + +$$ \text{Normal}(\mu, \sigma) = \text{Standard Normal} \cdot \sigma + \mu $$ + +where the standard normal is the normal with mean $\mu = 0$ and standard deviation $\sigma = 1$. +This is why is called Non-Centered Parametrization because we "decouple" the parameters and +reconstruct them before. + +````julia:ex11 +@model function ncp_funnel() + x̃ ~ Normal() + ỹ ~ Normal() + y = 3.0 * ỹ # implies y ~ Normal(0, 3) + return x = exp(y / 2) * x̃ # implies x ~ Normal(0, exp(y / 2)) +end + +chain_ncp_funnel = sample(ncp_funnel(), NUTS(), MCMCThreads(), 1_000, 4) +```` + +Much better now: all `rhat` are well below `1.01` (or below `0.99`). + +How we would implement this a real-world model in Turing? Let's go back to the `cheese` random-intercept model +in [11. **Multilevel Models (a.k.a. Hierarchical Models)**](/pages/11_multilevel_models/). Here was the +approach that we took, also known as Centered Parametrization (CP): + +````julia:ex12 +@model function varying_intercept( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + αⱼ ~ filldist(Normal(0, τ), n_gr) # CP group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ αⱼ[idx] + return y ~ MvNormal(ŷ, σ^2 * I) +end; +```` + +To perform a Non-Centered Parametrization (NCP) in this model we do as following: + +````julia:ex13 +@model function varying_intercept_ncp( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + zⱼ ~ filldist(Normal(0, 1), n_gr) # NCP group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ zⱼ[idx] .* τ + return y ~ MvNormal(ŷ, σ^2 * I) +end; +```` + +Here we are using a NCP with the `zⱼ`s following a standard normal and we reconstruct the +group-level intercepts by multiplying the `zⱼ`s by `τ`. Since the original `αⱼ`s had a prior +centered on 0 with standard deviation `τ`, we only have to use the multiplication by `τ` +to get back the `αⱼ`s. + +Now let's see how NCP compares to the CP. First, let's redo our CP hierarchical model: + +````julia:ex14 +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv" +cheese = CSV.read(HTTP.get(url).body, DataFrame) + +for c in unique(cheese[:, :cheese]) + cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0) +end + +cheese[:, :background_int] = map(cheese[:, :background]) do b + if b == "rural" + 1 + elseif b == "urban" + 2 + else + missing + end +end + +X = Matrix(select(cheese, Between(:cheese_A, :cheese_D))); +y = cheese[:, :y]; +idx = cheese[:, :background_int]; + +model_cp = varying_intercept(X, idx, y) +chain_cp = sample(model_cp, NUTS(), MCMCThreads(), 1_000, 4) +```` + +Now let's do the NCP hierarchical model: + +````julia:ex15 +model_ncp = varying_intercept_ncp(X, idx, y) +chain_ncp = sample(model_ncp, NUTS(), MCMCThreads(), 1_000, 4) +```` + +Notice that some models are better off with a standard Centered Parametrization (as is our `cheese` case here). +While others are better off with a Non-Centered Parametrization. But now you know how to apply both parametrizations +in Turing. Before we conclude, we need to recover our original `αⱼ`s. We can do this by multiplying `zⱼ[idx] .* τ`: + +````julia:ex16 +τ = summarystats(chain_ncp)[:τ, :mean] +αⱼ = mapslices( + x -> x * τ, chain_ncp[:, namesingroup(chain_ncp, :zⱼ), :].value.data; dims=[2] +) +chain_ncp_reconstructed = hcat( + MCMCChains.resetrange(chain_ncp), Chains(αⱼ, ["αⱼ[$i]" for i in 1:length(unique(idx))]) +) +```` + +## References + +Neal, Radford M. (2003). Slice Sampling. The Annals of Statistics, 31(3), 705–741. Retrieved from https://www.jstor.org/stable/3448413 + diff --git a/assets/literate/12_Turing_tricks_script.jl b/assets/literate/12_Turing_tricks_script.jl new file mode 100644 index 00000000..bb491e89 --- /dev/null +++ b/assets/literate/12_Turing_tricks_script.jl @@ -0,0 +1,152 @@ +# This file was generated, do not modify it. + +A = rand(3, 2) + +using LinearAlgebra: qr, I +Q, R = qr(A) + +Matrix(Q') ≈ Matrix(Q^-1) + +Q' * Q ≈ I(3) + +using Turing +using LinearAlgebra: I +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function linreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + + #likelihood + return y ~ MvNormal(α .+ X * β, σ^2 * I) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv" +kidiq = CSV.read(HTTP.get(url).body, DataFrame) +X = Matrix(select(kidiq, Not(:kid_score))) +y = kidiq[:, :kid_score] +model = linreg(X, y) +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) + +Q, R = qr(X) +Q_ast = Matrix(Q) * sqrt(size(X, 1) - 1) +R_ast = R / sqrt(size(X, 1) - 1); + +model_qr = linreg(Q_ast, y) +chain_qr = sample(model_qr, NUTS(1_000, 0.65), MCMCThreads(), 1_000, 4) + +betas = mapslices( + x -> R_ast^-1 * x, chain_qr[:, namesingroup(chain_qr, :β), :].value.data; dims=[2] +) +chain_beta = setrange( + Chains(betas, ["real_β[$i]" for i in 1:size(Q_ast, 2)]), 1_001:1:2_000 +) +chain_qr_reconstructed = hcat(chain_beta, chain_qr) + +using CairoMakie +using Distributions +funnel_y = rand(Normal(0, 3), 10_000) +funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2) + +f, ax, s = scatter( + funnel_x, + funnel_y; + color=(:steelblue, 0.3), + axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)), +) +save(joinpath(@OUTPUT, "funnel.svg"), f); # hide + +@model function funnel() + y ~ Normal(0, 3) + return x ~ Normal(0, exp(y / 2)) +end + +chain_funnel = sample(funnel(), NUTS(), MCMCThreads(), 1_000, 4) + +@model function ncp_funnel() + x̃ ~ Normal() + ỹ ~ Normal() + y = 3.0 * ỹ # implies y ~ Normal(0, 3) + return x = exp(y / 2) * x̃ # implies x ~ Normal(0, exp(y / 2)) +end + +chain_ncp_funnel = sample(ncp_funnel(), NUTS(), MCMCThreads(), 1_000, 4) + +@model function varying_intercept( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + αⱼ ~ filldist(Normal(0, τ), n_gr) # CP group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ αⱼ[idx] + return y ~ MvNormal(ŷ, σ^2 * I) +end; + +@model function varying_intercept_ncp( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + zⱼ ~ filldist(Normal(0, 1), n_gr) # NCP group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ zⱼ[idx] .* τ + return y ~ MvNormal(ŷ, σ^2 * I) +end; + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv" +cheese = CSV.read(HTTP.get(url).body, DataFrame) + +for c in unique(cheese[:, :cheese]) + cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0) +end + +cheese[:, :background_int] = map(cheese[:, :background]) do b + if b == "rural" + 1 + elseif b == "urban" + 2 + else + missing + end +end + +X = Matrix(select(cheese, Between(:cheese_A, :cheese_D))); +y = cheese[:, :y]; +idx = cheese[:, :background_int]; + +model_cp = varying_intercept(X, idx, y) +chain_cp = sample(model_cp, NUTS(), MCMCThreads(), 1_000, 4) + +model_ncp = varying_intercept_ncp(X, idx, y) +chain_ncp = sample(model_ncp, NUTS(), MCMCThreads(), 1_000, 4) + +τ = summarystats(chain_ncp)[:τ, :mean] +αⱼ = mapslices( + x -> x * τ, chain_ncp[:, namesingroup(chain_ncp, :zⱼ), :].value.data; dims=[2] +) +chain_ncp_reconstructed = hcat( + MCMCChains.resetrange(chain_ncp), Chains(αⱼ, ["αⱼ[$i]" for i in 1:length(unique(idx))]) +) diff --git a/assets/literate/13_epi_models.md b/assets/literate/13_epi_models.md new file mode 100644 index 00000000..bc0684b0 --- /dev/null +++ b/assets/literate/13_epi_models.md @@ -0,0 +1,236 @@ + +# Bonus: Epidemiological Models using ODE Solvers in Turing + +Ok, now this is something that really makes me very excited with Julia's +ecosystem. If you want to use an Ordinary Differential Equation solver +in your Turing model, you don't need to code it from scratch. You've just +**borrow a pre-made one** from [`DifferentialEquations.jl`](https://diffeq.sciml.ai/dev/). +This is what makes Julia so great. We can use functions and types +defined in other packages into another package and it will probably work either +straight out of the bat or without much effort! + +For this tutorial I'll be using Brazil's COVID data from the [Media Consortium](https://brasil.io/covid19/). +For reproducibility, we'll restrict the data to the year of 2020: + +````julia:ex1 +using Downloads +using DataFrames +using CSV +using Chain +using Dates + +url = "https://data.brasil.io/dataset/covid19/caso_full.csv.gz" +file = Downloads.download(url) +df = DataFrame(CSV.File(file)) +br = @chain df begin + filter( + [:date, :city] => + (date, city) -> + date < Dates.Date("2021-01-01") && + date > Dates.Date("2020-04-01") && + ismissing(city), + _, + ) + groupby(:date) + combine( + [ + :estimated_population_2019, + :last_available_confirmed_per_100k_inhabitants, + :last_available_deaths, + :new_confirmed, + :new_deaths, + ] .=> + sum .=> [ + :estimated_population_2019, + :last_available_confirmed_per_100k_inhabitants, + :last_available_deaths, + :new_confirmed, + :new_deaths, + ], + ) +end; +```` + +Let's take a look in the first observations + +````julia:ex2 +first(br, 5) +```` + +Also the bottom rows + +````julia:ex3 +last(br, 5) +```` + +Here is a plot of the data: + +````julia:ex4 +using AlgebraOfGraphics +using CairoMakie +f = Figure() +plt = data(br) * mapping(:date => L"t", :new_confirmed => "infected daily") * visual(Lines) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "infected.svg"), f); # hide +```` + +\fig{infected} +\center{*Infected in Brazil during COVID in 2020*} \\ + +The Susceptible-Infected-Recovered (SIR) (Grinsztajn, Semenova, Margossian & Riou, 2021) model splits +the population in three time-dependent compartments: +the susceptible, the infected (and infectious), and the +recovered (and not infectious) compartments. When a susceptible individual comes into contact with an infectious individual, +the former can become infected for some time, and then recover and become immune. The dynamics can be summarized in a system ODEs: + +![SIR Model](/pages/images/SIR.png) + +\center{*Susceptible-Infected-Recovered (SIR) model*} \\ + +$$ +\begin{aligned} +\frac{dS}{dt} &= -\beta S \frac{I}{N}\\ +\frac{dI}{dt} &= \beta S \frac{I}{N} - \gamma I \\ +\frac{dR}{dt} &= \gamma I +\end{aligned} +$$ + +where: +* $S(t)$ -- the number of people susceptible to becoming infected (no immunity) +* $I(t)$ -- the number of people currently infected (and infectious) +* $R(t)$ -- the number of recovered people (we assume they remain immune indefinitely) +* $\beta$ -- the constant rate of infectious contact between people +* $\gamma$ -- constant recovery rate of infected individuals + +## How to code an ODE in Julia? + +It's very easy: + +1. Use [`DifferentialEquations.jl`](https://diffeq.sciml.ai/) +2. Create a ODE function +3. Choose: + * Initial Conditions -- $u_0$ + * Parameters -- $p$ + * Time Span -- $t$ + * *Optional* -- [Solver](https://diffeq.sciml.ai/stable/solvers/ode_solve/) or leave blank for auto + +PS: If you like SIR models checkout [`epirecipes/sir-julia`](https://github.com/epirecipes/sir-julia) + +The following function provides the derivatives of the model, which it changes in-place. +State variables and parameters are unpacked from `u` and `p`; this incurs a slight performance hit, +but makes the equations much easier to read. + +````julia:ex5 +using DifferentialEquations + +function sir_ode!(du, u, p, t) + (S, I, R) = u + (β, γ) = p + N = S + I + R + infection = β * I * S / N + recovery = γ * I + @inbounds begin + du[1] = -infection # Susceptible + du[2] = infection - recovery # Infected + du[3] = recovery # Recovered + end + return nothing +end; +```` + +This is what the infection would look with some fixed `β` and `γ` +in a timespan of 100 days starting from day one with 1,167 infected (Brazil in April 2020): + +````julia:ex6 +i₀ = first(br[:, :new_confirmed]) +N = maximum(br[:, :estimated_population_2019]) + +u = [N - i₀, i₀, 0.0] +p = [0.5, 0.05] +prob = ODEProblem(sir_ode!, u, (1.0, 100.0), p) +sol_ode = solve(prob) +f = Figure() +plt = + data(stack(DataFrame(sol_ode), Not(:timestamp))) * + mapping( + :timestamp => L"t", + :value => L"N"; + color=:variable => renamer(["value1" => "S", "value2" => "I", "value3" => "R"]), + ) * + visual(Lines; linewidth=3) +draw!(f[1, 1], plt; axis=(; title="SIR Model for 100 days, β = $(p[1]), γ = $(p[2])")) +save(joinpath(@OUTPUT, "ode_solve.svg"), f); # hide +```` + +\fig{ode_solve} +\center{*SIR ODE Solution for Brazil's 100 days of COVID in early 2020*} \\ + +## How to use a ODE solver in a Turing Model + +Please note that we are using the alternative negative binomial parameterization as specified in [8. **Bayesian Regression with Count Data**](/pages/8_count_reg/): + +````julia:ex7 +function NegativeBinomial2(μ, ϕ) + p = 1 / (1 + μ / ϕ) + r = ϕ + + return NegativeBinomial(r, p) +end +```` + +Now this is the fun part. It's easy: just stick it inside! + +````julia:ex8 +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function bayes_sir(infected, i₀, r₀, N) + #calculate number of timepoints + l = length(infected) + + #priors + β ~ TruncatedNormal(2, 1, 1e-4, 10) # using 10 because numerical issues arose + γ ~ TruncatedNormal(0.4, 0.5, 1e-4, 10) # using 10 because numerical issues arose + ϕ⁻ ~ truncated(Exponential(5); lower=0, upper=1e5) + ϕ = 1.0 / ϕ⁻ + + #ODE Stuff + I = i₀ + u0 = [N - I, I, r₀] # S,I,R + p = [β, γ] + tspan = (1.0, float(l)) + prob = ODEProblem(sir_ode!, u0, tspan, p) + sol = solve( + prob, + Tsit5(); # similar to Dormand-Prince RK45 in Stan but 20% faster + saveat=1.0, + ) + solᵢ = Array(sol)[2, :] # New Infected + solᵢ = max.(1e-4, solᵢ) # numerical issues arose + + #likelihood + return infected ~ arraydist(LazyArray(@~ NegativeBinomial2.(solᵢ, ϕ))) +end; +```` + +Now run the model and inspect our parameters estimates. +We will be using the default `NUTS()` sampler with `1_000` samples on only one Markov chain: + +````julia:ex9 +infected = br[:, :new_confirmed] +r₀ = first(br[:, :new_deaths]) +model_sir = bayes_sir(infected, i₀, r₀, N) +chain_sir = sample(model_sir, NUTS(), 1_000) +summarystats(chain_sir[[:β, :γ]]) +```` + +Hope you had learned some new bayesian computational skills and also took notice +of the amazing potential of Julia's ecosystem of packages. + +## References + +Grinsztajn, L., Semenova, E., Margossian, C. C., & Riou, J. (2021). Bayesian workflow for disease transmission modeling in Stan. ArXiv:2006.02985 [q-Bio, Stat]. http://arxiv.org/abs/2006.02985 + diff --git a/assets/literate/13_epi_models_script.jl b/assets/literate/13_epi_models_script.jl new file mode 100644 index 00000000..e94cc1cf --- /dev/null +++ b/assets/literate/13_epi_models_script.jl @@ -0,0 +1,131 @@ +# This file was generated, do not modify it. + +using Downloads +using DataFrames +using CSV +using Chain +using Dates + +url = "https://data.brasil.io/dataset/covid19/caso_full.csv.gz" +file = Downloads.download(url) +df = DataFrame(CSV.File(file)) +br = @chain df begin + filter( + [:date, :city] => + (date, city) -> + date < Dates.Date("2021-01-01") && + date > Dates.Date("2020-04-01") && + ismissing(city), + _, + ) + groupby(:date) + combine( + [ + :estimated_population_2019, + :last_available_confirmed_per_100k_inhabitants, + :last_available_deaths, + :new_confirmed, + :new_deaths, + ] .=> + sum .=> [ + :estimated_population_2019, + :last_available_confirmed_per_100k_inhabitants, + :last_available_deaths, + :new_confirmed, + :new_deaths, + ], + ) +end; + +first(br, 5) + +last(br, 5) + +using AlgebraOfGraphics +using CairoMakie +f = Figure() +plt = data(br) * mapping(:date => L"t", :new_confirmed => "infected daily") * visual(Lines) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "infected.svg"), f); # hide + +using DifferentialEquations + +function sir_ode!(du, u, p, t) + (S, I, R) = u + (β, γ) = p + N = S + I + R + infection = β * I * S / N + recovery = γ * I + @inbounds begin + du[1] = -infection # Susceptible + du[2] = infection - recovery # Infected + du[3] = recovery # Recovered + end + return nothing +end; + +i₀ = first(br[:, :new_confirmed]) +N = maximum(br[:, :estimated_population_2019]) + +u = [N - i₀, i₀, 0.0] +p = [0.5, 0.05] +prob = ODEProblem(sir_ode!, u, (1.0, 100.0), p) +sol_ode = solve(prob) +f = Figure() +plt = + data(stack(DataFrame(sol_ode), Not(:timestamp))) * + mapping( + :timestamp => L"t", + :value => L"N"; + color=:variable => renamer(["value1" => "S", "value2" => "I", "value3" => "R"]), + ) * + visual(Lines; linewidth=3) +draw!(f[1, 1], plt; axis=(; title="SIR Model for 100 days, β = $(p[1]), γ = $(p[2])")) +save(joinpath(@OUTPUT, "ode_solve.svg"), f); # hide + +function NegativeBinomial2(μ, ϕ) + p = 1 / (1 + μ / ϕ) + r = ϕ + + return NegativeBinomial(r, p) +end + +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function bayes_sir(infected, i₀, r₀, N) + #calculate number of timepoints + l = length(infected) + + #priors + β ~ TruncatedNormal(2, 1, 1e-4, 10) # using 10 because numerical issues arose + γ ~ TruncatedNormal(0.4, 0.5, 1e-4, 10) # using 10 because numerical issues arose + ϕ⁻ ~ truncated(Exponential(5); lower=0, upper=1e5) + ϕ = 1.0 / ϕ⁻ + + #ODE Stuff + I = i₀ + u0 = [N - I, I, r₀] # S,I,R + p = [β, γ] + tspan = (1.0, float(l)) + prob = ODEProblem(sir_ode!, u0, tspan, p) + sol = solve( + prob, + Tsit5(); # similar to Dormand-Prince RK45 in Stan but 20% faster + saveat=1.0, + ) + solᵢ = Array(sol)[2, :] # New Infected + solᵢ = max.(1e-4, solᵢ) # numerical issues arose + + #likelihood + return infected ~ arraydist(LazyArray(@~ NegativeBinomial2.(solᵢ, ϕ))) +end; + +infected = br[:, :new_confirmed] +r₀ = first(br[:, :new_deaths]) +model_sir = bayes_sir(infected, i₀, r₀, N) +chain_sir = sample(model_sir, NUTS(), 1_000) +summarystats(chain_sir[[:β, :γ]]) diff --git a/assets/pages/01_why_Julia/code/ex1.jl b/assets/pages/01_why_Julia/code/ex1.jl new file mode 100644 index 00000000..2f4bc5c2 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex1.jl @@ -0,0 +1,18 @@ +# This file was generated, do not modify it. # hide +abstract type Pet end +struct Dog <: Pet + name::String +end +struct Cat <: Pet + name::String +end + +function encounter(a::Pet, b::Pet) + verb = meets(a, b) + return println("$(a.name) meets $(b.name) and $verb") +end + +meets(a::Dog, b::Dog) = "sniffs"; +meets(a::Dog, b::Cat) = "chases"; +meets(a::Cat, b::Dog) = "hisses"; +meets(a::Cat, b::Cat) = "slinks"; \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex10.jl b/assets/pages/01_why_Julia/code/ex10.jl new file mode 100644 index 00000000..ca645e54 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex10.jl @@ -0,0 +1,5 @@ +# This file was generated, do not modify it. # hide +import Base: * + +*(A::AbstractMatrix, v::OneHotVector) = A[:, v.ind] +inner(v::OneHotVector, A, w::OneHotVector) = A[v.ind, w.ind] \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex11.jl b/assets/pages/01_why_Julia/code/ex11.jl new file mode 100644 index 00000000..c9d7999e --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex11.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +@btime inner_sum($A, $onehot); \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex2.jl b/assets/pages/01_why_Julia/code/ex2.jl new file mode 100644 index 00000000..97e3d89b --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex2.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +fido = Dog("Fido"); +rex = Dog("Rex"); +whiskers = Cat("Whiskers"); +spots = Cat("Spots"); + +encounter(fido, rex) +encounter(rex, whiskers) +encounter(spots, fido) +encounter(whiskers, spots) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex3.jl b/assets/pages/01_why_Julia/code/ex3.jl new file mode 100644 index 00000000..effea0ed --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex3.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +import Base: size, getindex + +struct OneHotVector <: AbstractVector{Int} + len::Int + ind::Int +end + +size(v::OneHotVector) = (v.len,) + +getindex(v::OneHotVector, i::Integer) = Int(i == v.ind) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex4.jl b/assets/pages/01_why_Julia/code/ex4.jl new file mode 100644 index 00000000..cc1e6f40 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex4.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +onehot = [OneHotVector(3, rand(1:3)) for _ in 1:4] \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex5.jl b/assets/pages/01_why_Julia/code/ex5.jl new file mode 100644 index 00000000..adcefd44 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex5.jl @@ -0,0 +1,12 @@ +# This file was generated, do not modify it. # hide +using LinearAlgebra + +function inner_sum(A, vs) + t = zero(eltype(A)) + for v in vs + t += inner(v, A, v) # multiple dispatch! + end + return t +end + +inner(v, A, w) = dot(v, A * w) # very general definition \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex6.jl b/assets/pages/01_why_Julia/code/ex6.jl new file mode 100644 index 00000000..2d915949 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex6.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +A = rand(3, 3) +vs = [rand(3) for _ in 1:4] +inner_sum(A, vs) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex7.jl b/assets/pages/01_why_Julia/code/ex7.jl new file mode 100644 index 00000000..0e407abe --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex7.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +supertype(OneHotVector) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex8.jl b/assets/pages/01_why_Julia/code/ex8.jl new file mode 100644 index 00000000..d854b3c0 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex8.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +inner_sum(A, onehot) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/ex9.jl b/assets/pages/01_why_Julia/code/ex9.jl new file mode 100644 index 00000000..06337616 --- /dev/null +++ b/assets/pages/01_why_Julia/code/ex9.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +using BenchmarkTools + +@btime inner_sum($A, $onehot); \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex1.out b/assets/pages/01_why_Julia/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex1.res b/assets/pages/01_why_Julia/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex10.out b/assets/pages/01_why_Julia/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex10.res b/assets/pages/01_why_Julia/code/output/ex10.res new file mode 100644 index 00000000..d8d8a8f3 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex10.res @@ -0,0 +1 @@ +inner (generic function with 2 methods) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex11.out b/assets/pages/01_why_Julia/code/output/ex11.out new file mode 100644 index 00000000..123bd745 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex11.out @@ -0,0 +1 @@ + 4.648 ns (0 allocations: 0 bytes) diff --git a/assets/pages/01_why_Julia/code/output/ex11.res b/assets/pages/01_why_Julia/code/output/ex11.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex11.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex2.out b/assets/pages/01_why_Julia/code/output/ex2.out new file mode 100644 index 00000000..4b91f35b --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex2.out @@ -0,0 +1,4 @@ +Fido meets Rex and sniffs +Rex meets Whiskers and chases +Spots meets Fido and hisses +Whiskers meets Spots and slinks diff --git a/assets/pages/01_why_Julia/code/output/ex2.res b/assets/pages/01_why_Julia/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex3.out b/assets/pages/01_why_Julia/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex3.res b/assets/pages/01_why_Julia/code/output/ex3.res new file mode 100644 index 00000000..74006065 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex3.res @@ -0,0 +1 @@ +getindex (generic function with 928 methods) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex4.out b/assets/pages/01_why_Julia/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex4.res b/assets/pages/01_why_Julia/code/output/ex4.res new file mode 100644 index 00000000..38d30fdd --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex4.res @@ -0,0 +1,5 @@ +4-element Vector{Main.FD_SANDBOX_547765167833527747.OneHotVector}: + [1, 0, 0] + [1, 0, 0] + [1, 0, 0] + [0, 0, 1] \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex5.out b/assets/pages/01_why_Julia/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex5.res b/assets/pages/01_why_Julia/code/output/ex5.res new file mode 100644 index 00000000..df5d6ae2 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex5.res @@ -0,0 +1 @@ +inner (generic function with 1 method) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex6.out b/assets/pages/01_why_Julia/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex6.res b/assets/pages/01_why_Julia/code/output/ex6.res new file mode 100644 index 00000000..32f4af79 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex6.res @@ -0,0 +1 @@ +2.4426500326493357 \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex7.out b/assets/pages/01_why_Julia/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex7.res b/assets/pages/01_why_Julia/code/output/ex7.res new file mode 100644 index 00000000..89fd7904 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex7.res @@ -0,0 +1 @@ +AbstractVector{Int64} (alias for AbstractArray{Int64, 1}) \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex8.out b/assets/pages/01_why_Julia/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/01_why_Julia/code/output/ex8.res b/assets/pages/01_why_Julia/code/output/ex8.res new file mode 100644 index 00000000..00d31569 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex8.res @@ -0,0 +1 @@ +1.604358084381523 \ No newline at end of file diff --git a/assets/pages/01_why_Julia/code/output/ex9.out b/assets/pages/01_why_Julia/code/output/ex9.out new file mode 100644 index 00000000..a5143c40 --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex9.out @@ -0,0 +1 @@ + 189.279 ns (4 allocations: 320 bytes) diff --git a/assets/pages/01_why_Julia/code/output/ex9.res b/assets/pages/01_why_Julia/code/output/ex9.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/01_why_Julia/code/output/ex9.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/02_bayes_stats/code/ex1.jl b/assets/pages/02_bayes_stats/code/ex1.jl new file mode 100644 index 00000000..2eaa25b4 --- /dev/null +++ b/assets/pages/02_bayes_stats/code/ex1.jl @@ -0,0 +1,18 @@ +# This file was generated, do not modify it. # hide +using CairoMakie +using Distributions + +d = LogNormal(0, 2) +range_d = 0:0.001:4 +q25 = quantile(d, 0.25) +q75 = quantile(d, 0.75) +credint = range(q25; stop=q75, length=100) +f, ax, l = lines( + range_d, + pdf.(d, range_d); + linewidth=3, + axis=(; limits=(-0.2, 4.2, nothing, nothing), xlabel=L"\theta", ylabel="Density"), +) +scatter!(ax, mode(d), pdf(d, mode(d)); color=:green, markersize=12) +band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5)) +save(joinpath(@OUTPUT, "lognormal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/02_bayes_stats/code/ex2.jl b/assets/pages/02_bayes_stats/code/ex2.jl new file mode 100644 index 00000000..83f6d6ad --- /dev/null +++ b/assets/pages/02_bayes_stats/code/ex2.jl @@ -0,0 +1,25 @@ +# This file was generated, do not modify it. # hide +d1 = Normal(10, 1) +d2 = Normal(2, 1) +mix_d = [0.4, 0.6] +d = MixtureModel([d1, d2], mix_d) +range_d = -2:0.01:14 +sim_d = rand(d, 10_000) +q25 = quantile(sim_d, 0.25) +q75 = quantile(sim_d, 0.75) +credint = range(q25; stop=q75, length=100) + +f, ax, l = lines( + range_d, + pdf.(d, range_d); + linewidth=3, + axis=(; + limits=(-2, 14, nothing, nothing), + xticks=[0, 5, 10], + xlabel=L"\theta", + ylabel="Density", + ), +) +scatter!(ax, mode(d2), pdf(d, mode(d2)); color=:green, markersize=12) +band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5)) +save(joinpath(@OUTPUT, "mixture.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/02_bayes_stats/code/output/ex1.out b/assets/pages/02_bayes_stats/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/02_bayes_stats/code/output/ex1.res b/assets/pages/02_bayes_stats/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/02_bayes_stats/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/02_bayes_stats/code/output/ex2.out b/assets/pages/02_bayes_stats/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/02_bayes_stats/code/output/ex2.res b/assets/pages/02_bayes_stats/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/02_bayes_stats/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/02_bayes_stats/code/output/lognormal.svg b/assets/pages/02_bayes_stats/code/output/lognormal.svg new file mode 100644 index 00000000..1c3c36a5 --- /dev/null +++ b/assets/pages/02_bayes_stats/code/output/lognormal.svg @@ -0,0 +1,219 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/02_bayes_stats/code/output/mixture.svg b/assets/pages/02_bayes_stats/code/output/mixture.svg new file mode 100644 index 00000000..74368867 --- /dev/null +++ b/assets/pages/02_bayes_stats/code/output/mixture.svg @@ -0,0 +1,176 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/ex1.jl b/assets/pages/03_prob_dist/code/ex1.jl new file mode 100644 index 00000000..b7e675ca --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex1.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +using CairoMakie +using Distributions + +f, ax, b = barplot( + DiscreteUniform(1, 6); + axis=(; + title="6-sided Dice", + xlabel=L"\theta", + ylabel="Mass", + xticks=1:6, + limits=(nothing, nothing, 0, 0.3), + ), +) +save(joinpath(@OUTPUT, "discrete_uniform.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex10.jl b/assets/pages/03_prob_dist/code/ex10.jl new file mode 100644 index 00000000..e59bdb82 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex10.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + Beta(1, 1); + label=L"a=b=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 1, nothing, nothing)), +) +lines!(ax, Beta(3, 2); label=L"a=3, b=2", linewidth=5) +lines!(ax, Beta(2, 3); label=L"a=2, b=3", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "beta.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex2.jl b/assets/pages/03_prob_dist/code/ex2.jl new file mode 100644 index 00000000..28ac1662 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex2.jl @@ -0,0 +1,18 @@ +# This file was generated, do not modify it. # hide +f, ax1, b = barplot( + Bernoulli(0.5); + width=0.3, + axis=(; + title=L"p=0.5", + xlabel=L"\theta", + ylabel="Mass", + xticks=0:1, + limits=(nothing, nothing, 0, 1), + ), +) +ax2 = Axis( + f[1, 2]; title=L"p=0.2", xlabel=L"\theta", xticks=0:1, limits=(nothing, nothing, 0, 1) +) +barplot!(ax2, Bernoulli(0.2); width=0.3) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "bernoulli.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex3.jl b/assets/pages/03_prob_dist/code/ex3.jl new file mode 100644 index 00000000..4a268d00 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex3.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +f, ax1, b = barplot( + Binomial(5, 0.5); axis=(; title=L"p=0.5", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"p=0.2", xlabel=L"\theta") +barplot!(ax2, Binomial(5, 0.2)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "binomial.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex4.jl b/assets/pages/03_prob_dist/code/ex4.jl new file mode 100644 index 00000000..a06e626f --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex4.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +f, ax1, b = barplot( + Poisson(1); axis=(; title=L"\lambda=1", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"\lambda=4", xlabel=L"\theta") +barplot!(ax2, Poisson(4)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "poisson.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex5.jl b/assets/pages/03_prob_dist/code/ex5.jl new file mode 100644 index 00000000..0dc07bb8 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex5.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +f, ax1, b = barplot( + NegativeBinomial(1, 0.5); axis=(; title=L"k=1", xlabel=L"\theta", ylabel="Mass") +) +ax2 = Axis(f[1, 2]; title=L"k=2", xlabel=L"\theta") +barplot!(ax2, NegativeBinomial(2, 0.5)) +linkaxes!(ax1, ax2) +save(joinpath(@OUTPUT, "negbinomial.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex6.jl b/assets/pages/03_prob_dist/code/ex6.jl new file mode 100644 index 00000000..9e1d1170 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex6.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + Normal(0, 1); + label=L"\sigma=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)), +) +lines!(ax, Normal(0, 0.5); label=L"\sigma=0.5", linewidth=5) +lines!(ax, Normal(0, 2); label=L"\sigma=2", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "normal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex7.jl b/assets/pages/03_prob_dist/code/ex7.jl new file mode 100644 index 00000000..935e41e4 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex7.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + LogNormal(0, 1); + label=L"\sigma=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 3, nothing, nothing)), +) +lines!(ax, LogNormal(0, 0.25); label=L"\sigma=0.25", linewidth=5) +lines!(ax, LogNormal(0, 0.5); label=L"\sigma=0.5", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "lognormal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex8.jl b/assets/pages/03_prob_dist/code/ex8.jl new file mode 100644 index 00000000..bcea92b2 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex8.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + Exponential(1); + label=L"\lambda=1", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 4.5, nothing, nothing)), +) +lines!(ax, Exponential(0.5); label=L"\lambda=0.5", linewidth=5) +lines!(ax, Exponential(1.5); label=L"\lambda=2", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "exponential.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/ex9.jl b/assets/pages/03_prob_dist/code/ex9.jl new file mode 100644 index 00000000..47d1b2f7 --- /dev/null +++ b/assets/pages/03_prob_dist/code/ex9.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + TDist(2); + label=L"\nu=2", + linewidth=5, + axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)), +) +lines!(ax, TDist(8); label=L"\nu=8", linewidth=5) +lines!(ax, TDist(30); label=L"\nu=30", linewidth=5) +axislegend(ax) +save(joinpath(@OUTPUT, "tdist.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/bernoulli.svg b/assets/pages/03_prob_dist/code/output/bernoulli.svg new file mode 100644 index 00000000..2af73db1 --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/bernoulli.svg @@ -0,0 +1,358 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/beta.svg b/assets/pages/03_prob_dist/code/output/beta.svg new file mode 100644 index 00000000..4a4dea0c --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/beta.svg @@ -0,0 +1,352 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/binomial.svg b/assets/pages/03_prob_dist/code/output/binomial.svg new file mode 100644 index 00000000..d4d2bc2f --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/binomial.svg @@ -0,0 +1,415 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/discrete_uniform.svg b/assets/pages/03_prob_dist/code/output/discrete_uniform.svg new file mode 100644 index 00000000..6d7809d8 --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/discrete_uniform.svg @@ -0,0 +1,332 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/ex1.out b/assets/pages/03_prob_dist/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex1.res b/assets/pages/03_prob_dist/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex10.out b/assets/pages/03_prob_dist/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex10.res b/assets/pages/03_prob_dist/code/output/ex10.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex10.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex2.out b/assets/pages/03_prob_dist/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex2.res b/assets/pages/03_prob_dist/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex3.out b/assets/pages/03_prob_dist/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex3.res b/assets/pages/03_prob_dist/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex4.out b/assets/pages/03_prob_dist/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex4.res b/assets/pages/03_prob_dist/code/output/ex4.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex4.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex5.out b/assets/pages/03_prob_dist/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex5.res b/assets/pages/03_prob_dist/code/output/ex5.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex5.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex6.out b/assets/pages/03_prob_dist/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex6.res b/assets/pages/03_prob_dist/code/output/ex6.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex6.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex7.out b/assets/pages/03_prob_dist/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex7.res b/assets/pages/03_prob_dist/code/output/ex7.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex7.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex8.out b/assets/pages/03_prob_dist/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex8.res b/assets/pages/03_prob_dist/code/output/ex8.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex8.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/ex9.out b/assets/pages/03_prob_dist/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/03_prob_dist/code/output/ex9.res b/assets/pages/03_prob_dist/code/output/ex9.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/ex9.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/03_prob_dist/code/output/exponential.svg b/assets/pages/03_prob_dist/code/output/exponential.svg new file mode 100644 index 00000000..e228c357 --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/exponential.svg @@ -0,0 +1,448 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/lognormal.svg b/assets/pages/03_prob_dist/code/output/lognormal.svg new file mode 100644 index 00000000..84440f53 --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/lognormal.svg @@ -0,0 +1,284 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/negbinomial.svg b/assets/pages/03_prob_dist/code/output/negbinomial.svg new file mode 100644 index 00000000..9bca0bcc --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/negbinomial.svg @@ -0,0 +1,367 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/normal.svg b/assets/pages/03_prob_dist/code/output/normal.svg new file mode 100644 index 00000000..07bd6495 --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/normal.svg @@ -0,0 +1,317 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/poisson.svg b/assets/pages/03_prob_dist/code/output/poisson.svg new file mode 100644 index 00000000..5e4e5878 --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/poisson.svg @@ -0,0 +1,348 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/03_prob_dist/code/output/tdist.svg b/assets/pages/03_prob_dist/code/output/tdist.svg new file mode 100644 index 00000000..a78aa43b --- /dev/null +++ b/assets/pages/03_prob_dist/code/output/tdist.svg @@ -0,0 +1,336 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/04_Turing/code/ex1.jl b/assets/pages/04_Turing/code/ex1.jl new file mode 100644 index 00000000..a91fe213 --- /dev/null +++ b/assets/pages/04_Turing/code/ex1.jl @@ -0,0 +1,13 @@ +# This file was generated, do not modify it. # hide +using CairoMakie +using Distributions + +dice = DiscreteUniform(1, 6) +f, ax, b = barplot( + dice; + label="six-sided Dice", + axis=(; xlabel=L"\theta", ylabel="Mass", xticks=1:6, limits=(nothing, nothing, 0, 0.3)), +) +vlines!(ax, [mean(dice)]; linewidth=5, color=:red, label=L"E(\theta)") +axislegend(ax) +save(joinpath(@OUTPUT, "dice.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex10.jl b/assets/pages/04_Turing/code/ex10.jl new file mode 100644 index 00000000..794c681e --- /dev/null +++ b/assets/pages/04_Turing/code/ex10.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summaries \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex11.jl b/assets/pages/04_Turing/code/ex11.jl new file mode 100644 index 00000000..8945dd61 --- /dev/null +++ b/assets/pages/04_Turing/code/ex11.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +sum(summaries[:, :mean]) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex12.jl b/assets/pages/04_Turing/code/ex12.jl new file mode 100644 index 00000000..c5220908 --- /dev/null +++ b/assets/pages/04_Turing/code/ex12.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summarystats(chain[:, 1:3, :]) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex13.jl b/assets/pages/04_Turing/code/ex13.jl new file mode 100644 index 00000000..72035ec9 --- /dev/null +++ b/assets/pages/04_Turing/code/ex13.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summarystats(chain[[:var"p[1]", :var"p[2]"]]) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex14.jl b/assets/pages/04_Turing/code/ex14.jl new file mode 100644 index 00000000..9e464ba5 --- /dev/null +++ b/assets/pages/04_Turing/code/ex14.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +sum([idx * i for (i, idx) in enumerate(summaries[:, :mean])]) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex15.jl b/assets/pages/04_Turing/code/ex15.jl new file mode 100644 index 00000000..6f5d4266 --- /dev/null +++ b/assets/pages/04_Turing/code/ex15.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +typeof(chain) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex16.jl b/assets/pages/04_Turing/code/ex16.jl new file mode 100644 index 00000000..9fe8b910 --- /dev/null +++ b/assets/pages/04_Turing/code/ex16.jl @@ -0,0 +1,14 @@ +# This file was generated, do not modify it. # hide +using AlgebraOfGraphics +using AlgebraOfGraphics: density +#exclude additional information such as log probability +params = names(chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt1 = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines) +plt2 = data(chain) * chain_mapping * density() +f = Figure(; resolution=(800, 600)) +draw!(f[1, 1], plt1) +draw!(f[1, 2], plt2; axis=(; ylabel="density")) +save(joinpath(@OUTPUT, "chain.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex17.jl b/assets/pages/04_Turing/code/ex17.jl new file mode 100644 index 00000000..a8978c3c --- /dev/null +++ b/assets/pages/04_Turing/code/ex17.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +prior_chain = sample(model, Prior(), 2_000); \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex18.jl b/assets/pages/04_Turing/code/ex18.jl new file mode 100644 index 00000000..91722aab --- /dev/null +++ b/assets/pages/04_Turing/code/ex18.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +missing_data = similar(my_data, Missing) # vector of `missing` +model_missing = dice_throw(missing_data) # instantiate the "predictive model +prior_check = predict(model_missing, prior_chain); \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex19.jl b/assets/pages/04_Turing/code/ex19.jl new file mode 100644 index 00000000..c2753944 --- /dev/null +++ b/assets/pages/04_Turing/code/ex19.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +typeof(prior_check) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex2.jl b/assets/pages/04_Turing/code/ex2.jl new file mode 100644 index 00000000..aca99b37 --- /dev/null +++ b/assets/pages/04_Turing/code/ex2.jl @@ -0,0 +1,14 @@ +# This file was generated, do not modify it. # hide +using Turing +setprogress!(false) # hide + +@model function dice_throw(y) + #Our prior belief about the probability of each result in a six-sided dice. + #p is a vector of length 6 each with probability p that sums up to 1. + p ~ Dirichlet(6, 1) + + #Each outcome of the six-sided dice has a probability p. + for i in eachindex(y) + y[i] ~ Categorical(p) + end +end; \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex20.jl b/assets/pages/04_Turing/code/ex20.jl new file mode 100644 index 00000000..beeea9c7 --- /dev/null +++ b/assets/pages/04_Turing/code/ex20.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summarystats(prior_check[:, 1:5, :]) # just the first 5 prior samples \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex21.jl b/assets/pages/04_Turing/code/ex21.jl new file mode 100644 index 00000000..cc7e0ab7 --- /dev/null +++ b/assets/pages/04_Turing/code/ex21.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +posterior_check = predict(model_missing, chain); +summarystats(posterior_check[:, 1:5, :]) # just the first 5 posterior samples \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex3.jl b/assets/pages/04_Turing/code/ex3.jl new file mode 100644 index 00000000..ef58bb11 --- /dev/null +++ b/assets/pages/04_Turing/code/ex3.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +mean(Dirichlet(6, 1)) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex4.jl b/assets/pages/04_Turing/code/ex4.jl new file mode 100644 index 00000000..f86c3ca5 --- /dev/null +++ b/assets/pages/04_Turing/code/ex4.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +sum(mean(Dirichlet(6, 1))) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex5.jl b/assets/pages/04_Turing/code/ex5.jl new file mode 100644 index 00000000..5d5a1601 --- /dev/null +++ b/assets/pages/04_Turing/code/ex5.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +using Random + +Random.seed!(123); + +my_data = rand(DiscreteUniform(1, 6), 1_000); \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex6.jl b/assets/pages/04_Turing/code/ex6.jl new file mode 100644 index 00000000..5908f9af --- /dev/null +++ b/assets/pages/04_Turing/code/ex6.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +first(my_data, 5) \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex7.jl b/assets/pages/04_Turing/code/ex7.jl new file mode 100644 index 00000000..e079af61 --- /dev/null +++ b/assets/pages/04_Turing/code/ex7.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +model = dice_throw(my_data); \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex8.jl b/assets/pages/04_Turing/code/ex8.jl new file mode 100644 index 00000000..cbed0567 --- /dev/null +++ b/assets/pages/04_Turing/code/ex8.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +chain = sample(model, NUTS(), 1_000); \ No newline at end of file diff --git a/assets/pages/04_Turing/code/ex9.jl b/assets/pages/04_Turing/code/ex9.jl new file mode 100644 index 00000000..765193cc --- /dev/null +++ b/assets/pages/04_Turing/code/ex9.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summaries, quantiles = describe(chain); \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/chain.svg b/assets/pages/04_Turing/code/output/chain.svg new file mode 100644 index 00000000..797d1607 --- /dev/null +++ b/assets/pages/04_Turing/code/output/chain.svg @@ -0,0 +1,1518 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/04_Turing/code/output/dice.svg b/assets/pages/04_Turing/code/output/dice.svg new file mode 100644 index 00000000..5cc4629b --- /dev/null +++ b/assets/pages/04_Turing/code/output/dice.svg @@ -0,0 +1,356 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/04_Turing/code/output/ex1.out b/assets/pages/04_Turing/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex1.res b/assets/pages/04_Turing/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex10.out b/assets/pages/04_Turing/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex10.res b/assets/pages/04_Turing/code/output/ex10.res new file mode 100644 index 00000000..12644fa0 --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex10.res @@ -0,0 +1,10 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + p[1] 0.1571 0.0124 0.0003 2069.1239 657.7236 1.0047 536.1814 + p[2] 0.1449 0.0115 0.0002 2261.9424 741.2396 1.0061 586.1473 + p[3] 0.1401 0.0108 0.0002 1911.4282 868.5186 1.0069 495.3170 + p[4] 0.1808 0.0120 0.0003 2087.7925 842.7605 1.0005 541.0190 + p[5] 0.1971 0.0126 0.0003 1720.0950 673.6157 1.0019 445.7359 + p[6] 0.1800 0.0126 0.0003 1786.2325 742.7542 1.0029 462.8745 diff --git a/assets/pages/04_Turing/code/output/ex11.out b/assets/pages/04_Turing/code/output/ex11.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex11.res b/assets/pages/04_Turing/code/output/ex11.res new file mode 100644 index 00000000..9f8e9b69 --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex11.res @@ -0,0 +1 @@ +1.0 \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex12.out b/assets/pages/04_Turing/code/output/ex12.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex12.res b/assets/pages/04_Turing/code/output/ex12.res new file mode 100644 index 00000000..bb82aaed --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex12.res @@ -0,0 +1,7 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + p[1] 0.1571 0.0124 0.0003 2069.1239 657.7236 1.0047 536.1814 + p[2] 0.1449 0.0115 0.0002 2261.9424 741.2396 1.0061 586.1473 + p[3] 0.1401 0.0108 0.0002 1911.4282 868.5186 1.0069 495.3170 diff --git a/assets/pages/04_Turing/code/output/ex13.out b/assets/pages/04_Turing/code/output/ex13.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex13.res b/assets/pages/04_Turing/code/output/ex13.res new file mode 100644 index 00000000..8096e81b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex13.res @@ -0,0 +1,6 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + p[1] 0.1571 0.0124 0.0003 2069.1239 657.7236 1.0047 536.1814 + p[2] 0.1449 0.0115 0.0002 2261.9424 741.2396 1.0061 586.1473 diff --git a/assets/pages/04_Turing/code/output/ex14.out b/assets/pages/04_Turing/code/output/ex14.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex14.res b/assets/pages/04_Turing/code/output/ex14.res new file mode 100644 index 00000000..30ca84aa --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex14.res @@ -0,0 +1 @@ +3.655798033295476 \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex15.out b/assets/pages/04_Turing/code/output/ex15.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex15.res b/assets/pages/04_Turing/code/output/ex15.res new file mode 100644 index 00000000..73c44bd1 --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex15.res @@ -0,0 +1 @@ +MCMCChains.Chains{Float64, AxisArrays.AxisArray{Float64, 3, Array{Float64, 3}, Tuple{AxisArrays.Axis{:iter, StepRange{Int64, Int64}}, AxisArrays.Axis{:var, Vector{Symbol}}, AxisArrays.Axis{:chain, UnitRange{Int64}}}}, Missing, @NamedTuple{parameters::Vector{Symbol}, internals::Vector{Symbol}}, @NamedTuple{varname_to_symbol::OrderedCollections.OrderedDict{AbstractPPL.VarName, Symbol}, start_time::Float64, stop_time::Float64}} \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex16.out b/assets/pages/04_Turing/code/output/ex16.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex16.res b/assets/pages/04_Turing/code/output/ex16.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex16.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex17.out b/assets/pages/04_Turing/code/output/ex17.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex17.res b/assets/pages/04_Turing/code/output/ex17.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex17.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex18.out b/assets/pages/04_Turing/code/output/ex18.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex18.res b/assets/pages/04_Turing/code/output/ex18.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex18.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex19.out b/assets/pages/04_Turing/code/output/ex19.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex19.res b/assets/pages/04_Turing/code/output/ex19.res new file mode 100644 index 00000000..0f349bab --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex19.res @@ -0,0 +1 @@ +MCMCChains.Chains{Float64, AxisArrays.AxisArray{Float64, 3, Array{Float64, 3}, Tuple{AxisArrays.Axis{:iter, StepRange{Int64, Int64}}, AxisArrays.Axis{:var, Vector{Symbol}}, AxisArrays.Axis{:chain, UnitRange{Int64}}}}, Missing, @NamedTuple{parameters::Vector{Symbol}, internals::Vector{Symbol}}, @NamedTuple{varname_to_symbol::OrderedCollections.OrderedDict{AbstractPPL.VarName, Symbol}}} \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex2.out b/assets/pages/04_Turing/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex2.res b/assets/pages/04_Turing/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex20.out b/assets/pages/04_Turing/code/output/ex20.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex20.res b/assets/pages/04_Turing/code/output/ex20.res new file mode 100644 index 00000000..1a206266 --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex20.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Missing + + y[1] 3.4265 1.7433 0.0396 1966.1989 NaN 1.0016 missing + y[2] 3.4815 1.6974 0.0414 1672.3452 NaN 1.0024 missing + y[3] 3.4620 1.7149 0.0396 1878.7524 NaN 1.0022 missing + y[4] 3.4905 1.7177 0.0387 1960.3938 NaN 1.0006 missing + y[5] 3.5095 1.7101 0.0386 2002.7276 NaN 0.9998 missing diff --git a/assets/pages/04_Turing/code/output/ex21.out b/assets/pages/04_Turing/code/output/ex21.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex21.res b/assets/pages/04_Turing/code/output/ex21.res new file mode 100644 index 00000000..72728bb2 --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex21.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Missing + + y[1] 3.8050 1.6637 0.0510 1065.1417 NaN 0.9992 missing + y[2] 3.6280 1.6638 0.0583 773.8349 NaN 1.0000 missing + y[3] 3.6430 1.7007 0.0554 921.8019 NaN 1.0044 missing + y[4] 3.5720 1.6845 0.0518 1052.7732 NaN 0.9993 missing + y[5] 3.6390 1.7400 0.0546 988.2165 NaN 1.0012 missing diff --git a/assets/pages/04_Turing/code/output/ex3.out b/assets/pages/04_Turing/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex3.res b/assets/pages/04_Turing/code/output/ex3.res new file mode 100644 index 00000000..a09f8eea --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex3.res @@ -0,0 +1 @@ +6-element Fill{Float64}, with entries equal to 0.16666666666666666 \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex4.out b/assets/pages/04_Turing/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex4.res b/assets/pages/04_Turing/code/output/ex4.res new file mode 100644 index 00000000..9f8e9b69 --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex4.res @@ -0,0 +1 @@ +1.0 \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex5.out b/assets/pages/04_Turing/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex5.res b/assets/pages/04_Turing/code/output/ex5.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex5.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex6.out b/assets/pages/04_Turing/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex6.res b/assets/pages/04_Turing/code/output/ex6.res new file mode 100644 index 00000000..89dba24d --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex6.res @@ -0,0 +1,6 @@ +5-element Vector{Int64}: + 4 + 4 + 6 + 2 + 4 \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex7.out b/assets/pages/04_Turing/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex7.res b/assets/pages/04_Turing/code/output/ex7.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex7.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex8.out b/assets/pages/04_Turing/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex8.res b/assets/pages/04_Turing/code/output/ex8.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex8.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/04_Turing/code/output/ex9.out b/assets/pages/04_Turing/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/04_Turing/code/output/ex9.res b/assets/pages/04_Turing/code/output/ex9.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/04_Turing/code/output/ex9.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex1.jl b/assets/pages/05_MCMC/code/ex1.jl new file mode 100644 index 00000000..68be1911 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex1.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +using CairoMakie +using Distributions +using Random + +Random.seed!(123); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex10.jl b/assets/pages/05_MCMC/code/ex10.jl new file mode 100644 index 00000000..76ec83e2 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex10.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +mean(summarystats(chain_met)[:, :ess_tail]) / S \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex11.jl b/assets/pages/05_MCMC/code/ex11.jl new file mode 100644 index 00000000..34638117 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex11.jl @@ -0,0 +1,55 @@ +# This file was generated, do not modify it. # hide +using LinearAlgebra: eigvals, eigvecs +#source: https://discourse.julialang.org/t/plot-ellipse-in-makie/82814/4 +function getellipsepoints(cx, cy, rx, ry, θ) + t = range(0, 2 * pi; length=100) + ellipse_x_r = @. rx * cos(t) + ellipse_y_r = @. ry * sin(t) + R = [cos(θ) sin(θ); -sin(θ) cos(θ)] + r_ellipse = [ellipse_x_r ellipse_y_r] * R + x = @. cx + r_ellipse[:, 1] + y = @. cy + r_ellipse[:, 2] + return (x, y) +end +function getellipsepoints(μ, Σ; confidence=0.95) + quant = sqrt(quantile(Chisq(2), confidence)) + cx = μ[1] + cy = μ[2] + + egvs = eigvals(Σ) + if egvs[1] > egvs[2] + idxmax = 1 + largestegv = egvs[1] + smallesttegv = egvs[2] + else + idxmax = 2 + largestegv = egvs[2] + smallesttegv = egvs[1] + end + + rx = quant * sqrt(largestegv) + ry = quant * sqrt(smallesttegv) + + eigvecmax = eigvecs(Σ)[:, idxmax] + θ = atan(eigvecmax[2] / eigvecmax[1]) + if θ < 0 + θ += 2 * π + end + + return getellipsepoints(cx, cy, rx, ry, θ) +end + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "met_anim.gif"); framerate=5) do frame + for i in 1:100 + scatter!(ax, (X_met[i, 1], X_met[i, 2]); color=(:red, 0.5)) + linesegments!(X_met[i:(i + 1), 1], X_met[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex12.jl b/assets/pages/05_MCMC/code/ex12.jl new file mode 100644 index 00000000..3103b4c3 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex12.jl @@ -0,0 +1,17 @@ +# This file was generated, do not modify it. # hide +const warmup = 1_000 + +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_met[warmup:(warmup + 1_000), 1], + X_met[warmup:(warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "met_first1000.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex13.jl b/assets/pages/05_MCMC/code/ex13.jl new file mode 100644 index 00000000..55341042 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex13.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_met[warmup:end, 1], X_met[warmup:end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "met_all.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex14.jl b/assets/pages/05_MCMC/code/ex14.jl new file mode 100644 index 00000000..79c37133 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex14.jl @@ -0,0 +1,33 @@ +# This file was generated, do not modify it. # hide +function gibbs( + S::Int64, + ρ::Float64; + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + x = start_x + y = start_y + β = ρ * σ_y / σ_x + λ = ρ * σ_x / σ_y + sqrt1mrho2 = sqrt(1 - ρ^2) + σ_YX = σ_y * sqrt1mrho2 + σ_XY = σ_x * sqrt1mrho2 + @inbounds draws[1, :] = [x y] + for s in 2:S + if s % 2 == 0 + y = rand(rgn, Normal(μ_y + β * (x - μ_x), σ_YX)) + else + x = rand(rgn, Normal(μ_x + λ * (y - μ_y), σ_XY)) + end + @inbounds draws[s, :] = [x y] + end + return draws +end \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex15.jl b/assets/pages/05_MCMC/code/ex15.jl new file mode 100644 index 00000000..0a8eb24a --- /dev/null +++ b/assets/pages/05_MCMC/code/ex15.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +X_gibbs = gibbs(S * 2, ρ); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex16.jl b/assets/pages/05_MCMC/code/ex16.jl new file mode 100644 index 00000000..6f804cb9 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex16.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +X_gibbs[1:10, :] \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex17.jl b/assets/pages/05_MCMC/code/ex17.jl new file mode 100644 index 00000000..988f2856 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex17.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +chain_gibbs = Chains(X_gibbs, [:X, :Y]); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex18.jl b/assets/pages/05_MCMC/code/ex18.jl new file mode 100644 index 00000000..c4db0efd --- /dev/null +++ b/assets/pages/05_MCMC/code/ex18.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summarystats(chain_gibbs) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex19.jl b/assets/pages/05_MCMC/code/ex19.jl new file mode 100644 index 00000000..a09231f2 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex19.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +(mean(summarystats(chain_gibbs)[:, :ess_tail]) / 2) / S \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex2.jl b/assets/pages/05_MCMC/code/ex2.jl new file mode 100644 index 00000000..e0896ea0 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex2.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +const N = 100_000 +const μ = [0, 0] +const Σ = [1 0.8; 0.8 1] + +const mvnormal = MvNormal(μ, Σ) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex20.jl b/assets/pages/05_MCMC/code/ex20.jl new file mode 100644 index 00000000..c30d279b --- /dev/null +++ b/assets/pages/05_MCMC/code/ex20.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "gibbs_anim.gif"); framerate=5) do frame + for i in 1:200 + scatter!(ax, (X_gibbs[i, 1], X_gibbs[i, 2]); color=(:red, 0.5)) + linesegments!(X_gibbs[i:(i + 1), 1], X_gibbs[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex21.jl b/assets/pages/05_MCMC/code/ex21.jl new file mode 100644 index 00000000..ee72a2c3 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex21.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_gibbs[(2 * warmup):(2 * warmup + 1_000), 1], + X_gibbs[(2 * warmup):(2 * warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "gibbs_first1000.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex22.jl b/assets/pages/05_MCMC/code/ex22.jl new file mode 100644 index 00000000..e53ef4d5 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex22.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_gibbs[(2 * warmup):end, 1], X_gibbs[(2 * warmup):end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "gibbs_all.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex23.jl b/assets/pages/05_MCMC/code/ex23.jl new file mode 100644 index 00000000..58f5740f --- /dev/null +++ b/assets/pages/05_MCMC/code/ex23.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +const starts = collect(Iterators.product((-2.5, 2.5), (2.5, -2.5))) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex24.jl b/assets/pages/05_MCMC/code/ex24.jl new file mode 100644 index 00000000..5d68d041 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex24.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +const S_parallel = 100; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex25.jl b/assets/pages/05_MCMC/code/ex25.jl new file mode 100644 index 00000000..6e28b517 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex25.jl @@ -0,0 +1,13 @@ +# This file was generated, do not modify it. # hide +X_met_1 = metropolis( + S_parallel, width, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1]) +); +X_met_2 = metropolis( + S_parallel, width, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2]) +); +X_met_3 = metropolis( + S_parallel, width, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3]) +); +X_met_4 = metropolis( + S_parallel, width, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4]) +); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex26.jl b/assets/pages/05_MCMC/code/ex26.jl new file mode 100644 index 00000000..587a8c51 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex26.jl @@ -0,0 +1,31 @@ +# This file was generated, do not modify it. # hide +using Colors +const logocolors = Colors.JULIA_LOGO_COLORS; +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "parallel_met.gif"); framerate=5) do frame + for i in 1:99 + scatter!(ax, (X_met_1[i, 1], X_met_1[i, 2]); color=(logocolors.blue, 0.5)) + linesegments!( + X_met_1[i:(i + 1), 1], X_met_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5) + ) + scatter!(ax, (X_met_2[i, 1], X_met_2[i, 2]); color=(logocolors.red, 0.5)) + linesegments!( + X_met_2[i:(i + 1), 1], X_met_2[i:(i + 1), 2]; color=(logocolors.red, 0.5) + ) + scatter!(ax, (X_met_3[i, 1], X_met_3[i, 2]); color=(logocolors.green, 0.5)) + linesegments!( + X_met_3[i:(i + 1), 1], X_met_3[i:(i + 1), 2]; color=(logocolors.green, 0.5) + ) + scatter!(ax, (X_met_4[i, 1], X_met_4[i, 2]); color=(logocolors.purple, 0.5)) + linesegments!( + X_met_4[i:(i + 1), 1], X_met_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5) + ) + recordframe!(frame) + end +end; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex27.jl b/assets/pages/05_MCMC/code/ex27.jl new file mode 100644 index 00000000..1aec3306 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex27.jl @@ -0,0 +1,13 @@ +# This file was generated, do not modify it. # hide +X_gibbs_1 = gibbs( + S_parallel * 2, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1]) +); +X_gibbs_2 = gibbs( + S_parallel * 2, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2]) +); +X_gibbs_3 = gibbs( + S_parallel * 2, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3]) +); +X_gibbs_4 = gibbs( + S_parallel * 2, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4]) +); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex28.jl b/assets/pages/05_MCMC/code/ex28.jl new file mode 100644 index 00000000..417b9916 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex28.jl @@ -0,0 +1,29 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "parallel_gibbs.gif"); framerate=5) do frame + for i in 1:199 + scatter!(ax, (X_gibbs_1[i, 1], X_gibbs_1[i, 2]); color=(logocolors.blue, 0.5)) + linesegments!( + X_gibbs_1[i:(i + 1), 1], X_gibbs_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5) + ) + scatter!(ax, (X_gibbs_2[i, 1], X_gibbs_2[i, 2]); color=(logocolors.red, 0.5)) + linesegments!( + X_gibbs_2[i:(i + 1), 1], X_gibbs_2[i:(i + 1), 2]; color=(logocolors.red, 0.5) + ) + scatter!(ax, (X_gibbs_3[i, 1], X_gibbs_3[i, 2]); color=(logocolors.green, 0.5)) + linesegments!( + X_gibbs_3[i:(i + 1), 1], X_gibbs_3[i:(i + 1), 2]; color=(logocolors.green, 0.5) + ) + scatter!(ax, (X_gibbs_4[i, 1], X_gibbs_4[i, 2]); color=(logocolors.purple, 0.5)) + linesegments!( + X_gibbs_4[i:(i + 1), 1], X_gibbs_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5) + ) + recordframe!(frame) + end +end; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex29.jl b/assets/pages/05_MCMC/code/ex29.jl new file mode 100644 index 00000000..34bf2910 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex29.jl @@ -0,0 +1,51 @@ +# This file was generated, do not modify it. # hide +using ForwardDiff: gradient +function hmc( + S::Int64, + width::Float64, + ρ::Float64; + L=40, + ϵ=0.001, + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + accepted = 0::Int64 + x = start_x + y = start_y + @inbounds draws[1, :] = [x y] + M = [1.0 0.0; 0.0 1.0] + ϕ_d = MvNormal([0.0, 0.0], M) + for s in 2:S + x_ = rand(rgn, Uniform(x - width, x + width)) + y_ = rand(rgn, Uniform(y - width, y + width)) + ϕ = rand(rgn, ϕ_d) + kinetic = sum(ϕ .^ 2) / 2 + log_p = logpdf(binormal, [x, y]) - kinetic + ϕ += 0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_]) + for l in 1:L + x_, y_ = [x_, y_] + (ϵ * M * ϕ) + ϕ += +0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_]) + end + ϕ = -ϕ # make the proposal symmetric + kinetic = sum(ϕ .^ 2) / 2 + log_p_ = logpdf(binormal, [x_, y_]) - kinetic + r = exp(log_p_ - log_p) + + if r > rand(rgn, Uniform()) + x = x_ + y = y_ + accepted += 1 + end + @inbounds draws[s, :] = [x y] + end + println("Acceptance rate is: $(accepted / S)") + return draws +end \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex3.jl b/assets/pages/05_MCMC/code/ex3.jl new file mode 100644 index 00000000..17f12f9e --- /dev/null +++ b/assets/pages/05_MCMC/code/ex3.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +x = -3:0.01:3 +y = -3:0.01:3 +dens_mvnormal = [pdf(mvnormal, [i, j]) for i in x, j in y] +f, ax, c = contourf(x, y, dens_mvnormal; axis=(; xlabel=L"X", ylabel=L"Y")) +Colorbar(f[1, 2], c) +save(joinpath(@OUTPUT, "countour_mvnormal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex30.jl b/assets/pages/05_MCMC/code/ex30.jl new file mode 100644 index 00000000..13252189 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex30.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +gradient(x -> logpdf(mvnormal, x), [1, -1]) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex31.jl b/assets/pages/05_MCMC/code/ex31.jl new file mode 100644 index 00000000..bff6cba4 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex31.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +X_hmc = hmc(S, width, ρ; ϵ=0.0856, L=40); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex32.jl b/assets/pages/05_MCMC/code/ex32.jl new file mode 100644 index 00000000..b834489e --- /dev/null +++ b/assets/pages/05_MCMC/code/ex32.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +X_hmc[1:10, :] \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex33.jl b/assets/pages/05_MCMC/code/ex33.jl new file mode 100644 index 00000000..6e1dd911 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex33.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +chain_hmc = Chains(X_hmc, [:X, :Y]); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex34.jl b/assets/pages/05_MCMC/code/ex34.jl new file mode 100644 index 00000000..71b342f8 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex34.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summarystats(chain_hmc) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex35.jl b/assets/pages/05_MCMC/code/ex35.jl new file mode 100644 index 00000000..076ef359 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex35.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +mean(summarystats(chain_hmc)[:, :ess_tail]) / S \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex36.jl b/assets/pages/05_MCMC/code/ex36.jl new file mode 100644 index 00000000..22100363 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex36.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +record(f, joinpath(@OUTPUT, "hmc_anim.gif"); framerate=5) do frame + for i in 1:100 + scatter!(ax, (X_hmc[i, 1], X_hmc[i, 2]); color=(:red, 0.5)) + linesegments!(X_hmc[i:(i + 1), 1], X_hmc[i:(i + 1), 2]; color=(:green, 0.5)) + recordframe!(frame) + end +end; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex37.jl b/assets/pages/05_MCMC/code/ex37.jl new file mode 100644 index 00000000..1c7ed01a --- /dev/null +++ b/assets/pages/05_MCMC/code/ex37.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!( + ax, + X_hmc[warmup:(warmup + 1_000), 1], + X_hmc[warmup:(warmup + 1_000), 2]; + color=(:red, 0.3), +) +save(joinpath(@OUTPUT, "hmc_first1000.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex38.jl b/assets/pages/05_MCMC/code/ex38.jl new file mode 100644 index 00000000..953330dd --- /dev/null +++ b/assets/pages/05_MCMC/code/ex38.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + getellipsepoints(μ, Σ; confidence=0.9)...; + label="90% HPD", + linewidth=2, + axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"), +) +axislegend(ax) +scatter!(ax, X_hmc[warmup:end, 1], X_hmc[warmup:end, 2]; color=(:red, 0.3)) +save(joinpath(@OUTPUT, "hmc_all.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex39.jl b/assets/pages/05_MCMC/code/ex39.jl new file mode 100644 index 00000000..194fa83e --- /dev/null +++ b/assets/pages/05_MCMC/code/ex39.jl @@ -0,0 +1,17 @@ +# This file was generated, do not modify it. # hide +d1 = MvNormal([10, 2], [1 0; 0 1]) +d2 = MvNormal([0, 0], [8.4 2.0; 2.0 1.7]) + +d = MixtureModel([d1, d2]) + +x = -6:0.01:15 +y = -2.5:0.01:4.2 +dens_mixture = [pdf(d, [i, j]) for i in x, j in y] + +f, ax, s = surface( + x, + y, + dens_mixture; + axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 4), +) +save(joinpath(@OUTPUT, "bimodal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex4.jl b/assets/pages/05_MCMC/code/ex4.jl new file mode 100644 index 00000000..85e01a88 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex4.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +f, ax, s = surface( + x, + y, + dens_mvnormal; + axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 8), +) +save(joinpath(@OUTPUT, "surface_mvnormal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex40.jl b/assets/pages/05_MCMC/code/ex40.jl new file mode 100644 index 00000000..d3d8765d --- /dev/null +++ b/assets/pages/05_MCMC/code/ex40.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +funnel_y = rand(Normal(0, 3), 10_000) +funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2) + +f, ax, s = scatter( + funnel_x, + funnel_y; + color=(:steelblue, 0.3), + axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)), +) +save(joinpath(@OUTPUT, "funnel.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex41.jl b/assets/pages/05_MCMC/code/ex41.jl new file mode 100644 index 00000000..ce019658 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex41.jl @@ -0,0 +1,12 @@ +# This file was generated, do not modify it. # hide +using Turing +setprogress!(false) # hide + +@model function dice_throw(y) + #Our prior belief about the probability of each result in a six-sided dice. + #p is a vector of length 6 each with probability p that sums up to 1. + p ~ Dirichlet(6, 1) + + #Each outcome of the six-sided dice has a probability p. + return y ~ filldist(Categorical(p), length(y)) +end; \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex42.jl b/assets/pages/05_MCMC/code/ex42.jl new file mode 100644 index 00000000..9990626d --- /dev/null +++ b/assets/pages/05_MCMC/code/ex42.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +data_dice = rand(DiscreteUniform(1, 6), 1_000); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex43.jl b/assets/pages/05_MCMC/code/ex43.jl new file mode 100644 index 00000000..8361ac5f --- /dev/null +++ b/assets/pages/05_MCMC/code/ex43.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +model = dice_throw(data_dice) +chain = sample(model, NUTS(), 1_000); +summarystats(chain) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex44.jl b/assets/pages/05_MCMC/code/ex44.jl new file mode 100644 index 00000000..082bbb6f --- /dev/null +++ b/assets/pages/05_MCMC/code/ex44.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +bad_chain = sample(model, NUTS(0.3), 500) +summarystats(bad_chain) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex45.jl b/assets/pages/05_MCMC/code/ex45.jl new file mode 100644 index 00000000..deefa117 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex45.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +sum(bad_chain[:numerical_error]) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex46.jl b/assets/pages/05_MCMC/code/ex46.jl new file mode 100644 index 00000000..2a42b03c --- /dev/null +++ b/assets/pages/05_MCMC/code/ex46.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +mean(bad_chain[:acceptance_rate]) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex47.jl b/assets/pages/05_MCMC/code/ex47.jl new file mode 100644 index 00000000..2ae70cf3 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex47.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +mean(chain[:acceptance_rate]) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex48.jl b/assets/pages/05_MCMC/code/ex48.jl new file mode 100644 index 00000000..5ae3ad37 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex48.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +using AlgebraOfGraphics +params = names(chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines) +f = Figure(; resolution=(1200, 900)) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "traceplot_chain.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex49.jl b/assets/pages/05_MCMC/code/ex49.jl new file mode 100644 index 00000000..efb7e5c5 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex49.jl @@ -0,0 +1,9 @@ +# This file was generated, do not modify it. # hide +params = names(bad_chain, :parameters) +chain_mapping = + mapping(params .=> "sample value") * + mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params)) +plt = data(bad_chain) * mapping(:iteration) * chain_mapping * visual(Lines) +f = Figure(; resolution=(1200, 900)) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "traceplot_bad_chain.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex5.jl b/assets/pages/05_MCMC/code/ex5.jl new file mode 100644 index 00000000..a1ac8f11 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex5.jl @@ -0,0 +1,35 @@ +# This file was generated, do not modify it. # hide +function metropolis( + S::Int64, + width::Float64, + ρ::Float64; + μ_x::Float64=0.0, + μ_y::Float64=0.0, + σ_x::Float64=1.0, + σ_y::Float64=1.0, + start_x=-2.5, + start_y=2.5, + seed=123, +) + rgn = MersenneTwister(seed) + binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]) + draws = Matrix{Float64}(undef, S, 2) + accepted = 0::Int64 + x = start_x + y = start_y + @inbounds draws[1, :] = [x y] + for s in 2:S + x_ = rand(rgn, Uniform(x - width, x + width)) + y_ = rand(rgn, Uniform(y - width, y + width)) + r = exp(logpdf(binormal, [x_, y_]) - logpdf(binormal, [x, y])) + + if r > rand(rgn, Uniform()) + x = x_ + y = y_ + accepted += 1 + end + @inbounds draws[s, :] = [x y] + end + println("Acceptance rate is: $(accepted / S)") + return draws +end \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex6.jl b/assets/pages/05_MCMC/code/ex6.jl new file mode 100644 index 00000000..70d7d4bf --- /dev/null +++ b/assets/pages/05_MCMC/code/ex6.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +const S = 10_000 +const width = 2.75 +const ρ = 0.8 + +X_met = metropolis(S, width, ρ); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex7.jl b/assets/pages/05_MCMC/code/ex7.jl new file mode 100644 index 00000000..4523ac8a --- /dev/null +++ b/assets/pages/05_MCMC/code/ex7.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +X_met[1:10, :] \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex8.jl b/assets/pages/05_MCMC/code/ex8.jl new file mode 100644 index 00000000..3fc9a5f7 --- /dev/null +++ b/assets/pages/05_MCMC/code/ex8.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +using MCMCChains + +chain_met = Chains(X_met, [:X, :Y]); \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/ex9.jl b/assets/pages/05_MCMC/code/ex9.jl new file mode 100644 index 00000000..fa6d242e --- /dev/null +++ b/assets/pages/05_MCMC/code/ex9.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +summarystats(chain_met) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/bimodal.svg b/assets/pages/05_MCMC/code/output/bimodal.svg new file mode 100644 index 00000000..50fa3f93 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/bimodal.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/countour_mvnormal.svg b/assets/pages/05_MCMC/code/output/countour_mvnormal.svg new file mode 100644 index 00000000..ede4805a --- /dev/null +++ b/assets/pages/05_MCMC/code/output/countour_mvnormal.svg @@ -0,0 +1,241 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/ex1.out b/assets/pages/05_MCMC/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex1.res b/assets/pages/05_MCMC/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex10.out b/assets/pages/05_MCMC/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex10.res b/assets/pages/05_MCMC/code/output/ex10.res new file mode 100644 index 00000000..da0cad40 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex10.res @@ -0,0 +1 @@ +0.10910286584687137 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex11.out b/assets/pages/05_MCMC/code/output/ex11.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex11.res b/assets/pages/05_MCMC/code/output/ex11.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex11.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex12.out b/assets/pages/05_MCMC/code/output/ex12.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex12.res b/assets/pages/05_MCMC/code/output/ex12.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex12.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex13.out b/assets/pages/05_MCMC/code/output/ex13.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex13.res b/assets/pages/05_MCMC/code/output/ex13.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex13.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex14.out b/assets/pages/05_MCMC/code/output/ex14.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex14.res b/assets/pages/05_MCMC/code/output/ex14.res new file mode 100644 index 00000000..ce9d5378 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex14.res @@ -0,0 +1 @@ +gibbs (generic function with 1 method) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex15.out b/assets/pages/05_MCMC/code/output/ex15.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex15.res b/assets/pages/05_MCMC/code/output/ex15.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex15.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex16.out b/assets/pages/05_MCMC/code/output/ex16.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex16.res b/assets/pages/05_MCMC/code/output/ex16.res new file mode 100644 index 00000000..fbaf1085 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex16.res @@ -0,0 +1,11 @@ +10×2 Matrix{Float64}: + -2.5 2.5 + -2.5 -1.28584 + 0.200236 -1.28584 + 0.200236 0.84578 + 0.952273 0.84578 + 0.952273 0.523811 + 0.0202213 0.523811 + 0.0202213 0.604758 + 0.438516 0.604758 + 0.438516 0.515102 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex17.out b/assets/pages/05_MCMC/code/output/ex17.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex17.res b/assets/pages/05_MCMC/code/output/ex17.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex17.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex18.out b/assets/pages/05_MCMC/code/output/ex18.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex18.res b/assets/pages/05_MCMC/code/output/ex18.res new file mode 100644 index 00000000..b5ac9909 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex18.res @@ -0,0 +1,6 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Missing + + X 0.0161 1.0090 0.0217 2152.7581 3985.5939 1.0005 missing + Y 0.0097 1.0071 0.0219 2114.7477 4038.1490 1.0003 missing diff --git a/assets/pages/05_MCMC/code/output/ex19.out b/assets/pages/05_MCMC/code/output/ex19.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex19.res b/assets/pages/05_MCMC/code/output/ex19.res new file mode 100644 index 00000000..28108f6d --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex19.res @@ -0,0 +1 @@ +0.20059357221090376 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex2.out b/assets/pages/05_MCMC/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex2.res b/assets/pages/05_MCMC/code/output/ex2.res new file mode 100644 index 00000000..a9c79714 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex2.res @@ -0,0 +1,5 @@ +FullNormal( +dim: 2 +μ: [0.0, 0.0] +Σ: [1.0 0.8; 0.8 1.0] +) diff --git a/assets/pages/05_MCMC/code/output/ex20.out b/assets/pages/05_MCMC/code/output/ex20.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex20.res b/assets/pages/05_MCMC/code/output/ex20.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex20.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex21.out b/assets/pages/05_MCMC/code/output/ex21.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex21.res b/assets/pages/05_MCMC/code/output/ex21.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex21.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex22.out b/assets/pages/05_MCMC/code/output/ex22.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex22.res b/assets/pages/05_MCMC/code/output/ex22.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex22.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex23.out b/assets/pages/05_MCMC/code/output/ex23.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex23.res b/assets/pages/05_MCMC/code/output/ex23.res new file mode 100644 index 00000000..f7c132e5 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex23.res @@ -0,0 +1,3 @@ +2×2 Matrix{Tuple{Float64, Float64}}: + (-2.5, 2.5) (-2.5, -2.5) + (2.5, 2.5) (2.5, -2.5) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex24.out b/assets/pages/05_MCMC/code/output/ex24.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex24.res b/assets/pages/05_MCMC/code/output/ex24.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex24.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex25.out b/assets/pages/05_MCMC/code/output/ex25.out new file mode 100644 index 00000000..c19593fd --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex25.out @@ -0,0 +1,4 @@ +Acceptance rate is: 0.24 +Acceptance rate is: 0.18 +Acceptance rate is: 0.18 +Acceptance rate is: 0.23 diff --git a/assets/pages/05_MCMC/code/output/ex25.res b/assets/pages/05_MCMC/code/output/ex25.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex25.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex26.out b/assets/pages/05_MCMC/code/output/ex26.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex26.res b/assets/pages/05_MCMC/code/output/ex26.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex26.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex27.out b/assets/pages/05_MCMC/code/output/ex27.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex27.res b/assets/pages/05_MCMC/code/output/ex27.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex27.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex28.out b/assets/pages/05_MCMC/code/output/ex28.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex28.res b/assets/pages/05_MCMC/code/output/ex28.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex28.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex29.out b/assets/pages/05_MCMC/code/output/ex29.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex29.res b/assets/pages/05_MCMC/code/output/ex29.res new file mode 100644 index 00000000..41432ed8 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex29.res @@ -0,0 +1 @@ +hmc (generic function with 1 method) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex3.out b/assets/pages/05_MCMC/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex3.res b/assets/pages/05_MCMC/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex30.out b/assets/pages/05_MCMC/code/output/ex30.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex30.res b/assets/pages/05_MCMC/code/output/ex30.res new file mode 100644 index 00000000..25f575f1 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex30.res @@ -0,0 +1,3 @@ +2-element Vector{Float64}: + -5.000000000000003 + 5.000000000000003 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex31.out b/assets/pages/05_MCMC/code/output/ex31.out new file mode 100644 index 00000000..eee4a62a --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex31.out @@ -0,0 +1 @@ +Acceptance rate is: 0.2079 diff --git a/assets/pages/05_MCMC/code/output/ex31.res b/assets/pages/05_MCMC/code/output/ex31.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex31.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex32.out b/assets/pages/05_MCMC/code/output/ex32.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex32.res b/assets/pages/05_MCMC/code/output/ex32.res new file mode 100644 index 00000000..d2ddc111 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex32.res @@ -0,0 +1,11 @@ +10×2 Matrix{Float64}: + -2.5 2.5 + -2.5 2.5 + -1.50904 1.52822 + -1.50904 1.52822 + -1.85829 -0.464346 + -0.739967 -1.15873 + -0.739967 -1.15873 + -0.739967 -1.15873 + -0.739967 -1.15873 + -0.739967 -1.15873 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex33.out b/assets/pages/05_MCMC/code/output/ex33.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex33.res b/assets/pages/05_MCMC/code/output/ex33.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex33.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex34.out b/assets/pages/05_MCMC/code/output/ex34.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex34.res b/assets/pages/05_MCMC/code/output/ex34.res new file mode 100644 index 00000000..2a8c42cd --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex34.res @@ -0,0 +1,6 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Missing + + X 0.0169 0.9373 0.0222 1738.2585 1537.0420 1.0014 missing + Y 0.0073 0.9475 0.0237 1556.0589 1640.5275 1.0003 missing diff --git a/assets/pages/05_MCMC/code/output/ex35.out b/assets/pages/05_MCMC/code/output/ex35.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex35.res b/assets/pages/05_MCMC/code/output/ex35.res new file mode 100644 index 00000000..e2711c36 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex35.res @@ -0,0 +1 @@ +0.15887847790933807 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex36.out b/assets/pages/05_MCMC/code/output/ex36.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex36.res b/assets/pages/05_MCMC/code/output/ex36.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex36.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex37.out b/assets/pages/05_MCMC/code/output/ex37.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex37.res b/assets/pages/05_MCMC/code/output/ex37.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex37.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex38.out b/assets/pages/05_MCMC/code/output/ex38.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex38.res b/assets/pages/05_MCMC/code/output/ex38.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex38.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex39.out b/assets/pages/05_MCMC/code/output/ex39.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex39.res b/assets/pages/05_MCMC/code/output/ex39.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex39.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex4.out b/assets/pages/05_MCMC/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex4.res b/assets/pages/05_MCMC/code/output/ex4.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex4.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex40.out b/assets/pages/05_MCMC/code/output/ex40.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex40.res b/assets/pages/05_MCMC/code/output/ex40.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex40.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex41.out b/assets/pages/05_MCMC/code/output/ex41.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex41.res b/assets/pages/05_MCMC/code/output/ex41.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex41.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex42.out b/assets/pages/05_MCMC/code/output/ex42.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex42.res b/assets/pages/05_MCMC/code/output/ex42.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex42.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex43.out b/assets/pages/05_MCMC/code/output/ex43.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex43.res b/assets/pages/05_MCMC/code/output/ex43.res new file mode 100644 index 00000000..da309501 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex43.res @@ -0,0 +1,10 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + p[1] 0.1689 0.0118 0.0003 1965.3374 876.7033 0.9995 613.7843 + p[2] 0.1661 0.0119 0.0003 1419.7589 739.3257 1.0001 443.3975 + p[3] 0.1600 0.0108 0.0003 1594.9950 873.7550 0.9990 498.1246 + p[4] 0.1700 0.0116 0.0003 1998.9151 884.6355 1.0045 624.2708 + p[5] 0.1710 0.0117 0.0003 1670.1179 813.8896 0.9994 521.5858 + p[6] 0.1641 0.0120 0.0003 2055.8182 792.1886 1.0011 642.0419 diff --git a/assets/pages/05_MCMC/code/output/ex44.out b/assets/pages/05_MCMC/code/output/ex44.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex44.res b/assets/pages/05_MCMC/code/output/ex44.res new file mode 100644 index 00000000..331c48a2 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex44.res @@ -0,0 +1,10 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + p[1] 0.1684 0.0098 0.0029 8.6088 16.0774 1.1119 128.4895 + p[2] 0.1657 0.0130 0.0029 12.6348 NaN 1.0553 188.5793 + p[3] 0.1637 0.0112 0.0025 20.4216 26.6760 1.2401 304.7994 + p[4] 0.1719 0.0102 0.0022 27.0245 23.5726 1.0613 403.3510 + p[5] 0.1728 0.0181 0.0056 13.6728 13.2203 1.1900 204.0718 + p[6] 0.1575 0.0175 0.0059 6.4771 3.5114 1.1389 96.6725 diff --git a/assets/pages/05_MCMC/code/output/ex45.out b/assets/pages/05_MCMC/code/output/ex45.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex45.res b/assets/pages/05_MCMC/code/output/ex45.res new file mode 100644 index 00000000..7f83a844 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex45.res @@ -0,0 +1 @@ +17.0 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex46.out b/assets/pages/05_MCMC/code/output/ex46.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex46.res b/assets/pages/05_MCMC/code/output/ex46.res new file mode 100644 index 00000000..539077fa --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex46.res @@ -0,0 +1 @@ +0.018655899447476025 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex47.out b/assets/pages/05_MCMC/code/output/ex47.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex47.res b/assets/pages/05_MCMC/code/output/ex47.res new file mode 100644 index 00000000..0bb85db7 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex47.res @@ -0,0 +1 @@ +0.7934378228300696 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex48.out b/assets/pages/05_MCMC/code/output/ex48.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex48.res b/assets/pages/05_MCMC/code/output/ex48.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex48.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex49.out b/assets/pages/05_MCMC/code/output/ex49.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex49.res b/assets/pages/05_MCMC/code/output/ex49.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex49.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex5.out b/assets/pages/05_MCMC/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex5.res b/assets/pages/05_MCMC/code/output/ex5.res new file mode 100644 index 00000000..be855cdd --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex5.res @@ -0,0 +1 @@ +metropolis (generic function with 1 method) \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex6.out b/assets/pages/05_MCMC/code/output/ex6.out new file mode 100644 index 00000000..6b34616a --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex6.out @@ -0,0 +1 @@ +Acceptance rate is: 0.2116 diff --git a/assets/pages/05_MCMC/code/output/ex6.res b/assets/pages/05_MCMC/code/output/ex6.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex6.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex7.out b/assets/pages/05_MCMC/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex7.res b/assets/pages/05_MCMC/code/output/ex7.res new file mode 100644 index 00000000..054a569f --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex7.res @@ -0,0 +1,11 @@ +10×2 Matrix{Float64}: + -2.5 2.5 + -2.5 2.5 + -3.07501 1.47284 + -2.60189 -0.990426 + -2.60189 -0.990426 + -0.592119 -0.34422 + -0.592119 -0.34422 + -0.592119 -0.34422 + -0.592119 -0.34422 + -0.592119 -0.34422 \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex8.out b/assets/pages/05_MCMC/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex8.res b/assets/pages/05_MCMC/code/output/ex8.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex8.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/05_MCMC/code/output/ex9.out b/assets/pages/05_MCMC/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/05_MCMC/code/output/ex9.res b/assets/pages/05_MCMC/code/output/ex9.res new file mode 100644 index 00000000..1c5d389d --- /dev/null +++ b/assets/pages/05_MCMC/code/output/ex9.res @@ -0,0 +1,6 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Missing + + X -0.0314 0.9984 0.0313 1022.8896 1135.7248 0.9999 missing + Y -0.0334 0.9784 0.0305 1028.0976 1046.3325 0.9999 missing diff --git a/assets/pages/05_MCMC/code/output/funnel.svg b/assets/pages/05_MCMC/code/output/funnel.svg new file mode 100644 index 00000000..ba848f77 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/funnel.svg @@ -0,0 +1,10205 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/gibbs_all.svg b/assets/pages/05_MCMC/code/output/gibbs_all.svg new file mode 100644 index 00000000..a77125ca --- /dev/null +++ b/assets/pages/05_MCMC/code/output/gibbs_all.svg @@ -0,0 +1,18335 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/gibbs_anim.gif b/assets/pages/05_MCMC/code/output/gibbs_anim.gif new file mode 100644 index 00000000..7f887877 Binary files /dev/null and b/assets/pages/05_MCMC/code/output/gibbs_anim.gif differ diff --git a/assets/pages/05_MCMC/code/output/gibbs_first1000.svg b/assets/pages/05_MCMC/code/output/gibbs_first1000.svg new file mode 100644 index 00000000..2d16f871 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/gibbs_first1000.svg @@ -0,0 +1,1264 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/hmc_all.svg b/assets/pages/05_MCMC/code/output/hmc_all.svg new file mode 100644 index 00000000..85f73e7a --- /dev/null +++ b/assets/pages/05_MCMC/code/output/hmc_all.svg @@ -0,0 +1,9269 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/hmc_anim.gif b/assets/pages/05_MCMC/code/output/hmc_anim.gif new file mode 100644 index 00000000..25f6d211 Binary files /dev/null and b/assets/pages/05_MCMC/code/output/hmc_anim.gif differ diff --git a/assets/pages/05_MCMC/code/output/hmc_first1000.svg b/assets/pages/05_MCMC/code/output/hmc_first1000.svg new file mode 100644 index 00000000..ef15811c --- /dev/null +++ b/assets/pages/05_MCMC/code/output/hmc_first1000.svg @@ -0,0 +1,1263 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/met_all.svg b/assets/pages/05_MCMC/code/output/met_all.svg new file mode 100644 index 00000000..a95036fd --- /dev/null +++ b/assets/pages/05_MCMC/code/output/met_all.svg @@ -0,0 +1,9232 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/met_anim.gif b/assets/pages/05_MCMC/code/output/met_anim.gif new file mode 100644 index 00000000..bed2a5cc Binary files /dev/null and b/assets/pages/05_MCMC/code/output/met_anim.gif differ diff --git a/assets/pages/05_MCMC/code/output/met_first1000.svg b/assets/pages/05_MCMC/code/output/met_first1000.svg new file mode 100644 index 00000000..0467b848 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/met_first1000.svg @@ -0,0 +1,1254 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/parallel_gibbs.gif b/assets/pages/05_MCMC/code/output/parallel_gibbs.gif new file mode 100644 index 00000000..8f16c887 Binary files /dev/null and b/assets/pages/05_MCMC/code/output/parallel_gibbs.gif differ diff --git a/assets/pages/05_MCMC/code/output/parallel_met.gif b/assets/pages/05_MCMC/code/output/parallel_met.gif new file mode 100644 index 00000000..a96ba69f Binary files /dev/null and b/assets/pages/05_MCMC/code/output/parallel_met.gif differ diff --git a/assets/pages/05_MCMC/code/output/surface_mvnormal.svg b/assets/pages/05_MCMC/code/output/surface_mvnormal.svg new file mode 100644 index 00000000..ec78f399 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/surface_mvnormal.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/traceplot_bad_chain.svg b/assets/pages/05_MCMC/code/output/traceplot_bad_chain.svg new file mode 100644 index 00000000..ee25b7c7 --- /dev/null +++ b/assets/pages/05_MCMC/code/output/traceplot_bad_chain.svg @@ -0,0 +1,888 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/05_MCMC/code/output/traceplot_chain.svg b/assets/pages/05_MCMC/code/output/traceplot_chain.svg new file mode 100644 index 00000000..acab3f1c --- /dev/null +++ b/assets/pages/05_MCMC/code/output/traceplot_chain.svg @@ -0,0 +1,1014 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/06_linear_reg/code/ex1.jl b/assets/pages/06_linear_reg/code/ex1.jl new file mode 100644 index 00000000..42b2dec3 --- /dev/null +++ b/assets/pages/06_linear_reg/code/ex1.jl @@ -0,0 +1,17 @@ +# This file was generated, do not modify it. # hide +using Turing +using LinearAlgebra: I +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function linreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + + #likelihood + return y ~ MvNormal(α .+ X * β, σ^2 * I) +end; \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/ex2.jl b/assets/pages/06_linear_reg/code/ex2.jl new file mode 100644 index 00000000..9201c961 --- /dev/null +++ b/assets/pages/06_linear_reg/code/ex2.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv" +kidiq = CSV.read(HTTP.get(url).body, DataFrame) +describe(kidiq) \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/ex3.jl b/assets/pages/06_linear_reg/code/ex3.jl new file mode 100644 index 00000000..ee877096 --- /dev/null +++ b/assets/pages/06_linear_reg/code/ex3.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +X = Matrix(select(kidiq, Not(:kid_score))) +y = kidiq[:, :kid_score] +model = linreg(X, y); \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/ex4.jl b/assets/pages/06_linear_reg/code/ex4.jl new file mode 100644 index 00000000..468b4763 --- /dev/null +++ b/assets/pages/06_linear_reg/code/ex4.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/ex5.jl b/assets/pages/06_linear_reg/code/ex5.jl new file mode 100644 index 00000000..ac887937 --- /dev/null +++ b/assets/pages/06_linear_reg/code/ex5.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +quantile(chain) \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/output/ex1.out b/assets/pages/06_linear_reg/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/06_linear_reg/code/output/ex1.res b/assets/pages/06_linear_reg/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/06_linear_reg/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/output/ex2.out b/assets/pages/06_linear_reg/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/06_linear_reg/code/output/ex2.res b/assets/pages/06_linear_reg/code/output/ex2.res new file mode 100644 index 00000000..f36a49a2 --- /dev/null +++ b/assets/pages/06_linear_reg/code/output/ex2.res @@ -0,0 +1,8 @@ +4×7 DataFrame + Row │ variable mean min median max nmissing eltype + │ Symbol Float64 Real Float64 Real Int64 DataType +─────┼────────────────────────────────────────────────────────────────────── + 1 │ kid_score 86.7972 20 90.0 144 0 Int64 + 2 │ mom_hs 0.785714 0 1.0 1 0 Int64 + 3 │ mom_iq 100.0 71.0374 97.9153 138.893 0 Float64 + 4 │ mom_age 22.7857 17 23.0 29 0 Int64 \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/output/ex3.out b/assets/pages/06_linear_reg/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/06_linear_reg/code/output/ex3.res b/assets/pages/06_linear_reg/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/06_linear_reg/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/06_linear_reg/code/output/ex4.out b/assets/pages/06_linear_reg/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/06_linear_reg/code/output/ex4.res b/assets/pages/06_linear_reg/code/output/ex4.res new file mode 100644 index 00000000..a46292e4 --- /dev/null +++ b/assets/pages/06_linear_reg/code/output/ex4.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 21.5126 8.6720 0.2217 1528.8886 1868.2612 1.0009 24.7261 + β[1] 2.0014 1.7943 0.0419 2281.1940 1734.6512 1.0016 36.8928 + β[2] 0.5788 0.0584 0.0013 2163.9754 2292.8814 1.0006 34.9971 + β[3] 0.2566 0.3092 0.0074 1762.0214 2135.6795 1.0010 28.4965 + σ 17.8859 0.6033 0.0106 3271.1669 2347.2435 1.0008 52.9033 diff --git a/assets/pages/06_linear_reg/code/output/ex5.out b/assets/pages/06_linear_reg/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/06_linear_reg/code/output/ex5.res b/assets/pages/06_linear_reg/code/output/ex5.res new file mode 100644 index 00000000..09cbfc17 --- /dev/null +++ b/assets/pages/06_linear_reg/code/output/ex5.res @@ -0,0 +1,9 @@ +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α 4.7278 15.7633 21.2942 27.4322 38.4426 + β[1] -0.5876 0.7324 1.6761 2.9919 6.3388 + β[2] 0.4662 0.5392 0.5793 0.6184 0.6924 + β[3] -0.3477 0.0440 0.2588 0.4733 0.8490 + σ 16.7525 17.4685 17.8796 18.2703 19.1238 diff --git a/assets/pages/07_logistic_reg/code/ex1.jl b/assets/pages/07_logistic_reg/code/ex1.jl new file mode 100644 index 00000000..c26ac585 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex1.jl @@ -0,0 +1,10 @@ +# This file was generated, do not modify it. # hide +using CairoMakie + +function logistic(x) + return 1 / (1 + exp(-x)) +end + +f, ax, l = lines(-10 .. 10, logistic; axis=(; xlabel=L"x", ylabel=L"\mathrm{Logistic}(x)")) +f +save(joinpath(@OUTPUT, "logistic.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/ex2.jl b/assets/pages/07_logistic_reg/code/ex2.jl new file mode 100644 index 00000000..9f6ab519 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex2.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function logreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + + #likelihood + return y ~ arraydist(LazyArray(@~ BernoulliLogit.(α .+ X * β))) +end; \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/ex3.jl b/assets/pages/07_logistic_reg/code/ex3.jl new file mode 100644 index 00000000..333d431f --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex3.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/wells.csv" +wells = CSV.read(HTTP.get(url).body, DataFrame) +describe(wells) \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/ex4.jl b/assets/pages/07_logistic_reg/code/ex4.jl new file mode 100644 index 00000000..35c3c07d --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex4.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +X = Matrix(select(wells, Not(:switch))) +y = wells[:, :switch] +model = logreg(X, y); \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/ex5.jl b/assets/pages/07_logistic_reg/code/ex5.jl new file mode 100644 index 00000000..468b4763 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex5.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/ex6.jl b/assets/pages/07_logistic_reg/code/ex6.jl new file mode 100644 index 00000000..5c0d6741 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex6.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +using Chain + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/ex7.jl b/assets/pages/07_logistic_reg/code/ex7.jl new file mode 100644 index 00000000..4307e368 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/ex7.jl @@ -0,0 +1,9 @@ +# This file was generated, do not modify it. # hide +function logodds2prob(logodds::Float64) + return exp(logodds) / (1 + exp(logodds)) +end + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false) +end \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/ex1.out b/assets/pages/07_logistic_reg/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex1.res b/assets/pages/07_logistic_reg/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/ex2.out b/assets/pages/07_logistic_reg/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex2.res b/assets/pages/07_logistic_reg/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/ex3.out b/assets/pages/07_logistic_reg/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex3.res b/assets/pages/07_logistic_reg/code/output/ex3.res new file mode 100644 index 00000000..c9b41cbe --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex3.res @@ -0,0 +1,9 @@ +5×7 DataFrame + Row │ variable mean min median max nmissing eltype + │ Symbol Float64 Real Float64 Real Int64 DataType +─────┼────────────────────────────────────────────────────────────────── + 1 │ switch 0.575166 0 1.0 1 0 Int64 + 2 │ arsenic 1.65693 0.51 1.3 9.65 0 Float64 + 3 │ dist 48.3319 0.387 36.7615 339.531 0 Float64 + 4 │ assoc 0.422848 0 0.0 1 0 Int64 + 5 │ educ 4.82848 0 5.0 17 0 Int64 \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/ex4.out b/assets/pages/07_logistic_reg/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex4.res b/assets/pages/07_logistic_reg/code/output/ex4.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex4.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/ex5.out b/assets/pages/07_logistic_reg/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex5.res b/assets/pages/07_logistic_reg/code/output/ex5.res new file mode 100644 index 00000000..5e5b88b1 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex5.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α -0.1537 0.1008 0.0026 1487.6006 2091.0499 1.0020 26.5174 + β[1] 0.4664 0.0419 0.0009 2237.3707 2405.0432 1.0010 39.8825 + β[2] -0.0090 0.0010 0.0000 4269.6543 3192.4775 1.0009 76.1093 + β[3] -0.1226 0.0777 0.0019 1680.2431 1877.4329 1.0002 29.9514 + β[4] 0.0424 0.0096 0.0002 2656.3110 2520.0618 1.0020 47.3504 diff --git a/assets/pages/07_logistic_reg/code/output/ex6.out b/assets/pages/07_logistic_reg/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex6.res b/assets/pages/07_logistic_reg/code/output/ex6.res new file mode 100644 index 00000000..97efcbed --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex6.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ parameters 2.5% 25.0% 50.0% 75.0% 97.5% + │ Symbol Float64 Float64 Float64 Float64 Float64 +─────┼────────────────────────────────────────────────────────────── + 1 │ α 0.700906 0.801908 0.859547 0.91731 1.04274 + 2 │ β[1] 1.47317 1.54865 1.59402 1.63747 1.7351 + 3 │ β[2] 0.989048 0.990354 0.991039 0.991766 0.993111 + 4 │ β[3] 0.75859 0.84017 0.885127 0.931194 1.03098 + 5 │ β[4] 1.0235 1.03671 1.04353 1.0502 1.06242 \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/ex7.out b/assets/pages/07_logistic_reg/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/07_logistic_reg/code/output/ex7.res b/assets/pages/07_logistic_reg/code/output/ex7.res new file mode 100644 index 00000000..d292c543 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/ex7.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ parameters 2.5% 25.0% 50.0% 75.0% 97.5% + │ Symbol Float64 Float64 Float64 Float64 Float64 +─────┼────────────────────────────────────────────────────────────── + 1 │ α 0.412078 0.445033 0.462234 0.478436 0.510462 + 2 │ β[1] 0.59566 0.607635 0.614497 0.620848 0.634383 + 3 │ β[2] 0.497247 0.497577 0.49775 0.497933 0.498272 + 4 │ β[3] 0.431363 0.456572 0.469532 0.482186 0.507626 + 5 │ β[4] 0.505807 0.509012 0.510649 0.512243 0.515133 \ No newline at end of file diff --git a/assets/pages/07_logistic_reg/code/output/logistic.svg b/assets/pages/07_logistic_reg/code/output/logistic.svg new file mode 100644 index 00000000..7675ee60 --- /dev/null +++ b/assets/pages/07_logistic_reg/code/output/logistic.svg @@ -0,0 +1,248 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/08_ordinal_reg/code/ex1.jl b/assets/pages/08_ordinal_reg/code/ex1.jl new file mode 100644 index 00000000..2c0af7db --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex1.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CairoMakie +using AlgebraOfGraphics +using Distributions +using StatsFuns: logit \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex2.jl b/assets/pages/08_ordinal_reg/code/ex2.jl new file mode 100644 index 00000000..90984b98 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex2.jl @@ -0,0 +1,22 @@ +# This file was generated, do not modify it. # hide +let + probs = [0.10, 0.15, 0.33, 0.25, 0.10, 0.07] + dist = Categorical(probs) + x = 1:length(probs) + x_pmf = pdf.(dist, x) + x_cdf = cdf.(dist, x) + x_logodds_cdf = logit.(x_cdf) + df = DataFrame(; x, x_pmf, x_cdf, x_logodds_cdf) + labels = ["CDF", "Log-cumulative-odds"] + f = Figure() + plt1 = data(df) * mapping(:x, :x_pmf) * visual(BarPlot) + plt2 = + data(df) * + mapping(:x, [:x_cdf, :x_logodds_cdf]; col=dims(1) => renamer(labels)) * + visual(ScatterLines) + axis = (; xticks=1:6) + draw!(f[1, 2:3], plt1; axis) + draw!(f[2, 1:4], plt2; axis, facet=(; linkyaxes=:none)) + f + save(joinpath(@OUTPUT, "logodds.svg"), f) # hide +end \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex3.jl b/assets/pages/08_ordinal_reg/code/ex3.jl new file mode 100644 index 00000000..bb639914 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex3.jl @@ -0,0 +1,18 @@ +# This file was generated, do not modify it. # hide +using Turing +using Bijectors +using LazyArrays +using LinearAlgebra +using Random: seed! +using Bijectors: transformed, OrderedBijector +seed!(123) +setprogress!(false) # hide + +@model function ordreg(X, y; predictors=size(X, 2), ncateg=maximum(y)) + #priors + cutpoints ~ transformed(filldist(TDist(3) * 5, ncateg - 1), OrderedBijector()) + β ~ filldist(TDist(3) * 2.5, predictors) + + #likelihood + return y ~ arraydist([OrderedLogistic(X[i, :] ⋅ β, cutpoints) for i in 1:length(y)]) +end; \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex4.jl b/assets/pages/08_ordinal_reg/code/ex4.jl new file mode 100644 index 00000000..50f197e3 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex4.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/esoph.csv" +esoph = CSV.read(HTTP.get(url).body, DataFrame) \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex5.jl b/assets/pages/08_ordinal_reg/code/ex5.jl new file mode 100644 index 00000000..5511034d --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex5.jl @@ -0,0 +1,22 @@ +# This file was generated, do not modify it. # hide +using CategoricalArrays + +DataFrames.transform!( + esoph, + :agegp => + x -> categorical( + x; levels=["25-34", "35-44", "45-54", "55-64", "65-74", "75+"], ordered=true + ), + :alcgp => + x -> categorical(x; levels=["0-39g/day", "40-79", "80-119", "120+"], ordered=true), + :tobgp => + x -> categorical(x; levels=["0-9g/day", "10-19", "20-29", "30+"], ordered=true); + renamecols=false, +) +DataFrames.transform!( + esoph, [:agegp, :alcgp, :tobgp] .=> ByRow(levelcode); renamecols=false +) + +X = Matrix(select(esoph, [:agegp, :alcgp])) +y = esoph[:, :tobgp] +model = ordreg(X, y); \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex6.jl b/assets/pages/08_ordinal_reg/code/ex6.jl new file mode 100644 index 00000000..468b4763 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex6.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex7.jl b/assets/pages/08_ordinal_reg/code/ex7.jl new file mode 100644 index 00000000..5c0d6741 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex7.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +using Chain + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/ex8.jl b/assets/pages/08_ordinal_reg/code/ex8.jl new file mode 100644 index 00000000..4307e368 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/ex8.jl @@ -0,0 +1,9 @@ +# This file was generated, do not modify it. # hide +function logodds2prob(logodds::Float64) + return exp(logodds) / (1 + exp(logodds)) +end + +@chain quantile(chain) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false) +end \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/ex1.out b/assets/pages/08_ordinal_reg/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex1.res b/assets/pages/08_ordinal_reg/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/ex2.out b/assets/pages/08_ordinal_reg/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex2.res b/assets/pages/08_ordinal_reg/code/output/ex2.res new file mode 100644 index 00000000..7706520e --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex2.res @@ -0,0 +1 @@ +CairoMakie.Screen{SVG} diff --git a/assets/pages/08_ordinal_reg/code/output/ex3.out b/assets/pages/08_ordinal_reg/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex3.res b/assets/pages/08_ordinal_reg/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/ex4.out b/assets/pages/08_ordinal_reg/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex4.res b/assets/pages/08_ordinal_reg/code/output/ex4.res new file mode 100644 index 00000000..44d74a37 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex4.res @@ -0,0 +1,92 @@ +88×5 DataFrame + Row │ agegp alcgp tobgp ncases ncontrols + │ String7 String15 String15 Int64 Int64 +─────┼───────────────────────────────────────────────── + 1 │ 25-34 0-39g/day 0-9g/day 0 40 + 2 │ 25-34 0-39g/day 10-19 0 10 + 3 │ 25-34 0-39g/day 20-29 0 6 + 4 │ 25-34 0-39g/day 30+ 0 5 + 5 │ 25-34 40-79 0-9g/day 0 27 + 6 │ 25-34 40-79 10-19 0 7 + 7 │ 25-34 40-79 20-29 0 4 + 8 │ 25-34 40-79 30+ 0 7 + 9 │ 25-34 80-119 0-9g/day 0 2 + 10 │ 25-34 80-119 10-19 0 1 + 11 │ 25-34 80-119 30+ 0 2 + 12 │ 25-34 120+ 0-9g/day 0 1 + 13 │ 25-34 120+ 10-19 1 0 + 14 │ 25-34 120+ 20-29 0 1 + 15 │ 25-34 120+ 30+ 0 2 + 16 │ 35-44 0-39g/day 0-9g/day 0 60 + 17 │ 35-44 0-39g/day 10-19 1 13 + 18 │ 35-44 0-39g/day 20-29 0 7 + 19 │ 35-44 0-39g/day 30+ 0 8 + 20 │ 35-44 40-79 0-9g/day 0 35 + 21 │ 35-44 40-79 10-19 3 20 + 22 │ 35-44 40-79 20-29 1 13 + 23 │ 35-44 40-79 30+ 0 8 + 24 │ 35-44 80-119 0-9g/day 0 11 + 25 │ 35-44 80-119 10-19 0 6 + 26 │ 35-44 80-119 20-29 0 2 + 27 │ 35-44 80-119 30+ 0 1 + 28 │ 35-44 120+ 0-9g/day 2 1 + 29 │ 35-44 120+ 10-19 0 3 + 30 │ 35-44 120+ 20-29 2 2 + 31 │ 45-54 0-39g/day 0-9g/day 1 45 + 32 │ 45-54 0-39g/day 10-19 0 18 + 33 │ 45-54 0-39g/day 20-29 0 10 + 34 │ 45-54 0-39g/day 30+ 0 4 + 35 │ 45-54 40-79 0-9g/day 6 32 + 36 │ 45-54 40-79 10-19 4 17 + 37 │ 45-54 40-79 20-29 5 10 + 38 │ 45-54 40-79 30+ 5 2 + 39 │ 45-54 80-119 0-9g/day 3 13 + 40 │ 45-54 80-119 10-19 6 8 + 41 │ 45-54 80-119 20-29 1 4 + 42 │ 45-54 80-119 30+ 2 2 + 43 │ 45-54 120+ 0-9g/day 4 0 + 44 │ 45-54 120+ 10-19 3 1 + 45 │ 45-54 120+ 20-29 2 1 + 46 │ 45-54 120+ 30+ 4 0 + 47 │ 55-64 0-39g/day 0-9g/day 2 47 + 48 │ 55-64 0-39g/day 10-19 3 19 + 49 │ 55-64 0-39g/day 20-29 3 9 + 50 │ 55-64 0-39g/day 30+ 4 2 + 51 │ 55-64 40-79 0-9g/day 9 31 + 52 │ 55-64 40-79 10-19 6 15 + 53 │ 55-64 40-79 20-29 4 13 + 54 │ 55-64 40-79 30+ 3 3 + 55 │ 55-64 80-119 0-9g/day 9 9 + 56 │ 55-64 80-119 10-19 8 7 + 57 │ 55-64 80-119 20-29 3 3 + 58 │ 55-64 80-119 30+ 4 0 + 59 │ 55-64 120+ 0-9g/day 5 5 + 60 │ 55-64 120+ 10-19 6 1 + 61 │ 55-64 120+ 20-29 2 1 + 62 │ 55-64 120+ 30+ 5 1 + 63 │ 65-74 0-39g/day 0-9g/day 5 43 + 64 │ 65-74 0-39g/day 10-19 4 10 + 65 │ 65-74 0-39g/day 20-29 2 5 + 66 │ 65-74 0-39g/day 30+ 0 2 + 67 │ 65-74 40-79 0-9g/day 17 17 + 68 │ 65-74 40-79 10-19 3 7 + 69 │ 65-74 40-79 20-29 5 4 + 70 │ 65-74 80-119 0-9g/day 6 7 + 71 │ 65-74 80-119 10-19 4 8 + 72 │ 65-74 80-119 20-29 2 1 + 73 │ 65-74 80-119 30+ 1 0 + 74 │ 65-74 120+ 0-9g/day 3 1 + 75 │ 65-74 120+ 10-19 1 1 + 76 │ 65-74 120+ 20-29 1 0 + 77 │ 65-74 120+ 30+ 1 0 + 78 │ 75+ 0-39g/day 0-9g/day 1 17 + 79 │ 75+ 0-39g/day 10-19 2 4 + 80 │ 75+ 0-39g/day 30+ 1 2 + 81 │ 75+ 40-79 0-9g/day 2 3 + 82 │ 75+ 40-79 10-19 1 2 + 83 │ 75+ 40-79 20-29 0 3 + 84 │ 75+ 40-79 30+ 1 0 + 85 │ 75+ 80-119 0-9g/day 1 0 + 86 │ 75+ 80-119 10-19 1 0 + 87 │ 75+ 120+ 0-9g/day 2 0 + 88 │ 75+ 120+ 10-19 1 0 \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/ex5.out b/assets/pages/08_ordinal_reg/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex5.res b/assets/pages/08_ordinal_reg/code/output/ex5.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex5.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/ex6.out b/assets/pages/08_ordinal_reg/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex6.res b/assets/pages/08_ordinal_reg/code/output/ex6.res new file mode 100644 index 00000000..ae472d93 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex6.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + cutpoints[1] -1.3806 0.6268 0.0119 2788.6718 2472.5534 1.0009 69.0744 + cutpoints[2] -0.2021 0.6072 0.0113 2897.1099 2438.5542 1.0013 71.7604 + cutpoints[3] 0.8493 0.6203 0.0114 2962.0272 2770.1618 1.0021 73.3684 + β[1] -0.0696 0.1161 0.0021 3108.6746 2347.4018 1.0007 77.0008 + β[2] -0.0658 0.1691 0.0030 3128.0391 2909.9287 1.0016 77.4804 diff --git a/assets/pages/08_ordinal_reg/code/output/ex7.out b/assets/pages/08_ordinal_reg/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex7.res b/assets/pages/08_ordinal_reg/code/output/ex7.res new file mode 100644 index 00000000..c1601bc7 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex7.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ parameters 2.5% 25.0% 50.0% 75.0% 97.5% + │ Symbol Float64 Float64 Float64 Float64 Float64 +─────┼───────────────────────────────────────────────────────────────── + 1 │ cutpoints[1] 0.0738932 0.163178 0.252302 0.389008 0.845115 + 2 │ cutpoints[2] 0.254298 0.534801 0.818166 1.2405 2.71577 + 3 │ cutpoints[3] 0.704014 1.52676 2.32062 3.56916 8.04289 + 4 │ β[1] 0.745092 0.863281 0.933127 1.00918 1.17439 + 5 │ β[2] 0.677071 0.832166 0.935842 1.05535 1.29072 \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/ex8.out b/assets/pages/08_ordinal_reg/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/08_ordinal_reg/code/output/ex8.res b/assets/pages/08_ordinal_reg/code/output/ex8.res new file mode 100644 index 00000000..82257486 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/ex8.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ parameters 2.5% 25.0% 50.0% 75.0% 97.5% + │ Symbol Float64 Float64 Float64 Float64 Float64 +─────┼───────────────────────────────────────────────────────────────── + 1 │ cutpoints[1] 0.0688087 0.140286 0.201471 0.280062 0.458028 + 2 │ cutpoints[2] 0.202741 0.34845 0.449995 0.553671 0.730877 + 3 │ cutpoints[3] 0.41315 0.604236 0.698851 0.781141 0.889416 + 4 │ β[1] 0.426964 0.463312 0.482703 0.502285 0.540101 + 5 │ β[2] 0.403722 0.454198 0.483429 0.513466 0.563457 \ No newline at end of file diff --git a/assets/pages/08_ordinal_reg/code/output/logodds.svg b/assets/pages/08_ordinal_reg/code/output/logodds.svg new file mode 100644 index 00000000..cb6148b2 --- /dev/null +++ b/assets/pages/08_ordinal_reg/code/output/logodds.svg @@ -0,0 +1,689 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/09_count_reg/code/ex1.jl b/assets/pages/09_count_reg/code/ex1.jl new file mode 100644 index 00000000..825b8791 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex1.jl @@ -0,0 +1,5 @@ +# This file was generated, do not modify it. # hide +using CairoMakie + +f, ax, l = lines(-6 .. 6, exp; axis=(xlabel=L"x", ylabel=L"e^x")) +save(joinpath(@OUTPUT, "exponential.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex10.jl b/assets/pages/09_count_reg/code/ex10.jl new file mode 100644 index 00000000..946c1905 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex10.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +model_negbin = negbinreg(X, y); \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex11.jl b/assets/pages/09_count_reg/code/ex11.jl new file mode 100644 index 00000000..0c4227a0 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex11.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +seed!(111) # hide +chain_negbin = sample(model_negbin, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain_negbin) \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex12.jl b/assets/pages/09_count_reg/code/ex12.jl new file mode 100644 index 00000000..5b4fe8eb --- /dev/null +++ b/assets/pages/09_count_reg/code/ex12.jl @@ -0,0 +1,5 @@ +# This file was generated, do not modify it. # hide +@chain quantile(chain_negbin) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex2.jl b/assets/pages/09_count_reg/code/ex2.jl new file mode 100644 index 00000000..e1b18d4d --- /dev/null +++ b/assets/pages/09_count_reg/code/ex2.jl @@ -0,0 +1,15 @@ +# This file was generated, do not modify it. # hide +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function poissonreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + + #likelihood + return y ~ arraydist(LazyArray(@~ LogPoisson.(α .+ X * β))) +end; \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex3.jl b/assets/pages/09_count_reg/code/ex3.jl new file mode 100644 index 00000000..75639220 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex3.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +using Distributions +f, ax, l = lines( + Gamma(0.01, 0.01); + linewidth=2, + axis=(xlabel=L"\phi", ylabel="Density", limits=(0, 0.03, nothing, nothing)), +) +save(joinpath(@OUTPUT, "gamma.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex4.jl b/assets/pages/09_count_reg/code/ex4.jl new file mode 100644 index 00000000..1a1f1cf1 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex4.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +function NegativeBinomial2(μ, ϕ) + p = 1 / (1 + μ / ϕ) + p = p > 0 ? p : 1e-4 # numerical stability + r = ϕ + + return NegativeBinomial(r, p) +end \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex5.jl b/assets/pages/09_count_reg/code/ex5.jl new file mode 100644 index 00000000..579f8468 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex5.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +@model function negbinreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(0, 2.5) + β ~ filldist(TDist(3), predictors) + ϕ⁻ ~ Gamma(0.01, 0.01) + ϕ = 1 / ϕ⁻ + + #likelihood + return y ~ arraydist(LazyArray(@~ NegativeBinomial2.(exp.(α .+ X * β), ϕ))) +end; \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex6.jl b/assets/pages/09_count_reg/code/ex6.jl new file mode 100644 index 00000000..90f5f7d1 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex6.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/roaches.csv" +roaches = CSV.read(HTTP.get(url).body, DataFrame) +describe(roaches) \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex7.jl b/assets/pages/09_count_reg/code/ex7.jl new file mode 100644 index 00000000..b73962e4 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex7.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +X = Matrix(select(roaches, Not(:y))) +y = roaches[:, :y] +model_poisson = poissonreg(X, y); \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex8.jl b/assets/pages/09_count_reg/code/ex8.jl new file mode 100644 index 00000000..14f04047 --- /dev/null +++ b/assets/pages/09_count_reg/code/ex8.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +chain_poisson = sample(model_poisson, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain_poisson) \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/ex9.jl b/assets/pages/09_count_reg/code/ex9.jl new file mode 100644 index 00000000..0a18c9ab --- /dev/null +++ b/assets/pages/09_count_reg/code/ex9.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +using Chain + +@chain quantile(chain_poisson) begin + DataFrame + select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false) +end \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex1.out b/assets/pages/09_count_reg/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex1.res b/assets/pages/09_count_reg/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex10.out b/assets/pages/09_count_reg/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex10.res b/assets/pages/09_count_reg/code/output/ex10.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex10.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex11.out b/assets/pages/09_count_reg/code/output/ex11.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex11.res b/assets/pages/09_count_reg/code/output/ex11.res new file mode 100644 index 00000000..fea57090 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex11.res @@ -0,0 +1,10 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 2.3935 0.3639 0.0077 2289.8720 1937.3708 1.0026 50.6474 + β[1] 0.0127 0.0015 0.0000 4640.3636 2667.0438 1.0014 102.6357 + β[2] -0.7327 0.1499 0.0027 3027.2982 2637.8716 1.0028 66.9578 + β[3] -0.3199 0.1553 0.0027 3278.7372 2381.6903 1.0018 72.5192 + β[4] 0.4266 0.3365 0.0073 2198.4749 1778.9698 1.0027 48.6259 + ϕ⁻ 1.4058 0.0800 0.0014 3056.0376 2581.1948 1.0003 67.5935 diff --git a/assets/pages/09_count_reg/code/output/ex12.out b/assets/pages/09_count_reg/code/output/ex12.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex12.res b/assets/pages/09_count_reg/code/output/ex12.res new file mode 100644 index 00000000..c26d4979 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex12.res @@ -0,0 +1,10 @@ +6×6 DataFrame + Row │ parameters 2.5% 25.0% 50.0% 75.0% 97.5% + │ Symbol Float64 Float64 Float64 Float64 Float64 +─────┼───────────────────────────────────────────────────────────────── + 1 │ α 5.28389 8.5925 11.0566 13.9834 21.9849 + 2 │ β[1] 1.00987 1.01173 1.01277 1.01381 1.016 + 3 │ β[2] 0.358343 0.433873 0.481535 0.53245 0.645215 + 4 │ β[3] 0.540157 0.650629 0.724575 0.806835 0.984437 + 5 │ β[4] 0.829193 1.21638 1.50724 1.92373 3.05504 + 6 │ ϕ⁻ 3.52309 3.85985 4.07161 4.30516 4.80385 \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex2.out b/assets/pages/09_count_reg/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex2.res b/assets/pages/09_count_reg/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex3.out b/assets/pages/09_count_reg/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex3.res b/assets/pages/09_count_reg/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex4.out b/assets/pages/09_count_reg/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex4.res b/assets/pages/09_count_reg/code/output/ex4.res new file mode 100644 index 00000000..8c8799e0 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex4.res @@ -0,0 +1 @@ +NegativeBinomial2 (generic function with 1 method) \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex5.out b/assets/pages/09_count_reg/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex5.res b/assets/pages/09_count_reg/code/output/ex5.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex5.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex6.out b/assets/pages/09_count_reg/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex6.res b/assets/pages/09_count_reg/code/output/ex6.res new file mode 100644 index 00000000..9a88a495 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex6.res @@ -0,0 +1,9 @@ +5×7 DataFrame + Row │ variable mean min median max nmissing eltype + │ Symbol Float64 Real Float64 Real Int64 DataType +─────┼──────────────────────────────────────────────────────────────────── + 1 │ y 25.6489 0 3.0 357 0 Int64 + 2 │ roach1 42.1935 0.0 7.0 450.0 0 Float64 + 3 │ treatment 0.603053 0 1.0 1 0 Int64 + 4 │ senior 0.305344 0 0.0 1 0 Int64 + 5 │ exposure2 1.02105 0.2 1.0 4.28571 0 Float64 \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex7.out b/assets/pages/09_count_reg/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex7.res b/assets/pages/09_count_reg/code/output/ex7.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex7.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/ex8.out b/assets/pages/09_count_reg/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex8.res b/assets/pages/09_count_reg/code/output/ex8.res new file mode 100644 index 00000000..6a819297 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex8.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 2.9867 0.1526 0.0257 307.5895 81.9752 1.0121 9.3558 + β[1] 0.0065 0.0002 0.0000 3084.6402 2620.6876 1.0023 93.8237 + β[2] -0.5007 0.1066 0.0179 277.4972 68.4987 1.0220 8.4405 + β[3] -0.3620 0.1247 0.0215 319.4964 79.5812 1.0136 9.7179 + β[4] 0.1265 0.2666 0.0452 310.0447 75.1405 1.0124 9.4304 diff --git a/assets/pages/09_count_reg/code/output/ex9.out b/assets/pages/09_count_reg/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/09_count_reg/code/output/ex9.res b/assets/pages/09_count_reg/code/output/ex9.res new file mode 100644 index 00000000..a10ebf2d --- /dev/null +++ b/assets/pages/09_count_reg/code/output/ex9.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ parameters 2.5% 25.0% 50.0% 75.0% 97.5% + │ Symbol Float64 Float64 Float64 Float64 Float64 +─────┼─────────────────────────────────────────────────────────────────── + 1 │ α 17.8944 18.8715 19.4401 20.0227 21.6618 + 2 │ β[1] 1.00636 1.00648 1.00655 1.00661 1.00672 + 3 │ β[2] 0.569218 0.587878 0.598309 0.608326 0.642261 + 4 │ β[3] 0.640725 0.669002 0.685442 0.701448 0.74567 + 5 │ β[4] 1.07174 1.14706 1.17526 1.20443 1.26015 \ No newline at end of file diff --git a/assets/pages/09_count_reg/code/output/exponential.svg b/assets/pages/09_count_reg/code/output/exponential.svg new file mode 100644 index 00000000..b4a38c69 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/exponential.svg @@ -0,0 +1,185 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/09_count_reg/code/output/gamma.svg b/assets/pages/09_count_reg/code/output/gamma.svg new file mode 100644 index 00000000..83be9cb9 --- /dev/null +++ b/assets/pages/09_count_reg/code/output/gamma.svg @@ -0,0 +1,204 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/10_robust_reg/code/ex1.jl b/assets/pages/10_robust_reg/code/ex1.jl new file mode 100644 index 00000000..8c924aa8 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex1.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +using CairoMakie +using Distributions + +f, ax, l = lines(-4 .. 4, Normal(0, 1); linewidth=5, axis=(; xlabel=L"x", ylabel="Density")) +save(joinpath(@OUTPUT, "normal.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex10.jl b/assets/pages/10_robust_reg/code/ex10.jl new file mode 100644 index 00000000..ac887937 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex10.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +quantile(chain) \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex2.jl b/assets/pages/10_robust_reg/code/ex2.jl new file mode 100644 index 00000000..4a23516e --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex2.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines(-4 .. 4, TDist(2); linewidth=5, axis=(xlabel=L"x", ylabel="Density")) +save(joinpath(@OUTPUT, "tdist.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex3.jl b/assets/pages/10_robust_reg/code/ex3.jl new file mode 100644 index 00000000..4d2ce018 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex3.jl @@ -0,0 +1,11 @@ +# This file was generated, do not modify it. # hide +f, ax, l = lines( + -4 .. 4, + Normal(0, 1); + linewidth=5, + label="Normal", + axis=(; xlabel=L"x", ylabel="Density"), +) +lines!(ax, -4 .. 4, TDist(2); linewidth=5, label="Student") +axislegend(ax) +save(joinpath(@OUTPUT, "comparison_normal_student.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex4.jl b/assets/pages/10_robust_reg/code/ex4.jl new file mode 100644 index 00000000..608c8a60 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex4.jl @@ -0,0 +1,19 @@ +# This file was generated, do not modify it. # hide +using Turing +using Statistics: mean, std +using StatsBase: mad +using Random: seed! +seed!(123) +seed!(456) # hide +setprogress!(false) # hide + +@model function robustreg(X, y; predictors=size(X, 2)) + #priors + α ~ LocationScale(median(y), 2.5 * mad(y), TDist(3)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + ν ~ LogNormal(2, 1) + + #likelihood + return y ~ arraydist(LocationScale.(α .+ X * β, σ, TDist.(ν))) +end; \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex5.jl b/assets/pages/10_robust_reg/code/ex5.jl new file mode 100644 index 00000000..b4aa7c89 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex5.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/duncan.csv" +duncan = CSV.read(HTTP.get(url).body, DataFrame) +describe(duncan) \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex6.jl b/assets/pages/10_robust_reg/code/ex6.jl new file mode 100644 index 00000000..9807a1e4 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex6.jl @@ -0,0 +1,5 @@ +# This file was generated, do not modify it. # hide +f = Figure() +plt = data(duncan) * mapping(:prestige) * AlgebraOfGraphics.density() +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "prestige_density.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex7.jl b/assets/pages/10_robust_reg/code/ex7.jl new file mode 100644 index 00000000..e83f418b --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex7.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +gdf = groupby(duncan, :type) +f = Figure() +plt = + data(combine(gdf, :prestige => mean)) * mapping(:type, :prestige_mean) * visual(BarPlot) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "prestige_per_type.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex8.jl b/assets/pages/10_robust_reg/code/ex8.jl new file mode 100644 index 00000000..9ebce60d --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex8.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +X = Matrix(select(duncan, [:income, :education])) +y = duncan[:, :prestige] +model = robustreg(X, y); \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/ex9.jl b/assets/pages/10_robust_reg/code/ex9.jl new file mode 100644 index 00000000..468b4763 --- /dev/null +++ b/assets/pages/10_robust_reg/code/ex9.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) +summarystats(chain) \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/comparison_normal_student.svg b/assets/pages/10_robust_reg/code/output/comparison_normal_student.svg new file mode 100644 index 00000000..f70b03da --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/comparison_normal_student.svg @@ -0,0 +1,348 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/10_robust_reg/code/output/ex1.out b/assets/pages/10_robust_reg/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex1.res b/assets/pages/10_robust_reg/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex10.out b/assets/pages/10_robust_reg/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex10.res b/assets/pages/10_robust_reg/code/output/ex10.res new file mode 100644 index 00000000..53876be9 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex10.res @@ -0,0 +1,9 @@ +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α -13.3670 -9.6056 -7.5776 -5.5600 -1.7272 + β[1] 0.5567 0.7114 0.7743 0.8380 0.9659 + β[2] 0.2796 0.3936 0.4460 0.4993 0.6170 + σ 4.1120 5.8269 7.0177 8.3067 10.9849 + ν 1.0269 1.6074 2.1733 3.1841 8.1579 diff --git a/assets/pages/10_robust_reg/code/output/ex2.out b/assets/pages/10_robust_reg/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex2.res b/assets/pages/10_robust_reg/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex3.out b/assets/pages/10_robust_reg/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex3.res b/assets/pages/10_robust_reg/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex4.out b/assets/pages/10_robust_reg/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex4.res b/assets/pages/10_robust_reg/code/output/ex4.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex4.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex5.out b/assets/pages/10_robust_reg/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex5.res b/assets/pages/10_robust_reg/code/output/ex5.res new file mode 100644 index 00000000..22f9a304 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex5.res @@ -0,0 +1,9 @@ +5×7 DataFrame + Row │ variable mean min median max nmissing eltype + │ Symbol Union… Any Union… Any Int64 DataType +─────┼────────────────────────────────────────────────────────────────────────────── + 1 │ profession RR.engineer welfare.worker 0 String31 + 2 │ type bc wc 0 String7 + 3 │ income 41.8667 7 42.0 81 0 Int64 + 4 │ education 52.5556 7 45.0 100 0 Int64 + 5 │ prestige 47.6889 3 41.0 97 0 Int64 \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex6.out b/assets/pages/10_robust_reg/code/output/ex6.out new file mode 100644 index 00000000..f3d870e7 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex6.out @@ -0,0 +1 @@ +UndefVarError: `data` not defined diff --git a/assets/pages/10_robust_reg/code/output/ex6.res b/assets/pages/10_robust_reg/code/output/ex6.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex6.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex7.out b/assets/pages/10_robust_reg/code/output/ex7.out new file mode 100644 index 00000000..f3d870e7 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex7.out @@ -0,0 +1 @@ +UndefVarError: `data` not defined diff --git a/assets/pages/10_robust_reg/code/output/ex7.res b/assets/pages/10_robust_reg/code/output/ex7.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex7.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex8.out b/assets/pages/10_robust_reg/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex8.res b/assets/pages/10_robust_reg/code/output/ex8.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex8.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/10_robust_reg/code/output/ex9.out b/assets/pages/10_robust_reg/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/10_robust_reg/code/output/ex9.res b/assets/pages/10_robust_reg/code/output/ex9.res new file mode 100644 index 00000000..039382a2 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/ex9.res @@ -0,0 +1,9 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α -7.5790 3.0116 0.0628 2309.7654 1978.4710 1.0042 70.5574 + β[1] 0.7719 0.1024 0.0025 1748.2750 1843.0804 1.0028 53.4053 + β[2] 0.4464 0.0836 0.0020 1702.5880 1820.3078 1.0052 52.0097 + σ 7.1578 1.7728 0.0477 1225.3095 826.2654 1.0027 37.4300 + ν 2.8148 2.4661 0.0635 1452.4245 1333.5578 1.0009 44.3678 diff --git a/assets/pages/10_robust_reg/code/output/normal.svg b/assets/pages/10_robust_reg/code/output/normal.svg new file mode 100644 index 00000000..b24d03f8 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/normal.svg @@ -0,0 +1,260 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/10_robust_reg/code/output/tdist.svg b/assets/pages/10_robust_reg/code/output/tdist.svg new file mode 100644 index 00000000..f4405177 --- /dev/null +++ b/assets/pages/10_robust_reg/code/output/tdist.svg @@ -0,0 +1,234 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/11_multilevel_models/code/ex1.jl b/assets/pages/11_multilevel_models/code/ex1.jl new file mode 100644 index 00000000..7dbf642e --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex1.jl @@ -0,0 +1,24 @@ +# This file was generated, do not modify it. # hide +using Turing +using LinearAlgebra +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function varying_intercept( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + αⱼ ~ filldist(Normal(0, τ), n_gr) # group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ αⱼ[idx] + return y ~ MvNormal(ŷ, σ^2 * I) +end; \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex10.jl b/assets/pages/11_multilevel_models/code/ex10.jl new file mode 100644 index 00000000..3022bf0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex10.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +model_intercept_slope = varying_intercept_slope(X, idx, y) +chain_intercept_slope = sample(model_intercept_slope, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_intercept_slope))) \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex11.jl b/assets/pages/11_multilevel_models/code/ex11.jl new file mode 100644 index 00000000..63b1336f --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex11.jl @@ -0,0 +1,5 @@ +# This file was generated, do not modify it. # hide +X_correlated = hcat(fill(1, size(X, 1)), X) +model_correlated = correlated_varying_intercept_slope(X_correlated, idx, y) +chain_correlated = sample(model_correlated, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_correlated))) \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex2.jl b/assets/pages/11_multilevel_models/code/ex2.jl new file mode 100644 index 00000000..3cd838fe --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex2.jl @@ -0,0 +1,14 @@ +# This file was generated, do not modify it. # hide +@model function varying_slope(X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random slopes + #usually requires thoughtful specification + τ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs + βⱼ ~ filldist(Normal(0, 1), predictors, n_gr) # group-level standard normal slopes + + #likelihood + ŷ = α .+ X * βⱼ * τ + return y ~ MvNormal(ŷ, σ^2 * I) +end; \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex3.jl b/assets/pages/11_multilevel_models/code/ex3.jl new file mode 100644 index 00000000..0af18305 --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex3.jl @@ -0,0 +1,18 @@ +# This file was generated, do not modify it. # hide +@model function varying_intercept_slope( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts and slopes + #usually requires thoughtful specification + τₐ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + τᵦ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs + αⱼ ~ filldist(Normal(0, τₐ), n_gr) # group-level intercepts + βⱼ ~ filldist(Normal(0, 1), predictors, n_gr) # group-level standard normal slopes + + #likelihood + ŷ = α .+ αⱼ[idx] .+ X * βⱼ * τᵦ + return y ~ MvNormal(ŷ, σ^2 * I) +end; \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex4.jl b/assets/pages/11_multilevel_models/code/ex4.jl new file mode 100644 index 00000000..3aa16b6d --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex4.jl @@ -0,0 +1,24 @@ +# This file was generated, do not modify it. # hide +using PDMats + +@model function correlated_varying_intercept_slope( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + Ω ~ LKJCholesky(predictors, 2.0) # Cholesky decomposition correlation matrix + σ ~ Exponential(std(y)) + + #prior for variance of random correlated intercepts and slopes + #usually requires thoughtful specification + τ ~ filldist(truncated(Cauchy(0, 2); lower=0), predictors) # group-level SDs + γ ~ filldist(Normal(0, 5), predictors, n_gr) # matrix of group coefficients + + #reconstruct Σ from Ω and τ + Σ_L = Diagonal(τ) * Ω.L + Σ = PDMat(Cholesky(Σ_L + 1e-6 * I)) # numerical instability + #reconstruct β from Σ and γ + β = Σ * γ + + #likelihood + return y ~ arraydist([Normal(X[i, :] ⋅ β[:, idx[i]], σ) for i in 1:length(y)]) +end; \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex5.jl b/assets/pages/11_multilevel_models/code/ex5.jl new file mode 100644 index 00000000..e2a8e053 --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex5.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv" +cheese = CSV.read(HTTP.get(url).body, DataFrame) +describe(cheese) \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex6.jl b/assets/pages/11_multilevel_models/code/ex6.jl new file mode 100644 index 00000000..c5202a77 --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex6.jl @@ -0,0 +1,16 @@ +# This file was generated, do not modify it. # hide +for c in unique(cheese[:, :cheese]) + cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0) +end + +cheese[:, :background_int] = map(cheese[:, :background]) do b + if b == "rural" + 1 + elseif b == "urban" + 2 + else + missing + end +end + +first(cheese, 5) \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex7.jl b/assets/pages/11_multilevel_models/code/ex7.jl new file mode 100644 index 00000000..abbce534 --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex7.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +X = Matrix(select(cheese, Between(:cheese_A, :cheese_D))); +y = cheese[:, :y]; +idx = cheese[:, :background_int]; \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex8.jl b/assets/pages/11_multilevel_models/code/ex8.jl new file mode 100644 index 00000000..dd778d1d --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex8.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +model_intercept = varying_intercept(X, idx, y) +chain_intercept = sample(model_intercept, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_intercept))) \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/ex9.jl b/assets/pages/11_multilevel_models/code/ex9.jl new file mode 100644 index 00000000..e5ab311e --- /dev/null +++ b/assets/pages/11_multilevel_models/code/ex9.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +model_slope = varying_slope(X, idx, y) +chain_slope = sample(model_slope, NUTS(), MCMCThreads(), 1_000, 4) +println(DataFrame(summarystats(chain_slope))) \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex1.out b/assets/pages/11_multilevel_models/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex1.res b/assets/pages/11_multilevel_models/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex10.out b/assets/pages/11_multilevel_models/code/output/ex10.out new file mode 100644 index 00000000..fe57b28a --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex10.out @@ -0,0 +1,19 @@ +15×8 DataFrame + Row │ parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + │ Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 +─────┼─────────────────────────────────────────────────────────────────────────────────────── + 1 │ α 70.8973 7.06973 0.226273 1080.56 988.602 1.00354 13.1307 + 2 │ σ 7.07716 0.403315 0.00763609 2791.06 2389.42 1.00236 33.9161 + 3 │ τₐ 6.19068 5.86481 0.187349 1286.35 1007.7 1.00342 15.6313 + 4 │ τᵦ[1] 6.19739 5.39243 0.202217 646.032 1159.5 1.00601 7.85039 + 5 │ τᵦ[2] 6.24244 5.30258 0.215136 598.407 1527.72 1.00463 7.27167 + 6 │ αⱼ[1] -3.62952 5.05254 0.176822 1253.95 761.597 1.00336 15.2377 + 7 │ αⱼ[2] 3.42934 5.05582 0.176181 1271.82 839.894 1.00348 15.4548 + 8 │ βⱼ[1,1] 0.273658 0.779529 0.0188846 1727.94 1703.37 1.00193 20.9975 + 9 │ βⱼ[2,1] -0.88519 1.07495 0.0402493 794.926 375.533 1.00615 9.6597 + 10 │ βⱼ[3,1] 0.540418 0.897065 0.0325307 849.719 418.537 1.00343 10.3255 + 11 │ βⱼ[4,1] 0.119311 0.761924 0.0183527 1730.92 1000.11 1.00092 21.0336 + 12 │ βⱼ[1,2] 0.230796 0.819673 0.0210441 1571.83 1681.27 1.00307 19.1004 + 13 │ βⱼ[2,2] -0.916136 1.02804 0.0319528 1027.79 2087.51 1.00208 12.4894 + 14 │ βⱼ[3,2] 0.577911 0.873303 0.0269948 1094.9 1738.12 1.00304 13.3049 + 15 │ βⱼ[4,2] 0.115502 0.784112 0.0186258 1758.58 1981.55 1.0031 21.3698 diff --git a/assets/pages/11_multilevel_models/code/output/ex10.res b/assets/pages/11_multilevel_models/code/output/ex10.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex10.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex11.out b/assets/pages/11_multilevel_models/code/output/ex11.out new file mode 100644 index 00000000..4df267cc --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex11.out @@ -0,0 +1,35 @@ +31×8 DataFrame + Row │ parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + │ Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 +─────┼─────────────────────────────────────────────────────────────────────────────────────────────── + 1 │ Ω.L[1,1] 1.0 0.0 NaN NaN NaN NaN NaN + 2 │ Ω.L[2,1] 0.0499316 0.31051 0.00698543 1975.54 2314.64 1.00168 1.52283 + 3 │ Ω.L[3,1] -0.454537 0.249772 0.00571052 1977.49 2262.51 1.00096 1.52433 + 4 │ Ω.L[4,1] 0.308835 0.25287 0.00599969 1843.28 2199.73 1.00014 1.42088 + 5 │ Ω.L[5,1] -0.125632 0.307712 0.00654629 2239.37 2376.57 1.00314 1.72619 + 6 │ Ω.L[2,2] 0.946451 0.0731193 0.00168809 2176.21 2182.87 1.00018 1.67751 + 7 │ Ω.L[3,2] 0.00316296 0.326756 0.00512707 4010.69 2567.33 1.0003 3.0916 + 8 │ Ω.L[4,2] 0.0351896 0.354115 0.00527131 4546.36 3111.1 1.00178 3.50452 + 9 │ Ω.L[5,2] 0.0261588 0.366052 0.00538327 4345.52 2064.09 1.00291 3.3497 + 10 │ Ω.L[3,3] 0.775999 0.148672 0.00348151 1845.72 2482.95 1.00082 1.42276 + 11 │ Ω.L[4,3] -0.0330662 0.349329 0.00604209 3242.42 2090.97 1.00099 2.49939 + 12 │ Ω.L[5,3] 0.0374594 0.360744 0.00516219 4901.98 2431.25 1.00176 3.77865 + 13 │ Ω.L[4,4] 0.753896 0.150472 0.00357772 1850.93 2236.89 1.00165 1.42677 + 14 │ Ω.L[5,4] 0.0116934 0.349883 0.00506856 4664.43 2772.27 1.00128 3.59553 + 15 │ Ω.L[5,5] 0.684545 0.179677 0.00469272 1541.5 1667.6 1.00377 1.18825 + 16 │ σ 7.08991 0.392342 0.00609954 4281.99 2994.66 1.00037 3.30073 + 17 │ τ[1] 3.82687 0.792461 0.0243249 1071.64 1274.54 1.00157 0.82606 + 18 │ τ[2] 0.515217 0.457244 0.0118942 1162.98 1701.28 1.00367 0.896471 + 19 │ τ[3] 1.57042 0.675595 0.0194601 1075.05 1066.81 1.00327 0.82869 + 20 │ τ[4] 0.79098 0.502001 0.0137349 1042.75 683.387 1.0037 0.803794 + 21 │ τ[5] 0.536124 0.435909 0.0118638 1106.15 1058.94 1.00122 0.852668 + 22 │ γ[1,1] 5.1869 2.19555 0.0618344 1249.75 1539.01 1.00063 0.963357 + 23 │ γ[2,1] -0.104914 4.6473 0.0843589 3038.11 2765.66 0.999955 2.3419 + 24 │ γ[3,1] -2.26726 3.24333 0.0677281 2307.05 2102.85 1.00399 1.77836 + 25 │ γ[4,1] 0.786036 4.18682 0.0682086 3757.86 2648.24 1.00005 2.89671 + 26 │ γ[5,1] 0.317867 4.53047 0.067744 4456.83 2826.46 1.0041 3.43551 + 27 │ γ[1,2] 5.68517 2.38212 0.0679177 1209.5 1530.76 1.00015 0.932333 + 28 │ γ[2,2] 0.406946 4.69823 0.0761723 3785.93 2816.48 1.00131 2.91835 + 29 │ γ[3,2] -2.43749 3.41662 0.0695933 2451.7 2481.02 1.00083 1.88987 + 30 │ γ[4,2] 1.40997 4.31934 0.0742324 3389.91 2550.09 1.00076 2.61308 + 31 │ γ[5,2] -0.565495 4.69425 0.0822518 3281.19 2340.3 1.00369 2.52928 diff --git a/assets/pages/11_multilevel_models/code/output/ex11.res b/assets/pages/11_multilevel_models/code/output/ex11.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex11.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex2.out b/assets/pages/11_multilevel_models/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex2.res b/assets/pages/11_multilevel_models/code/output/ex2.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex2.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex3.out b/assets/pages/11_multilevel_models/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex3.res b/assets/pages/11_multilevel_models/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex4.out b/assets/pages/11_multilevel_models/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex4.res b/assets/pages/11_multilevel_models/code/output/ex4.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex4.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex5.out b/assets/pages/11_multilevel_models/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex5.res b/assets/pages/11_multilevel_models/code/output/ex5.res new file mode 100644 index 00000000..ac3bd60f --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex5.res @@ -0,0 +1,8 @@ +4×7 DataFrame + Row │ variable mean min median max nmissing eltype + │ Symbol Union… Any Union… Any Int64 DataType +─────┼─────────────────────────────────────────────────────────────── + 1 │ cheese A D 0 String1 + 2 │ rater 5.5 1 5.5 10 0 Int64 + 3 │ background rural urban 0 String7 + 4 │ y 70.8438 33 71.5 91 0 Int64 \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex6.out b/assets/pages/11_multilevel_models/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex6.res b/assets/pages/11_multilevel_models/code/output/ex6.res new file mode 100644 index 00000000..ac7466e6 --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex6.res @@ -0,0 +1,9 @@ +5×9 DataFrame + Row │ cheese rater background y cheese_A cheese_B cheese_C cheese_D background_int + │ String1 Int64 String7 Int64 Int64 Int64 Int64 Int64 Int64 +─────┼─────────────────────────────────────────────────────────────────────────────────────────── + 1 │ A 1 rural 67 1 0 0 0 1 + 2 │ A 1 rural 66 1 0 0 0 1 + 3 │ B 1 rural 51 0 1 0 0 1 + 4 │ B 1 rural 53 0 1 0 0 1 + 5 │ C 1 rural 75 0 0 1 0 1 \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex7.out b/assets/pages/11_multilevel_models/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/11_multilevel_models/code/output/ex7.res b/assets/pages/11_multilevel_models/code/output/ex7.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex7.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex8.out b/assets/pages/11_multilevel_models/code/output/ex8.out new file mode 100644 index 00000000..9d92896f --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex8.out @@ -0,0 +1,13 @@ +9×8 DataFrame + Row │ parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + │ Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 +─────┼─────────────────────────────────────────────────────────────────────────────────────── + 1 │ α 70.7489 5.02573 0.176587 1049.21 944.855 1.00255 19.8948 + 2 │ β[1] 2.91501 1.30963 0.0257346 2596.92 2763.9 1.00097 49.2419 + 3 │ β[2] -10.632 1.34056 0.028425 2224.99 2724.4 1.00189 42.1895 + 4 │ β[3] 6.52144 1.3159 0.0265587 2450.36 2667.92 1.00052 46.463 + 5 │ β[4] 1.09178 1.31719 0.0264376 2475.97 2627.54 1.00117 46.9485 + 6 │ σ 7.3913 0.449209 0.00807096 3127.77 2530.83 1.00102 59.3078 + 7 │ τ 6.11784 5.54993 0.152948 1863.81 1849.7 1.00172 35.341 + 8 │ αⱼ[1] -3.39662 4.92084 0.174051 1057.29 929.725 1.00312 20.0479 + 9 │ αⱼ[2] 3.63474 4.95267 0.175295 1054.17 920.198 1.00331 19.9889 diff --git a/assets/pages/11_multilevel_models/code/output/ex8.res b/assets/pages/11_multilevel_models/code/output/ex8.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex8.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/11_multilevel_models/code/output/ex9.out b/assets/pages/11_multilevel_models/code/output/ex9.out new file mode 100644 index 00000000..67ab7322 --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex9.out @@ -0,0 +1,16 @@ +12×8 DataFrame + Row │ parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + │ Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 +─────┼──────────────────────────────────────────────────────────────────────────────────────── + 1 │ α 71.6225 6.12929 0.80038 70.2758 24.8834 1.0542 1.3079 + 2 │ σ 7.98871 0.435528 0.0142789 879.095 1907.61 1.00481 16.3607 + 3 │ τ[1] 5.95148 5.05219 0.283585 297.117 1630.98 1.01456 5.52961 + 4 │ τ[2] 7.13182 7.54656 1.22127 120.798 26.97 1.03962 2.24817 + 5 │ βⱼ[1,1] 0.199936 0.839602 0.0512732 282.448 223.989 1.01652 5.25661 + 6 │ βⱼ[2,1] -0.933251 1.06721 0.0676919 231.737 252.835 1.01661 4.31283 + 7 │ βⱼ[3,1] 0.522532 0.883634 0.035598 636.854 1246.34 1.02562 11.8524 + 8 │ βⱼ[4,1] 0.0664401 0.797414 0.0514516 278.881 154.864 1.02163 5.19022 + 9 │ βⱼ[1,2] 0.290895 0.843615 0.0732187 135.896 84.0176 1.03096 2.52915 + 10 │ βⱼ[2,2] -0.853134 1.05917 0.0568238 415.139 467.237 1.01995 7.7261 + 11 │ βⱼ[3,2] 0.563184 0.876935 0.0375404 474.159 818.21 1.01366 8.82451 + 12 │ βⱼ[4,2] 0.124051 0.787212 0.0525178 224.833 143.946 1.018 4.18434 diff --git a/assets/pages/11_multilevel_models/code/output/ex9.res b/assets/pages/11_multilevel_models/code/output/ex9.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/11_multilevel_models/code/output/ex9.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex1.jl b/assets/pages/12_Turing_tricks/code/ex1.jl new file mode 100644 index 00000000..d70c3002 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex1.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +A = rand(3, 2) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex10.jl b/assets/pages/12_Turing_tricks/code/ex10.jl new file mode 100644 index 00000000..0bb5a457 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex10.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +@model function funnel() + y ~ Normal(0, 3) + return x ~ Normal(0, exp(y / 2)) +end + +chain_funnel = sample(funnel(), NUTS(), MCMCThreads(), 1_000, 4) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex11.jl b/assets/pages/12_Turing_tricks/code/ex11.jl new file mode 100644 index 00000000..7a71e6b1 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex11.jl @@ -0,0 +1,9 @@ +# This file was generated, do not modify it. # hide +@model function ncp_funnel() + x̃ ~ Normal() + ỹ ~ Normal() + y = 3.0 * ỹ # implies y ~ Normal(0, 3) + return x = exp(y / 2) * x̃ # implies x ~ Normal(0, exp(y / 2)) +end + +chain_ncp_funnel = sample(ncp_funnel(), NUTS(), MCMCThreads(), 1_000, 4) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex12.jl b/assets/pages/12_Turing_tricks/code/ex12.jl new file mode 100644 index 00000000..53601944 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex12.jl @@ -0,0 +1,17 @@ +# This file was generated, do not modify it. # hide +@model function varying_intercept( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + αⱼ ~ filldist(Normal(0, τ), n_gr) # CP group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ αⱼ[idx] + return y ~ MvNormal(ŷ, σ^2 * I) +end; \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex13.jl b/assets/pages/12_Turing_tricks/code/ex13.jl new file mode 100644 index 00000000..4d58c080 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex13.jl @@ -0,0 +1,18 @@ +# This file was generated, do not modify it. # hide +@model function varying_intercept_ncp( + X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2) +) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) # population-level intercept + β ~ filldist(Normal(0, 2), predictors) # population-level coefficients + σ ~ Exponential(std(y)) # residual SD + + #prior for variance of random intercepts + #usually requires thoughtful specification + τ ~ truncated(Cauchy(0, 2); lower=0) # group-level SDs intercepts + zⱼ ~ filldist(Normal(0, 1), n_gr) # NCP group-level intercepts + + #likelihood + ŷ = α .+ X * β .+ zⱼ[idx] .* τ + return y ~ MvNormal(ŷ, σ^2 * I) +end; \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex14.jl b/assets/pages/12_Turing_tricks/code/ex14.jl new file mode 100644 index 00000000..d5511e7e --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex14.jl @@ -0,0 +1,24 @@ +# This file was generated, do not modify it. # hide +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv" +cheese = CSV.read(HTTP.get(url).body, DataFrame) + +for c in unique(cheese[:, :cheese]) + cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0) +end + +cheese[:, :background_int] = map(cheese[:, :background]) do b + if b == "rural" + 1 + elseif b == "urban" + 2 + else + missing + end +end + +X = Matrix(select(cheese, Between(:cheese_A, :cheese_D))); +y = cheese[:, :y]; +idx = cheese[:, :background_int]; + +model_cp = varying_intercept(X, idx, y) +chain_cp = sample(model_cp, NUTS(), MCMCThreads(), 1_000, 4) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex15.jl b/assets/pages/12_Turing_tricks/code/ex15.jl new file mode 100644 index 00000000..c8582bb4 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex15.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +model_ncp = varying_intercept_ncp(X, idx, y) +chain_ncp = sample(model_ncp, NUTS(), MCMCThreads(), 1_000, 4) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex16.jl b/assets/pages/12_Turing_tricks/code/ex16.jl new file mode 100644 index 00000000..d967d4a5 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex16.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +τ = summarystats(chain_ncp)[:τ, :mean] +αⱼ = mapslices( + x -> x * τ, chain_ncp[:, namesingroup(chain_ncp, :zⱼ), :].value.data; dims=[2] +) +chain_ncp_reconstructed = hcat( + MCMCChains.resetrange(chain_ncp), Chains(αⱼ, ["αⱼ[$i]" for i in 1:length(unique(idx))]) +) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex2.jl b/assets/pages/12_Turing_tricks/code/ex2.jl new file mode 100644 index 00000000..b5dae028 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex2.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +using LinearAlgebra: qr, I +Q, R = qr(A) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex3.jl b/assets/pages/12_Turing_tricks/code/ex3.jl new file mode 100644 index 00000000..5f60cb97 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex3.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +Matrix(Q') ≈ Matrix(Q^-1) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex4.jl b/assets/pages/12_Turing_tricks/code/ex4.jl new file mode 100644 index 00000000..769cfccd --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex4.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +Q' * Q ≈ I(3) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex5.jl b/assets/pages/12_Turing_tricks/code/ex5.jl new file mode 100644 index 00000000..3cd5e3a6 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex5.jl @@ -0,0 +1,28 @@ +# This file was generated, do not modify it. # hide +using Turing +using LinearAlgebra: I +using Statistics: mean, std +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function linreg(X, y; predictors=size(X, 2)) + #priors + α ~ Normal(mean(y), 2.5 * std(y)) + β ~ filldist(TDist(3), predictors) + σ ~ Exponential(1) + + #likelihood + return y ~ MvNormal(α .+ X * β, σ^2 * I) +end; + +using DataFrames +using CSV +using HTTP + +url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv" +kidiq = CSV.read(HTTP.get(url).body, DataFrame) +X = Matrix(select(kidiq, Not(:kid_score))) +y = kidiq[:, :kid_score] +model = linreg(X, y) +chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex6.jl b/assets/pages/12_Turing_tricks/code/ex6.jl new file mode 100644 index 00000000..0e045136 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex6.jl @@ -0,0 +1,4 @@ +# This file was generated, do not modify it. # hide +Q, R = qr(X) +Q_ast = Matrix(Q) * sqrt(size(X, 1) - 1) +R_ast = R / sqrt(size(X, 1) - 1); \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex7.jl b/assets/pages/12_Turing_tricks/code/ex7.jl new file mode 100644 index 00000000..724ad2c5 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex7.jl @@ -0,0 +1,3 @@ +# This file was generated, do not modify it. # hide +model_qr = linreg(Q_ast, y) +chain_qr = sample(model_qr, NUTS(1_000, 0.65), MCMCThreads(), 1_000, 4) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex8.jl b/assets/pages/12_Turing_tricks/code/ex8.jl new file mode 100644 index 00000000..0db6d297 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex8.jl @@ -0,0 +1,8 @@ +# This file was generated, do not modify it. # hide +betas = mapslices( + x -> R_ast^-1 * x, chain_qr[:, namesingroup(chain_qr, :β), :].value.data; dims=[2] +) +chain_beta = setrange( + Chains(betas, ["real_β[$i]" for i in 1:size(Q_ast, 2)]), 1_001:1:2_000 +) +chain_qr_reconstructed = hcat(chain_beta, chain_qr) \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/ex9.jl b/assets/pages/12_Turing_tricks/code/ex9.jl new file mode 100644 index 00000000..e506e665 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/ex9.jl @@ -0,0 +1,13 @@ +# This file was generated, do not modify it. # hide +using CairoMakie +using Distributions +funnel_y = rand(Normal(0, 3), 10_000) +funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2) + +f, ax, s = scatter( + funnel_x, + funnel_y; + color=(:steelblue, 0.3), + axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)), +) +save(joinpath(@OUTPUT, "funnel.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex1.out b/assets/pages/12_Turing_tricks/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex1.res b/assets/pages/12_Turing_tricks/code/output/ex1.res new file mode 100644 index 00000000..e0c63a04 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex1.res @@ -0,0 +1,4 @@ +3×2 Matrix{Float64}: + 0.720103 0.295367 + 0.573619 0.276597 + 0.664468 0.983436 \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex10.out b/assets/pages/12_Turing_tricks/code/output/ex10.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex10.res b/assets/pages/12_Turing_tricks/code/output/ex10.res new file mode 100644 index 00000000..3690b64e --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex10.res @@ -0,0 +1,23 @@ +Chains MCMC chain (1000×14×4 Array{Float64, 3}): + +Iterations = 501:1:1500 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 5.91 seconds +Compute duration = 23.42 seconds +parameters = y, x +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + y 0.3304 2.5005 0.4266 34.1757 64.0933 1.0981 1.4589 + x -0.7113 8.8403 0.6150 467.3926 188.6939 1.0944 19.9527 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + y -3.2368 -2.1271 0.0819 1.9243 5.8793 + x -13.3185 -0.6603 -0.2764 0.4686 6.4055 diff --git a/assets/pages/12_Turing_tricks/code/output/ex11.out b/assets/pages/12_Turing_tricks/code/output/ex11.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex11.res b/assets/pages/12_Turing_tricks/code/output/ex11.res new file mode 100644 index 00000000..d4896835 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex11.res @@ -0,0 +1,23 @@ +Chains MCMC chain (1000×14×4 Array{Float64, 3}): + +Iterations = 501:1:1500 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 5.38 seconds +Compute duration = 21.32 seconds +parameters = x̃, ỹ +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + x̃ -0.0006 1.0002 0.0161 3829.5014 2972.2855 0.9999 179.6454 + ỹ -0.0444 1.0054 0.0160 3917.0636 2987.0567 0.9999 183.7530 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + x̃ -1.8916 -0.6865 -0.0231 0.6704 2.0065 + ỹ -2.0393 -0.7107 -0.0524 0.6362 1.9541 diff --git a/assets/pages/12_Turing_tricks/code/output/ex12.out b/assets/pages/12_Turing_tricks/code/output/ex12.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex12.res b/assets/pages/12_Turing_tricks/code/output/ex12.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex12.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex13.out b/assets/pages/12_Turing_tricks/code/output/ex13.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex13.res b/assets/pages/12_Turing_tricks/code/output/ex13.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex13.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex14.out b/assets/pages/12_Turing_tricks/code/output/ex14.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex14.res b/assets/pages/12_Turing_tricks/code/output/ex14.res new file mode 100644 index 00000000..6a8aa4a2 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex14.res @@ -0,0 +1,37 @@ +Chains MCMC chain (1000×21×4 Array{Float64, 3}): + +Iterations = 501:1:1500 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 7.72 seconds +Compute duration = 26.82 seconds +parameters = α, β[1], β[2], β[3], β[4], σ, τ, αⱼ[1], αⱼ[2] +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 70.6388 5.4554 0.2162 915.6164 759.2940 1.0028 34.1457 + β[1] 2.9639 1.3435 0.0265 2565.3867 2773.4856 1.0012 95.6698 + β[2] -10.5915 1.3904 0.0277 2522.8976 2504.4545 1.0016 94.0853 + β[3] 6.5485 1.3619 0.0264 2668.6532 2413.9865 0.9997 99.5209 + β[4] 1.0905 1.3750 0.0285 2338.0486 2471.5094 1.0010 87.1918 + σ 7.4087 0.4507 0.0088 2643.0429 2581.2624 1.0001 98.5658 + τ 6.2641 5.7017 0.1952 1222.7006 1203.5556 1.0038 45.5976 + αⱼ[1] -3.3228 5.3516 0.2167 865.3478 780.3551 1.0027 32.2710 + αⱼ[2] 3.7386 5.3657 0.2173 894.2647 746.3891 1.0034 33.3494 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α 59.8857 68.3215 70.7533 73.2141 81.3633 + β[1] 0.3781 2.0754 2.9757 3.8536 5.5841 + β[2] -13.2884 -11.5392 -10.6060 -9.6587 -7.8122 + β[3] 3.9443 5.6304 6.5351 7.4435 9.2425 + β[4] -1.6999 0.1515 1.1072 2.0425 3.7349 + σ 6.5891 7.0826 7.3872 7.7026 8.3437 + τ 1.8181 3.2859 4.6436 7.1806 20.4141 + αⱼ[1] -14.0825 -5.6883 -3.3988 -1.1424 7.5331 + αⱼ[2] -6.5136 1.3252 3.5179 5.9215 14.7231 diff --git a/assets/pages/12_Turing_tricks/code/output/ex15.out b/assets/pages/12_Turing_tricks/code/output/ex15.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex15.res b/assets/pages/12_Turing_tricks/code/output/ex15.res new file mode 100644 index 00000000..33dd7fcf --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex15.res @@ -0,0 +1,37 @@ +Chains MCMC chain (1000×21×4 Array{Float64, 3}): + +Iterations = 501:1:1500 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 11.84 seconds +Compute duration = 45.18 seconds +parameters = α, β[1], β[2], β[3], β[4], σ, τ, zⱼ[1], zⱼ[2] +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 71.1250 3.5397 0.1218 878.2014 784.3499 1.0054 19.4365 + β[1] 2.9093 1.2858 0.0297 1876.0233 2481.1601 1.0026 41.5206 + β[2] -10.6841 1.3789 0.0608 571.3700 192.4652 1.0074 12.6457 + β[3] 6.5318 1.3379 0.0326 1713.4983 1455.9135 1.0030 37.9235 + β[4] 1.0746 1.3279 0.0316 1767.8948 2009.5112 1.0071 39.1274 + σ 7.3765 0.4561 0.0158 777.6440 188.6146 1.0139 17.2110 + τ 5.0157 2.6654 0.1354 604.8463 193.9385 1.0075 13.3866 + zⱼ[1] -0.9156 0.7846 0.0225 1184.6917 1334.6682 1.0041 26.2199 + zⱼ[2] 0.8326 0.8046 0.0223 1253.2053 947.4690 1.0023 27.7362 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α 63.6608 68.9974 71.0956 73.1433 78.5094 + β[1] 0.3584 2.0549 2.9066 3.7865 5.4293 + β[2] -13.5656 -11.5643 -10.6594 -9.7752 -7.9711 + β[3] 3.9994 5.5582 6.4857 7.4393 9.0957 + β[4] -1.4433 0.2116 1.0321 1.9658 3.6415 + σ 6.5332 7.0626 7.3592 7.6827 8.3100 + τ 1.8047 3.1023 4.2758 6.3272 11.7515 + zⱼ[1] -2.5756 -1.4160 -0.8862 -0.3486 0.5239 + zⱼ[2] -0.6131 0.2661 0.7795 1.3822 2.4805 diff --git a/assets/pages/12_Turing_tricks/code/output/ex16.out b/assets/pages/12_Turing_tricks/code/output/ex16.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex16.res b/assets/pages/12_Turing_tricks/code/output/ex16.res new file mode 100644 index 00000000..5db1af12 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex16.res @@ -0,0 +1,41 @@ +Chains MCMC chain (1000×23×4 Array{Float64, 3}): + +Iterations = 1:1000 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 11.84 seconds +Compute duration = 45.18 seconds +parameters = α, β[1], β[2], β[3], β[4], σ, τ, zⱼ[1], zⱼ[2], αⱼ[1], αⱼ[2] +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 71.1250 3.5397 0.1218 878.2014 784.3499 1.0054 19.4365 + β[1] 2.9093 1.2858 0.0297 1876.0233 2481.1601 1.0026 41.5206 + β[2] -10.6841 1.3789 0.0608 571.3700 192.4652 1.0074 12.6457 + β[3] 6.5318 1.3379 0.0326 1713.4983 1455.9135 1.0030 37.9235 + β[4] 1.0746 1.3279 0.0316 1767.8948 2009.5112 1.0071 39.1274 + σ 7.3765 0.4561 0.0158 777.6440 188.6146 1.0139 17.2110 + τ 5.0157 2.6654 0.1354 604.8463 193.9385 1.0075 13.3866 + zⱼ[1] -0.9156 0.7846 0.0225 1184.6917 1334.6682 1.0041 26.2199 + zⱼ[2] 0.8326 0.8046 0.0223 1253.2053 947.4690 1.0023 27.7362 + αⱼ[1] -4.5922 3.9351 0.1128 1184.6917 1334.6682 1.0041 26.2199 + αⱼ[2] 4.1762 4.0358 0.1118 1253.2053 947.4690 1.0023 27.7362 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α 63.6608 68.9974 71.0956 73.1433 78.5094 + β[1] 0.3584 2.0549 2.9066 3.7865 5.4293 + β[2] -13.5656 -11.5643 -10.6594 -9.7752 -7.9711 + β[3] 3.9994 5.5582 6.4857 7.4393 9.0957 + β[4] -1.4433 0.2116 1.0321 1.9658 3.6415 + σ 6.5332 7.0626 7.3592 7.6827 8.3100 + τ 1.8047 3.1023 4.2758 6.3272 11.7515 + zⱼ[1] -2.5756 -1.4160 -0.8862 -0.3486 0.5239 + zⱼ[2] -0.6131 0.2661 0.7795 1.3822 2.4805 + αⱼ[1] -12.9183 -7.1023 -4.4447 -1.7483 2.6277 + αⱼ[2] -3.0749 1.3348 3.9098 6.9328 12.4414 diff --git a/assets/pages/12_Turing_tricks/code/output/ex2.out b/assets/pages/12_Turing_tricks/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex2.res b/assets/pages/12_Turing_tricks/code/output/ex2.res new file mode 100644 index 00000000..8703969d --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex2.res @@ -0,0 +1,6 @@ +LinearAlgebra.QRCompactWY{Float64, Matrix{Float64}, Matrix{Float64}} +Q factor: 3×3 LinearAlgebra.QRCompactWYQ{Float64, Matrix{Float64}, Matrix{Float64}} +R factor: +2×2 Matrix{Float64}: + -1.13539 -0.902615 + 0.0 0.562299 \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex3.out b/assets/pages/12_Turing_tricks/code/output/ex3.out new file mode 100644 index 00000000..d96b8c73 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex3.out @@ -0,0 +1,11 @@ +MethodError: no method matching ^(::LinearAlgebra.AdjointQ{Float64, LinearAlgebra.QRCompactWYQ{Float64, Matrix{Float64}, Matrix{Float64}}}, ::Int64) + +Closest candidates are: + ^(!Matched::Float16, ::Integer) + @ Base math.jl:1283 + ^(!Matched::Regex, ::Integer) + @ Base regex.jl:863 + ^(!Matched::Float32, ::Integer) + @ Base math.jl:1277 + ... + diff --git a/assets/pages/12_Turing_tricks/code/output/ex3.res b/assets/pages/12_Turing_tricks/code/output/ex3.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex3.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex4.out b/assets/pages/12_Turing_tricks/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex4.res b/assets/pages/12_Turing_tricks/code/output/ex4.res new file mode 100644 index 00000000..f32a5804 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex4.res @@ -0,0 +1 @@ +true \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex5.out b/assets/pages/12_Turing_tricks/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex5.res b/assets/pages/12_Turing_tricks/code/output/ex5.res new file mode 100644 index 00000000..1941cf8e --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex5.res @@ -0,0 +1,29 @@ +Chains MCMC chain (1000×17×4 Array{Float64, 3}): + +Iterations = 501:1:1500 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 6.24 seconds +Compute duration = 23.56 seconds +parameters = α, β[1], β[2], β[3], σ +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 21.5126 8.6720 0.2217 1528.8886 1868.2612 1.0009 64.8962 + β[1] 2.0014 1.7943 0.0419 2281.1940 1734.6512 1.0016 96.8290 + β[2] 0.5788 0.0584 0.0013 2163.9754 2292.8814 1.0006 91.8534 + β[3] 0.2566 0.3092 0.0074 1762.0214 2135.6795 1.0010 74.7919 + σ 17.8859 0.6033 0.0106 3271.1669 2347.2435 1.0008 138.8500 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α 4.7278 15.7633 21.2942 27.4322 38.4426 + β[1] -0.5876 0.7324 1.6761 2.9919 6.3388 + β[2] 0.4662 0.5392 0.5793 0.6184 0.6924 + β[3] -0.3477 0.0440 0.2588 0.4733 0.8490 + σ 16.7525 17.4685 17.8796 18.2703 19.1238 diff --git a/assets/pages/12_Turing_tricks/code/output/ex6.out b/assets/pages/12_Turing_tricks/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex6.res b/assets/pages/12_Turing_tricks/code/output/ex6.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex6.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/ex7.out b/assets/pages/12_Turing_tricks/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex7.res b/assets/pages/12_Turing_tricks/code/output/ex7.res new file mode 100644 index 00000000..9c613c38 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex7.res @@ -0,0 +1,29 @@ +Chains MCMC chain (1000×17×4 Array{Float64, 3}): + +Iterations = 1001:1:2000 +Number of chains = 4 +Samples per chain = 1000 +Wall duration = 2.65 seconds +Compute duration = 7.82 seconds +parameters = α, β[1], β[2], β[3], σ +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + α 32.9057 7.8274 0.2470 1006.3265 1242.3293 1.0026 128.7521 + β[1] -49.9922 7.0245 0.2210 1009.4898 1401.5358 1.0022 129.1568 + β[2] 22.0858 3.5735 0.1101 1054.2580 1418.8568 1.0028 134.8846 + β[3] 0.2869 0.8775 0.0238 1370.8734 1800.6496 1.0010 175.3932 + σ 17.8703 0.5859 0.0113 2699.9100 2464.9195 1.0019 345.4337 + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + α 17.6798 27.6922 32.7076 38.1740 47.9793 + β[1] -63.6746 -54.6970 -50.1683 -45.2700 -36.2948 + β[2] 15.1066 19.7019 22.1804 24.4485 29.1569 + β[3] -1.3554 -0.2981 0.2697 0.8374 2.1505 + σ 16.7293 17.4690 17.8683 18.2608 19.0084 diff --git a/assets/pages/12_Turing_tricks/code/output/ex8.out b/assets/pages/12_Turing_tricks/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex8.res b/assets/pages/12_Turing_tricks/code/output/ex8.res new file mode 100644 index 00000000..9d0ee1ba --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex8.res @@ -0,0 +1,33 @@ +Chains MCMC chain (1000×20×4 Array{Float64, 3}): + +Iterations = 1001:1:2000 +Number of chains = 4 +Samples per chain = 1000 +parameters = real_β[1], real_β[2], real_β[3], α, β[1], β[2], β[3], σ +internals = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size + +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Missing + + real_β[1] 6.2103 2.1904 0.0339 4190.5377 2075.4888 1.0015 missing + real_β[2] 0.5062 0.0616 0.0015 1657.4420 2135.3528 1.0021 missing + real_β[3] -0.0701 0.2143 0.0058 1370.8734 1800.6496 1.0010 missing + α 32.9057 7.8274 0.2470 1006.3265 1242.3293 1.0026 missing + β[1] -49.9922 7.0245 0.2210 1009.4898 1401.5358 1.0022 missing + β[2] 22.0858 3.5735 0.1101 1054.2580 1418.8568 1.0028 missing + β[3] 0.2869 0.8775 0.0238 1370.8734 1800.6496 1.0010 missing + σ 17.8703 0.5859 0.0113 2699.9100 2464.9195 1.0019 missing + +Quantiles + parameters 2.5% 25.0% 50.0% 75.0% 97.5% + Symbol Float64 Float64 Float64 Float64 Float64 + + real_β[1] 1.8485 4.7441 6.2339 7.7050 10.3766 + real_β[2] 0.3815 0.4656 0.5071 0.5480 0.6252 + real_β[3] -0.5252 -0.2045 -0.0659 0.0728 0.3310 + α 17.6798 27.6922 32.7076 38.1740 47.9793 + β[1] -63.6746 -54.6970 -50.1683 -45.2700 -36.2948 + β[2] 15.1066 19.7019 22.1804 24.4485 29.1569 + β[3] -1.3554 -0.2981 0.2697 0.8374 2.1505 + σ 16.7293 17.4690 17.8683 18.2608 19.0084 diff --git a/assets/pages/12_Turing_tricks/code/output/ex9.out b/assets/pages/12_Turing_tricks/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/12_Turing_tricks/code/output/ex9.res b/assets/pages/12_Turing_tricks/code/output/ex9.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/ex9.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/12_Turing_tricks/code/output/funnel.svg b/assets/pages/12_Turing_tricks/code/output/funnel.svg new file mode 100644 index 00000000..aa680663 --- /dev/null +++ b/assets/pages/12_Turing_tricks/code/output/funnel.svg @@ -0,0 +1,10191 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/13_epi_models/code/ex1.jl b/assets/pages/13_epi_models/code/ex1.jl new file mode 100644 index 00000000..ac6b4070 --- /dev/null +++ b/assets/pages/13_epi_models/code/ex1.jl @@ -0,0 +1,37 @@ +# This file was generated, do not modify it. # hide +using Downloads +using DataFrames +using CSV +using Chain +using Dates + +url = "https://data.brasil.io/dataset/covid19/caso_full.csv.gz" +file = Downloads.download(url) +df = DataFrame(CSV.File(file)) +br = @chain df begin + filter( + [:date, :city] => + (date, city) -> + date < Dates.Date("2021-01-01") && + date > Dates.Date("2020-04-01") && + ismissing(city), + _, + ) + groupby(:date) + combine( + [ + :estimated_population_2019, + :last_available_confirmed_per_100k_inhabitants, + :last_available_deaths, + :new_confirmed, + :new_deaths, + ] .=> + sum .=> [ + :estimated_population_2019, + :last_available_confirmed_per_100k_inhabitants, + :last_available_deaths, + :new_confirmed, + :new_deaths, + ], + ) +end; \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex2.jl b/assets/pages/13_epi_models/code/ex2.jl new file mode 100644 index 00000000..4382cd7d --- /dev/null +++ b/assets/pages/13_epi_models/code/ex2.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +first(br, 5) \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex3.jl b/assets/pages/13_epi_models/code/ex3.jl new file mode 100644 index 00000000..7023597b --- /dev/null +++ b/assets/pages/13_epi_models/code/ex3.jl @@ -0,0 +1,2 @@ +# This file was generated, do not modify it. # hide +last(br, 5) \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex4.jl b/assets/pages/13_epi_models/code/ex4.jl new file mode 100644 index 00000000..8084485e --- /dev/null +++ b/assets/pages/13_epi_models/code/ex4.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +using AlgebraOfGraphics +using CairoMakie +f = Figure() +plt = data(br) * mapping(:date => L"t", :new_confirmed => "infected daily") * visual(Lines) +draw!(f[1, 1], plt) +save(joinpath(@OUTPUT, "infected.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex5.jl b/assets/pages/13_epi_models/code/ex5.jl new file mode 100644 index 00000000..1622ea18 --- /dev/null +++ b/assets/pages/13_epi_models/code/ex5.jl @@ -0,0 +1,16 @@ +# This file was generated, do not modify it. # hide +using DifferentialEquations + +function sir_ode!(du, u, p, t) + (S, I, R) = u + (β, γ) = p + N = S + I + R + infection = β * I * S / N + recovery = γ * I + @inbounds begin + du[1] = -infection # Susceptible + du[2] = infection - recovery # Infected + du[3] = recovery # Recovered + end + return nothing +end; \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex6.jl b/assets/pages/13_epi_models/code/ex6.jl new file mode 100644 index 00000000..4677deb2 --- /dev/null +++ b/assets/pages/13_epi_models/code/ex6.jl @@ -0,0 +1,19 @@ +# This file was generated, do not modify it. # hide +i₀ = first(br[:, :new_confirmed]) +N = maximum(br[:, :estimated_population_2019]) + +u = [N - i₀, i₀, 0.0] +p = [0.5, 0.05] +prob = ODEProblem(sir_ode!, u, (1.0, 100.0), p) +sol_ode = solve(prob) +f = Figure() +plt = + data(stack(DataFrame(sol_ode), Not(:timestamp))) * + mapping( + :timestamp => L"t", + :value => L"N"; + color=:variable => renamer(["value1" => "S", "value2" => "I", "value3" => "R"]), + ) * + visual(Lines; linewidth=3) +draw!(f[1, 1], plt; axis=(; title="SIR Model for 100 days, β = $(p[1]), γ = $(p[2])")) +save(joinpath(@OUTPUT, "ode_solve.svg"), f); # hide \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex7.jl b/assets/pages/13_epi_models/code/ex7.jl new file mode 100644 index 00000000..706efcfa --- /dev/null +++ b/assets/pages/13_epi_models/code/ex7.jl @@ -0,0 +1,7 @@ +# This file was generated, do not modify it. # hide +function NegativeBinomial2(μ, ϕ) + p = 1 / (1 + μ / ϕ) + r = ϕ + + return NegativeBinomial(r, p) +end \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex8.jl b/assets/pages/13_epi_models/code/ex8.jl new file mode 100644 index 00000000..113f7760 --- /dev/null +++ b/assets/pages/13_epi_models/code/ex8.jl @@ -0,0 +1,34 @@ +# This file was generated, do not modify it. # hide +using Turing +using LazyArrays +using Random: seed! +seed!(123) +setprogress!(false) # hide + +@model function bayes_sir(infected, i₀, r₀, N) + #calculate number of timepoints + l = length(infected) + + #priors + β ~ TruncatedNormal(2, 1, 1e-4, 10) # using 10 because numerical issues arose + γ ~ TruncatedNormal(0.4, 0.5, 1e-4, 10) # using 10 because numerical issues arose + ϕ⁻ ~ truncated(Exponential(5); lower=0, upper=1e5) + ϕ = 1.0 / ϕ⁻ + + #ODE Stuff + I = i₀ + u0 = [N - I, I, r₀] # S,I,R + p = [β, γ] + tspan = (1.0, float(l)) + prob = ODEProblem(sir_ode!, u0, tspan, p) + sol = solve( + prob, + Tsit5(); # similar to Dormand-Prince RK45 in Stan but 20% faster + saveat=1.0, + ) + solᵢ = Array(sol)[2, :] # New Infected + solᵢ = max.(1e-4, solᵢ) # numerical issues arose + + #likelihood + return infected ~ arraydist(LazyArray(@~ NegativeBinomial2.(solᵢ, ϕ))) +end; \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/ex9.jl b/assets/pages/13_epi_models/code/ex9.jl new file mode 100644 index 00000000..2c134c6c --- /dev/null +++ b/assets/pages/13_epi_models/code/ex9.jl @@ -0,0 +1,6 @@ +# This file was generated, do not modify it. # hide +infected = br[:, :new_confirmed] +r₀ = first(br[:, :new_deaths]) +model_sir = bayes_sir(infected, i₀, r₀, N) +chain_sir = sample(model_sir, NUTS(), 1_000) +summarystats(chain_sir[[:β, :γ]]) \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex1.out b/assets/pages/13_epi_models/code/output/ex1.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex1.res b/assets/pages/13_epi_models/code/output/ex1.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex1.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex2.out b/assets/pages/13_epi_models/code/output/ex2.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex2.res b/assets/pages/13_epi_models/code/output/ex2.res new file mode 100644 index 00000000..a444e8a3 --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex2.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ date estimated_population_2019 last_available_confirmed_per_100k_inhabitants last_available_deaths new_confirmed new_deaths + │ Date Int64 Float64 Int64 Int64 Int64 +─────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + 1 │ 2020-04-02 210147125 79.6116 305 1167 61 + 2 │ 2020-04-03 210147125 90.9596 365 1114 60 + 3 │ 2020-04-04 210147125 103.622 445 1169 80 + 4 │ 2020-04-05 210147125 115.594 496 1040 51 + 5 │ 2020-04-06 210147125 125.766 569 840 73 \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex3.out b/assets/pages/13_epi_models/code/output/ex3.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex3.res b/assets/pages/13_epi_models/code/output/ex3.res new file mode 100644 index 00000000..b3ac01be --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex3.res @@ -0,0 +1,9 @@ +5×6 DataFrame + Row │ date estimated_population_2019 last_available_confirmed_per_100k_inhabitants last_available_deaths new_confirmed new_deaths + │ Date Int64 Float64 Int64 Int64 Int64 +─────┼──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────── + 1 │ 2020-12-27 210147125 1.22953e5 191250 17614 337 + 2 │ 2020-12-28 210147125 1.23554e5 191788 27437 538 + 3 │ 2020-12-29 210147125 1.24291e5 192839 56371 1051 + 4 │ 2020-12-30 210147125 1.25047e5 194056 55600 1217 + 5 │ 2020-12-31 210147125 1.25724e5 195072 54469 1016 \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex4.out b/assets/pages/13_epi_models/code/output/ex4.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex4.res b/assets/pages/13_epi_models/code/output/ex4.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex4.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex5.out b/assets/pages/13_epi_models/code/output/ex5.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex5.res b/assets/pages/13_epi_models/code/output/ex5.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex5.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex6.out b/assets/pages/13_epi_models/code/output/ex6.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex6.res b/assets/pages/13_epi_models/code/output/ex6.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex6.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex7.out b/assets/pages/13_epi_models/code/output/ex7.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex7.res b/assets/pages/13_epi_models/code/output/ex7.res new file mode 100644 index 00000000..8c8799e0 --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex7.res @@ -0,0 +1 @@ +NegativeBinomial2 (generic function with 1 method) \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex8.out b/assets/pages/13_epi_models/code/output/ex8.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex8.res b/assets/pages/13_epi_models/code/output/ex8.res new file mode 100644 index 00000000..97896a0b --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex8.res @@ -0,0 +1 @@ +nothing \ No newline at end of file diff --git a/assets/pages/13_epi_models/code/output/ex9.out b/assets/pages/13_epi_models/code/output/ex9.out new file mode 100644 index 00000000..e69de29b diff --git a/assets/pages/13_epi_models/code/output/ex9.res b/assets/pages/13_epi_models/code/output/ex9.res new file mode 100644 index 00000000..6c735786 --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ex9.res @@ -0,0 +1,6 @@ +Summary Statistics + parameters mean std mcse ess_bulk ess_tail rhat ess_per_sec + Symbol Float64 Float64 Float64 Float64 Float64 Float64 Float64 + + β 1.1199 0.0292 0.0013 477.2173 505.0551 0.9996 11.2615 + γ 1.0869 0.0296 0.0014 477.1981 493.8323 0.9996 11.2610 diff --git a/assets/pages/13_epi_models/code/output/infected.svg b/assets/pages/13_epi_models/code/output/infected.svg new file mode 100644 index 00000000..d1058152 --- /dev/null +++ b/assets/pages/13_epi_models/code/output/infected.svg @@ -0,0 +1,414 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/assets/pages/13_epi_models/code/output/ode_solve.svg b/assets/pages/13_epi_models/code/output/ode_solve.svg new file mode 100644 index 00000000..11c92362 --- /dev/null +++ b/assets/pages/13_epi_models/code/output/ode_solve.svg @@ -0,0 +1,520 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/build.log b/build.log new file mode 100644 index 00000000..0cef7af1 --- /dev/null +++ b/build.log @@ -0,0 +1,4 @@ + +up to date, audited 2 packages in 268ms + +found 0 vulnerabilities diff --git a/css/franklin.css b/css/franklin.css new file mode 100644 index 00000000..88f22e38 --- /dev/null +++ b/css/franklin.css @@ -0,0 +1 @@ +@charset "utf-8"; :root{--block-background:#f8f8f8;--small:14px;--normal:19px;--text-color:hsv(0,0%,20%)}html{font-family:Helvetica,Arial,sans-serif;font-size:var(--normal);color:var(--text-color)}.franklin-content .row{display:block}.franklin-content .left{float:left;margin-right:15px}.franklin-content .right{float:right}.franklin-content .container img{width:auto;padding-left:0;border-radius:10px}.franklin-content .footnote{position:relative;top:-0.5em;font-size:70%}.franklin-content .page-foot a{text-decoration:none;color:#a6a2a0;text-decoration:underline}.page-foot{font-size:80%;font-family:Arial,serif;color:#a6a2a0;text-align:center;margin-top:6em;border-top:1px solid lightgrey;padding-top:2em;margin-bottom:4em}.franklin-toc li{margin:.6rem 0}.franklin-content{position:relative;padding-left:12.5%;padding-right:12.5%;line-height:1.35em}@media (min-width:940px){.franklin-content{width:705px;margin-left:auto;margin-right:auto}}@media (max-width:480px){.franklin-content{padding-left:6%;padding-right:6%}}.franklin-content h1{font-size:24px}.franklin-content h2{font-size:22px}.franklin-content h3{font-size:20px}.franklin-content h1,h2,h3,h4,h5,h6{text-align:left}.franklin-content h1{padding-bottom:.5em;border-bottom:3px double lightgrey;margin-top:1.5em;margin-bottom:1em}.franklin-content h2{padding-bottom:.3em;border-bottom:1px solid lightgrey;margin-top:2em;margin-bottom:1em}.franklin-content h1 a{color:inherit}.franklin-content h1 a:hover{text-decoration:none}.franklin-content h2 a{color:inherit}.franklin-content h2 a:hover{text-decoration:none}.franklin-content h3 a{color:inherit}.franklin-content h3 a:hover{text-decoration:none}.franklin-content h4 a{color:inherit}.franklin-content h4 a:hover{text-decoration:none}.franklin-content h5 a{color:inherit}.franklin-content h5 a:hover{text-decoration:none}.franklin-content h6 a{color:inherit}.franklin-content h6 a:hover{text-decoration:none}.franklin-content table{margin-left:auto;margin-right:auto;border-collapse:collapse;text-align:center}.franklin-toc ol ol{list-style-type:lower-alpha}.franklin-content th,td{font-size:var(--small);padding:10px;border:1px solid black}.franklin-content blockquote{background:var(--block-background);border-left:7px solid #a8a8a8;margin:1.5em 10px;padding:.5em 10px;font-style:italic}.franklin-content blockquote p{display:inline}.franklin-content li p{margin:10px 0}.franklin-content a{color:#004de6;text-decoration:none}.franklin-content a:hover{text-decoration:underline}.franklin-content .eqref a{color:green}.franklin-content .bibref a{color:green}.franklin-content sup{font-size:70%;vertical-align:super;line-height:0}.franklin-content table.fndef{margin:0;margin-bottom:10px}.franklin-content .fndef tr,td{padding:0;border:0;text-align:left}.franklin-content .fndef tr{border-left:2px solid lightgray}.franklin-content .fndef td.fndef-backref{vertical-align:top;font-size:70%;padding-left:5px}.franklin-content .fndef td.fndef-content{font-size:80%;padding-left:10px;width:100%}.franklin-content img{width:70%;text-align:center;padding-left:10%}.franklin-content .img-small img{width:50%;text-align:center;padding-left:20%}body{counter-reset:eqnum}.katex{font-size:1em !important}.katex-display .katex{display:inline-block;white-space:normal}.katex-display::after{counter-increment:eqnum;content:"(" counter(eqnum) ")";position:relative;float:right;padding-right:5px}code{background-color:var(--block-background);padding:.1em .2em;border-radius:2px;font-size:var(--small)}.hljs{font-size:var(--small);line-height:1.35em;border-radius:10px}.hljs-meta,.hljs-metas,.hljs-metap{font-weight:bold}.hljs-meta{color:#19b333}.hljs-metas{color:red}.hljs-metap{color:#3383e7}.franklin-content .colbox-blue{background-color:#eef3f5;padding-top:5px;padding-right:10px;padding-left:10px;padding-bottom:5px;margin-left:5px;margin-top:5px;margin-bottom:5px;border-radius:0 10px 10px 0;border-left:5px solid #4c9cf1} \ No newline at end of file diff --git a/css/jtd.css b/css/jtd.css new file mode 100644 index 00000000..03e0586c --- /dev/null +++ b/css/jtd.css @@ -0,0 +1 @@ +@charset "utf-8"; html{font-family:sans-serif;-ms-text-size-adjust:100%;-webkit-text-size-adjust:100%;font-size:15px}@media (min-width:31.25rem){html{font-size:19px}}body{margin:0;position:relative;padding-bottom:4rem;font-family:-apple-system,"helvetica neue",helvetica,roboto,noto,"segoe ui",arial,sans-serif;font-size:inherit;line-height:1.5;color:#5c5962;background-color:#fff}@media (min-width:50rem){body{position:static;padding-bottom:0}}h1{font-size:2em;margin:.67em 0}sub,sup{font-size:75%;line-height:0;position:relative;vertical-align:baseline}sup{top:-0.5em}sub{bottom:-0.25em}img{border:0}p,h1,h2,h3,h4,h5,h6,ol,ul,pre,address,blockquote,dl,div,fieldset,form,hr,noscript,table{margin-top:0}h1,h2,h3,h4,h5,h6{margin-top:1.2em;margin-bottom:.8em;font-weight:500;line-height:1.15;color:#27262b}p{margin-bottom:1em}a{color:#0e39b0;text-decoration:none}pre{overflow:auto;margin:0;margin-bottom:.8em}code{font-family:"SFMono-Regular",Menlo,Consolas,monospace;font-size:14px;line-height:1.4;padding:.2em .15em;font-weight:400;background-color:#f5f6fa;border:1px solid #eeebee;border-radius:4px}li:not(:last-child){margin-bottom:5px}img{max-width:100%;height:auto}hr{height:1px;padding:0;margin:2rem 0;background-color:#eeebee;border:0}@media (min-width:50rem){.page-wrap{position:absolute;top:0;left:0;display:flex;width:100%;height:100%;overflow-x:hidden;overflow-y:hidden}}.side-bar{z-index:100;display:flex;flex-wrap:wrap;background-color:#f5f6fa}.side-bar .header{display:flex;min-height:60px;align-items:center}.side-bar .header .title{padding-right:1rem;padding-left:1rem;flex-grow:1;display:flex;height:100%;align-items:center;padding-top:.75rem;padding-bottom:.75rem;color:#27262b;font-size:18px !important;line-height:1.15em}.show-menu{cursor:pointer;appearance:none;display:flex;height:100%;line-height:60px;padding:1rem;align-items:center;color:#0e39b0;text-transform:uppercase;background:transparent;text-align:right;padding-right:2rem;border:0;width:calc(100% - 220px);background-color:white}.side-bar .menu{width:100%;padding-right:1rem;padding-left:1rem}.menu-list{padding:0;margin-top:0;margin-bottom:0;list-style:none}.menu-list-child-list{padding-left:.75rem;list-style:none}.menu-list-child-list .menu-list-link{color:#5c5962}.menu-list-child-list .menu-list-item{position:relative}.menu-list-child-list .menu-list-item::before{position:absolute;margin-top:.3em;margin-left:-0.8em;color:rgba(92,89,98,0.3);content:"- "}.menu-list-child-list .menu-list-item.active::before{color:#5c5962}.menu-list-item{font-size:14px !important;margin:0}.menu-list-item .menu-list-child-list{display:none}.menu-list-item.active .menu-list-child-list{display:block}.menu-list-link{display:block;padding-top:.25rem;padding-bottom:.25rem}.menu-list-link.active{font-weight:600;color:#27262b;text-decoration:none}.side-bar .footer{width:150px;font-size:13px;padding-right:1rem;padding-left:1rem;position:absolute;bottom:0;left:0;padding-top:1rem;padding-bottom:1rem}input[type=checkbox]{display:none}input[type=checkbox]:checked ~ #side-menu{display:block}@media (max-width:49.99rem){.side-bar .menu{display:none;padding-left:35px;padding-bottom:15px}.side-bar{border-bottom:1px solid #eeebee}.side-bar .header{padding-left:15px;width:150px}.show-menu{display:block}}@media (min-width:31.25rem){.side-bar .header .title{font-size:24px !important}.menu-list-item{font-size:16px !important}}@media (min-width:50rem){.side-bar{flex-wrap:nowrap;position:absolute;width:248px;height:100%;flex-direction:column;align-items:flex-end}.side-bar .header{z-index:101;height:60px;max-height:60px;border-bottom:1px solid #eeebee;padding-right:2rem;padding-left:2rem}.side-bar .header .title{padding-top:.5rem;padding-bottom:.5rem}.show-menu{display:none}.side-bar .menu{width:135px;padding-right:2rem;padding-left:2rem;padding-top:3rem;padding-bottom:1rem;overflow-y:auto;flex:1 1 auto}.side-bar .footer{padding-right:2rem;padding-left:2rem;position:static}}@media (min-width:66.5rem){.side-bar{width:calc((100% - 66.5rem) / 2 + 264px);min-width:264px}}@media (min-width:50rem){.main-content-wrap{position:absolute;top:0;left:0;width:100%;height:100%;-webkit-overflow-scrolling:touch;overflow-x:hidden;overflow-y:scroll}.main-content{position:relative;max-width:800px;margin-left:248px}}@media (min-width:66.5rem){.main-content{margin-left:calc((100% - 1064px) / 2 + 264px)}}.main-header{padding-right:1rem;padding-left:1rem;display:none;background-color:#f5f6fa;height:60px;line-height:60px}@media (min-width:50rem){.main-header{padding-right:2rem;padding-left:2rem}}@media (min-width:50rem){.main-header{display:flex;justify-content:flex-end;height:60px;background-color:#fff;border-bottom:1px solid #eeebee}}.main-footer{font-size:14px;color:darkgray;border-top:1px solid #eeebee;padding-top:15px}.franklin-content{padding-right:1rem;padding-left:1rem;padding-top:1rem;padding-bottom:1rem;line-height:1.5}@media (min-width:50rem){.franklin-content{padding-right:2rem;padding-left:2rem;padding-top:2rem;padding-bottom:2rem}}.franklin-content a{overflow:hidden;text-overflow:ellipsis;white-space:nowrap}.franklin-content ul,ol{line-height:1em}.franklin-content li{margin-top:.3em;line-height:1.3em}.franklin-content ul,.franklin-content ol{padding-left:1.5em}.franklin-content h1:first-of-type{margin-top:.5em}.franklin-content h1,h2,h3,h4,h5,h6{position:relative}td.fndef-backref{min-width:20px}.franklin-content table{margin-left:auto;margin-right:auto;border-collapse:collapse;text-align:center;margin-bottom:1em}.franklin-content tr,th,td{font-size:14px;padding:10px;border:1px solid black}.franklin-content table tbody tr td{border:1px solid black}.franklin-content blockquote{background:lemonchiffon;border-left:7px solid #a8a8a8;margin:1.5em 10px;padding:.5em 10px;font-style:italic}.franklin-content blockquote p{display:inline}.franklin-content .row{display:block}.franklin-content .left{float:left;margin-right:15px}.franklin-content .right{float:right}.franklin-content .container img{width:auto;padding-left:0;border-radius:10px;margin-bottom:1em}.franklin-content table.fndef{margin:0;margin-bottom:1em;border:0}.franklin-content .fndef tr{padding:0;border:0;text-align:left}.franklin-content .fndef td{padding:0;border:0}.franklin-content .fndef td.fndef-backref{vertical-align:top;font-size:80%;padding-left:5px}.franklin-content .fndef td.fndef-content{font-size:90%;padding-left:10px}.franklin-content .fndef tr{border-left:2px solid lightgray}.franklin-content img{width:70%;text-align:center;padding-left:10%}.franklin-content .img-small img{width:50%;text-align:center;padding-left:20%}body{counter-reset:eqnum}.katex{font-size:1em !important}.katex-display .katex{display:inline-block}.katex-display::after{counter-increment:eqnum;content:"(" counter(eqnum) ")";position:relative;float:right;padding-right:5px}.hljs{font-size:14px;line-height:1.35em;border-radius:10px;padding:1em}.hljs-meta,.hljs-metas,.hljs-metap{font-weight:bold}.hljs-meta{color:#19b333}.hljs-metas{color:red}.hljs-metap{color:#3383e7}franklin-toc{margin:.6rem 0}.note{margin-top:1.5em;width:95%;margin-left:auto;margin-right:auto;background-color:aliceblue;border-radius:5px;margin-bottom:1em}.note .content{padding:10px;padding-left:12px}.note .title{font-size:105%;border-top-left-radius:5px;border-top-right-radius:5px;padding-left:7px;padding-top:2px;color:white;background:cornflowerblue}.warning{margin-top:1.5em;width:95%;margin-left:auto;margin-right:auto;background-color:#f1b3b1;border-radius:5px;margin-bottom:1em}.warning .content{padding:10px;padding-left:12px}.warning .title{font-size:105%;border-top-left-radius:5px;border-top-right-radius:5px;padding-left:7px;padding-top:2px;color:white;background:crimson}.text-center{text-align:center} \ No newline at end of file diff --git a/images/bayes-meme.jpg b/images/bayes-meme.jpg new file mode 100644 index 00000000..06f76a79 Binary files /dev/null and b/images/bayes-meme.jpg differ diff --git a/images/julia_logo.svg b/images/julia_logo.svg new file mode 100644 index 00000000..e54ed4f1 --- /dev/null +++ b/images/julia_logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/index.html b/index.html new file mode 100644 index 00000000..4bcb4472 --- /dev/null +++ b/index.html @@ -0,0 +1,108 @@ + Bayesian Statistics using Julia and Turing
Bayesian Stats
Julia Logo
Code on GitHub

Bayesian Statistics using Julia and Turing

Bayesian for Everyone

Bayesian for Everyone

Welcome to the repository of tutorials on how to do Bayesian Statistics using Julia and Turing. Tutorials are available at storopoli.io/Bayesian-Julia.

Bayesian statistics is an approach to inferential statistics based on Bayes' theorem, where available knowledge about parameters in a statistical model is updated with the information in observed data. The background knowledge is expressed as a prior distribution and combined with observational data in the form of a likelihood function to determine the posterior distribution. The posterior can also be used for making predictions about future events.

Bayesian statistics is a departure from classical inferential statistics that prohibits probability statements about parameters and is based on asymptotically sampling infinite samples from a theoretical population and finding parameter values that maximize the likelihood function. Mostly notorious is null-hypothesis significance testing (NHST) based on p-values. Bayesian statistics incorporate uncertainty (and prior knowledge) by allowing probability statements about parameters, and the process of parameter value inference is a direct result of the Bayes' theorem.

  1. Julia
  2. Turing
  3. Author
  4. How to use the content?
  5. Tutorials
  6. Datasets
  7. What about other Turing tutorials?
  8. How to cite
  9. References
    1. Books
    2. Academic Papers
    3. Software
    4. Datasets
  10. License
  11. Environment

Julia

Julia is a fast dynamic-typed language that just-in-time (JIT) compiles into native code using LLVM. It "runs like C but reads like Python", meaning that is blazing fast, easy to prototype and to read/write code. It is multi-paradigm, combining features of imperative, functional, and object-oriented programming. I won't cover Julia basics and any sort of data manipulation using Julia in the tutorials, instead please take a look into the following resources which covers most of the introduction to Julia and how to work with tabular data in Julia:

  • Julia Documentation: Julia documentation is a very friendly and well-written resource that explains the basic design and functionality of the language.

  • Julia Data Science: open source and open access book on how to do Data Science using Julia.

  • Thinking Julia: introductory beginner-friendly book that explains the main concepts and functionality behind the Julia language.

  • Julia High Performance: book by two of the creators of the Julia Language (Avik Sengupta and Alan Edelman), it covers how to make Julia even faster with some principles and tricks of the trade.

  • An Introduction DataFrames: the package DataFrames.jl provides a set of tools for working with tabular data in Julia. Its design and functionality are similar to those of pandas (in Python) and data.frame, data.table and dplyr (in R), making it a great general purpose data science tool, especially for those coming to Julia from R or Python.This is a collection of notebooks that introduces DataFrames.jl made by one of its core contributors Bogumił Kamiński.

Turing

Turing is an ecosystem of Julia packages for Bayesian Inference using probabilistic programming. Models specified using Turing are easy to read and write — models work the way you write them. Like everything in Julia, Turing is fast.

Author

José Eduardo Storopoli, PhD - Lattes CV - ORCID - https://storopoli.io

How to use the content?

The content is licensed under a very permissive Creative Commons license (CC BY-SA). You are mostly welcome to contribute with issues and pull requests. My hope is to have more people into Bayesian statistics. The content is aimed towards social scientists and PhD candidates in social sciences. I chose to provide an intuitive approach rather than focusing on rigorous mathematical formulations. I've made it to be how I would have liked to be introduced to Bayesian statistics.

To configure a local environment:

  1. Download and install Julia

  2. Clone the repository from GitHub: git clone https://github.com/storopoli/Bayesian-Julia.git

  3. Access the directory: cd Bayesian-Julia

  4. Activate the environment by typing in the Julia REPL:

using Pkg
+Pkg.activate(".")
+Pkg.instantiate()

Tutorials

  1. Why Julia?

  2. What is Bayesian Statistics?

  3. Common Probability Distributions

  4. How to use Turing

  5. Markov Chain Monte Carlo (MCMC)

  6. Bayesian Linear Regression

  7. Bayesian Logistic Regression

  8. Bayesian Ordinal Regression

  9. Bayesian Regression with Count Data

  10. Robust Bayesian Regression

  11. Multilevel Models (a.k.a. Hierarchical Models)

  12. Computational Tricks with Turing (Non-Centered Parametrization and QR Decomposition)

  13. Epidemiological Models using ODE Solvers in Turing

Datasets

  • kidiq (linear regression): data from a survey of adult American women and their children (a subsample from the National Longitudinal Survey of Youth). Source: Gelman and Hill (2007).

  • wells (logistic regression): a survey of 3200 residents in a small area of Bangladesh suffering from arsenic contamination of groundwater. Respondents with elevated arsenic levels in their wells had been encouraged to switch their water source to a safe public or private well in the nearby area and the survey was conducted several years later to learn which of the affected residents had switched wells. Source: Gelman and Hill (2007).

  • esoph (ordinal regression): data from a case-control study of (o)esophageal cancer in Ille-et-Vilaine, France. Source: Breslow and Day (1980).

  • roaches (Poisson regression): data on the efficacy of a pest management system at reducing the number of roaches in urban apartments. Source: Gelman and Hill (2007).

  • duncan (robust regression): data from occupation's prestige filled with outliers. Source: Duncan (1961).

  • cheese (hierarchical models): data from cheese ratings. A group of 10 rural and 10 urban raters rated 4 types of different cheeses (A, B, C and D) in two samples. Source: Boatwright, McCulloch and Rossi (1999).

What about other Turing tutorials?

Despite not being the only Turing tutorial that exists, this tutorial aims to introduce Bayesian inference along with how to use Julia and Turing. Here is a (not complete) list of other Turing tutorials:

  1. Official Turing Tutorials: tutorials on how to implement common models in Turing

  2. Statistical Rethinking - Turing Models: Julia versions of the Bayesian models described in Statistical Rethinking Edition 1 (McElreath, 2016) and Edition 2 (McElreath, 2020)

  3. Håkan Kjellerstrand Turing Tutorials: a collection of Julia Turing models

I also have a free and open source graduate course on Bayesian Statistics with Turing and Stan code. You can find it at storopoli/Bayesian-Statistics.

How to cite

To cite these tutorials, please use:

Storopoli (2021). Bayesian Statistics with Julia and Turing. https://storopoli.io/Bayesian-Julia.
+

Or in BibTeX format LaTeX\LaTeX:

+
@misc{storopoli2021bayesianjulia,
+      author = {Storopoli, Jose},
+      title = {Bayesian Statistics with Julia and Turing},
+      url = {https://storopoli.io/Bayesian-Julia},
+      year = {2021}
+    }
+

References

+

The references are divided in books, papers, software, and datasets.

+

Books

+
    +

  • Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian Data Analysis. Chapman and Hall/CRC.

    + +

  • McElreath, R. (2020). Statistical rethinking: A Bayesian course with examples in R and Stan. CRC press.

    + +

  • Gelman, A., Hill, J., & Vehtari, A. (2020). Regression and other stories. Cambridge University Press.

    + +

  • Brooks, S., Gelman, A., Jones, G., & Meng, X.-L. (2011). Handbook of Markov Chain Monte Carlo. CRC Press. https://books.google.com?id=qfRsAIKZ4rIC

    +
      +

    • Geyer, C. J. (2011). Introduction to markov chain monte carlo. In S. Brooks, A. Gelman, G. L. Jones, & X.-L. Meng (Eds.), Handbook of markov chain monte carlo.

      + +
    + +
+

Academic Papers

+
    +

  • van de Schoot, R., Depaoli, S., King, R., Kramer, B., Märtens, K., Tadesse, M. G., Vannucci, M., Gelman, A., Veen, D., Willemsen, J., & Yau, C. (2021). Bayesian statistics and modelling. Nature Reviews Methods Primers, 1(1, 1), 1–26. https://doi.org/10.1038/s43586-020-00001-2

    + +

  • Gabry, J., Simpson, D., Vehtari, A., Betancourt, M., & Gelman, A. (2019). Visualization in Bayesian workflow. Journal of the Royal Statistical Society: Series A (Statistics in Society), 182(2), 389–402. https://doi.org/10.1111/rssa.12378

    + +

  • Gelman, A., Vehtari, A., Simpson, D., Margossian, C. C., Carpenter, B., Yao, Y., Kennedy, L., Gabry, J., Bürkner, P.-C., & Modr’ak, M. (2020, November 3). Bayesian Workflow. http://arxiv.org/abs/2011.01808

    + +

  • Benjamin, D. J., Berger, J. O., Johannesson, M., Nosek, B. A., Wagenmakers, E.-J., Berk, R., Bollen, K. A., Brembs, B., Brown, L., Camerer, C., Cesarini, D., Chambers, C. D., Clyde, M., Cook, T. D., De Boeck, P., Dienes, Z., Dreber, A., Easwaran, K., Efferson, C., … Johnson, V. E. (2018). Redefine statistical significance. Nature Human Behaviour, 2(1), 6–10. https://doi.org/10.1038/s41562-017-0189-z

    + +

  • McShane, B. B., Gal, D., Gelman, A., Robert, C., & Tackett, J. L. (2019). Abandon Statistical Significance. American Statistician, 73, 235–245. https://doi.org/10.1080/00031305.2018.1527253

    + +

  • Amrhein, V., Greenland, S., & McShane, B. (2019). Scientists rise up against statistical significance. Nature, 567(7748), 305–307. https://doi.org/10.1038/d41586-019-00857-9

    + +

  • van de Schoot, R., Kaplan, D., Denissen, J., Asendorpf, J. B., Neyer, F. J., & van Aken, M. A. G. (2014). A Gentle Introduction to Bayesian Analysis: Applications to Developmental Research. Child Development, 85(3), 842–860. https://doi.org/10.1111/cdev.12169

    + +
+

Software

+
    +

  • Bezanson, J., Edelman, A., Karpinski, S., & Shah, V. B. (2017). Julia: A fresh approach to numerical computing. SIAM Review, 59(1), 65–98.

    + +

  • Ge, H., Xu, K., & Ghahramani, Z. (2018). Turing: A Language for Flexible Probabilistic Inference. International Conference on Artificial Intelligence and Statistics, 1682–1690. http://proceedings.mlr.press/v84/ge18b.html

    + +

  • Tarek, M., Xu, K., Trapp, M., Ge, H., & Ghahramani, Z. (2020). DynamicPPL: Stan-like Speed for Dynamic Probabilistic Models. ArXiv:2002.02702 [Cs, Stat]. http://arxiv.org/abs/2002.02702

    + +

  • Xu, K., Ge, H., Tebbutt, W., Tarek, M., Trapp, M., & Ghahramani, Z. (2020). AdvancedHMC.jl: A robust, modular and efficient implementation of advanced HMC algorithms. Symposium on Advances in Approximate Bayesian Inference, 1–10. http://proceedings.mlr.press/v118/xu20a.html

    + +

  • Revels, J., Lubin, M., & Papamarkou, T. (2016). Forward-Mode Automatic Differentiation in Julia. ArXiv:1607.07892 [Cs]. http://arxiv.org/abs/1607.07892

    + +
+

Datasets

+
    +

  • Boatwright, P., McCulloch, R., & Rossi, P. (1999). Account-level modeling for trade promotion: An application of a constrained parameter hierarchical model. Journal of the American Statistical Association, 94(448), 1063–1073.

    + +

  • Breslow, N. E. & Day, N. E. (1980). Statistical Methods in Cancer Research. Volume 1: The Analysis of Case-Control Studies. IARC Lyon / Oxford University Press.

    + +

  • Duncan, O. D. (1961). A socioeconomic index for all occupations. Class: Critical Concepts, 1, 388–426.

    + +

  • Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press.

    + +
+

License

+

This content is licensed under Creative Commons Attribution-ShareAlike 4.0 International.

+

Environment

+ +

This website is built with Julia 1.10.0 and

+ +
AlgebraOfGraphics 0.6.18
+BenchmarkTools 1.4.0
+Bijectors 0.13.8
+CSV 0.10.12
+CairoMakie 0.11.6
+CategoricalArrays 0.10.8
+Chain 0.5.0
+Colors 0.12.10
+DataFrames 1.6.1
+DifferentialEquations 7.12.0
+Distributions 0.25.107
+Downloads 1.6.0
+FillArrays 1.9.3
+ForwardDiff 0.10.36
+Franklin 0.10.95
+HTTP 1.10.1
+LazyArrays 1.8.3
+Literate 2.16.1
+MCMCChains 6.0.4
+NodeJS 2.0.0
+PDMats 0.11.31
+StatsBase 0.34.2
+StatsFuns 1.3.0
+Turing 0.30.3
+
+
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/libs/highlight/github.min.css b/libs/highlight/github.min.css new file mode 100644 index 00000000..6bf39bdb --- /dev/null +++ b/libs/highlight/github.min.css @@ -0,0 +1,43 @@ +.hljs { display: block; font-size: 14px; line-height: 1.45em; overflow-x: auto; padding: 0.5em; color: var(--text-color); background: var(--block-background); } + +.hljs-comment, +.hljs-quote { color: #998; font-style: italic; } + +.hljs-keyword, +.hljs-selector-tag, +.hljs-subst { color: var(--text-color); font-weight: 700; } + +.hljs-literal, +.hljs-number, +.hljs-tag .hljs-attr, +.hljs-template-variable, +.hljs-variable { color: teal; } + +.hljs-doctag, +.hljs-string { color: #d14; } + +.hljs-section, +.hljs-selector-id, +.hljs-title { color: #900; font-weight: 700; } +.hljs-subst { font-weight: 400; } + +.hljs-class .hljs-title, +.hljs-type { color: #458; font-weight: 700; } + +.hljs-attribute, +.hljs-name, +.hljs-tag { color: navy; font-weight: 400; } + +.hljs-link, +.hljs-regexp { color: #009926; } + +.hljs-bullet, +.hljs-symbol { color: #990073; } + +.hljs-built_in, +.hljs-builtin-name { color: #0086b3; } +.hljs-meta { color: var(--text-color); font-weight: 700; } +.hljs-deletion { background: #fdd; } +.hljs-addition { background: #dfd; } +.hljs-emphasis { font-style: italic; } +.hljs-strong { font-weight: 700; } diff --git a/libs/highlight/highlight.pack.js b/libs/highlight/highlight.pack.js new file mode 100644 index 00000000..4cf513c8 --- /dev/null +++ b/libs/highlight/highlight.pack.js @@ -0,0 +1,671 @@ +/* + Highlight.js 10.7.1 (421b23b0) + License: BSD-3-Clause + Copyright (c) 2006-2021, Ivan Sagalaev +*/ +var hljs=function(){"use strict";function e(t){ +return t instanceof Map?t.clear=t.delete=t.set=()=>{ +throw Error("map is read-only")}:t instanceof Set&&(t.add=t.clear=t.delete=()=>{ +throw Error("set is read-only") +}),Object.freeze(t),Object.getOwnPropertyNames(t).forEach((n=>{var i=t[n] +;"object"!=typeof i||Object.isFrozen(i)||e(i)})),t}var t=e,n=e;t.default=n +;class i{constructor(e){ +void 0===e.data&&(e.data={}),this.data=e.data,this.isMatchIgnored=!1} +ignoreMatch(){this.isMatchIgnored=!0}}function s(e){ +return e.replace(/&/g,"&").replace(//g,">").replace(/"/g,""").replace(/'/g,"'") +}function a(e,...t){const n=Object.create(null);for(const t in e)n[t]=e[t] +;return t.forEach((e=>{for(const t in e)n[t]=e[t]})),n}const r=e=>!!e.kind +;class l{constructor(e,t){ +this.buffer="",this.classPrefix=t.classPrefix,e.walk(this)}addText(e){ +this.buffer+=s(e)}openNode(e){if(!r(e))return;let t=e.kind +;e.sublanguage||(t=`${this.classPrefix}${t}`),this.span(t)}closeNode(e){ +r(e)&&(this.buffer+="")}value(){return this.buffer}span(e){ +this.buffer+=``}}class o{constructor(){this.rootNode={ +children:[]},this.stack=[this.rootNode]}get top(){ +return this.stack[this.stack.length-1]}get root(){return this.rootNode}add(e){ +this.top.children.push(e)}openNode(e){const t={kind:e,children:[]} +;this.add(t),this.stack.push(t)}closeNode(){ +if(this.stack.length>1)return this.stack.pop()}closeAllNodes(){ +for(;this.closeNode(););}toJSON(){return JSON.stringify(this.rootNode,null,4)} +walk(e){return this.constructor._walk(e,this.rootNode)}static _walk(e,t){ +return"string"==typeof t?e.addText(t):t.children&&(e.openNode(t), +t.children.forEach((t=>this._walk(e,t))),e.closeNode(t)),e}static _collapse(e){ +"string"!=typeof e&&e.children&&(e.children.every((e=>"string"==typeof e))?e.children=[e.children.join("")]:e.children.forEach((e=>{ +o._collapse(e)})))}}class c extends o{constructor(e){super(),this.options=e} +addKeyword(e,t){""!==e&&(this.openNode(t),this.addText(e),this.closeNode())} +addText(e){""!==e&&this.add(e)}addSublanguage(e,t){const n=e.root +;n.kind=t,n.sublanguage=!0,this.add(n)}toHTML(){ +return new l(this,this.options).value()}finalize(){return!0}}function g(e){ +return e?"string"==typeof e?e:e.source:null} +const u=/\[(?:[^\\\]]|\\.)*\]|\(\??|\\([1-9][0-9]*)|\\./,h="[a-zA-Z]\\w*",d="[a-zA-Z_]\\w*",f="\\b\\d+(\\.\\d+)?",p="(-?)(\\b0[xX][a-fA-F0-9]+|(\\b\\d+(\\.\\d*)?|\\.\\d+)([eE][-+]?\\d+)?)",m="\\b(0b[01]+)",b={ +begin:"\\\\[\\s\\S]",relevance:0},E={className:"string",begin:"'",end:"'", +illegal:"\\n",contains:[b]},x={className:"string",begin:'"',end:'"', +illegal:"\\n",contains:[b]},v={ +begin:/\b(a|an|the|are|I'm|isn't|don't|doesn't|won't|but|just|should|pretty|simply|enough|gonna|going|wtf|so|such|will|you|your|they|like|more)\b/ +},w=(e,t,n={})=>{const i=a({className:"comment",begin:e,end:t,contains:[]},n) +;return i.contains.push(v),i.contains.push({className:"doctag", +begin:"(?:TODO|FIXME|NOTE|BUG|OPTIMIZE|HACK|XXX):",relevance:0}),i +},y=w("//","$"),N=w("/\\*","\\*/"),R=w("#","$");var _=Object.freeze({ +__proto__:null,MATCH_NOTHING_RE:/\b\B/,IDENT_RE:h,UNDERSCORE_IDENT_RE:d, +NUMBER_RE:f,C_NUMBER_RE:p,BINARY_NUMBER_RE:m, +RE_STARTERS_RE:"!|!=|!==|%|%=|&|&&|&=|\\*|\\*=|\\+|\\+=|,|-|-=|/=|/|:|;|<<|<<=|<=|<|===|==|=|>>>=|>>=|>=|>>>|>>|>|\\?|\\[|\\{|\\(|\\^|\\^=|\\||\\|=|\\|\\||~", +SHEBANG:(e={})=>{const t=/^#![ ]*\// +;return e.binary&&(e.begin=((...e)=>e.map((e=>g(e))).join(""))(t,/.*\b/,e.binary,/\b.*/)), +a({className:"meta",begin:t,end:/$/,relevance:0,"on:begin":(e,t)=>{ +0!==e.index&&t.ignoreMatch()}},e)},BACKSLASH_ESCAPE:b,APOS_STRING_MODE:E, +QUOTE_STRING_MODE:x,PHRASAL_WORDS_MODE:v,COMMENT:w,C_LINE_COMMENT_MODE:y, +C_BLOCK_COMMENT_MODE:N,HASH_COMMENT_MODE:R,NUMBER_MODE:{className:"number", +begin:f,relevance:0},C_NUMBER_MODE:{className:"number",begin:p,relevance:0}, +BINARY_NUMBER_MODE:{className:"number",begin:m,relevance:0},CSS_NUMBER_MODE:{ +className:"number", +begin:f+"(%|em|ex|ch|rem|vw|vh|vmin|vmax|cm|mm|in|pt|pc|px|deg|grad|rad|turn|s|ms|Hz|kHz|dpi|dpcm|dppx)?", +relevance:0},REGEXP_MODE:{begin:/(?=\/[^/\n]*\/)/,contains:[{className:"regexp", +begin:/\//,end:/\/[gimuy]*/,illegal:/\n/,contains:[b,{begin:/\[/,end:/\]/, +relevance:0,contains:[b]}]}]},TITLE_MODE:{className:"title",begin:h,relevance:0 +},UNDERSCORE_TITLE_MODE:{className:"title",begin:d,relevance:0},METHOD_GUARD:{ +begin:"\\.\\s*[a-zA-Z_]\\w*",relevance:0},END_SAME_AS_BEGIN:e=>Object.assign(e,{ +"on:begin":(e,t)=>{t.data._beginMatch=e[1]},"on:end":(e,t)=>{ +t.data._beginMatch!==e[1]&&t.ignoreMatch()}})});function k(e,t){ +"."===e.input[e.index-1]&&t.ignoreMatch()}function M(e,t){ +t&&e.beginKeywords&&(e.begin="\\b("+e.beginKeywords.split(" ").join("|")+")(?!\\.)(?=\\b|\\s)", +e.__beforeBegin=k,e.keywords=e.keywords||e.beginKeywords,delete e.beginKeywords, +void 0===e.relevance&&(e.relevance=0))}function O(e,t){ +Array.isArray(e.illegal)&&(e.illegal=((...e)=>"("+e.map((e=>g(e))).join("|")+")")(...e.illegal)) +}function A(e,t){if(e.match){ +if(e.begin||e.end)throw Error("begin & end are not supported with match") +;e.begin=e.match,delete e.match}}function L(e,t){ +void 0===e.relevance&&(e.relevance=1)} +const I=["of","and","for","in","not","or","if","then","parent","list","value"] +;function j(e,t,n="keyword"){const i={} +;return"string"==typeof e?s(n,e.split(" ")):Array.isArray(e)?s(n,e):Object.keys(e).forEach((n=>{ +Object.assign(i,j(e[n],t,n))})),i;function s(e,n){ +t&&(n=n.map((e=>e.toLowerCase()))),n.forEach((t=>{const n=t.split("|") +;i[n[0]]=[e,B(n[0],n[1])]}))}}function B(e,t){ +return t?Number(t):(e=>I.includes(e.toLowerCase()))(e)?0:1} +function T(e,{plugins:t}){function n(t,n){ +return RegExp(g(t),"m"+(e.case_insensitive?"i":"")+(n?"g":""))}class i{ +constructor(){ +this.matchIndexes={},this.regexes=[],this.matchAt=1,this.position=0} +addRule(e,t){ +t.position=this.position++,this.matchIndexes[this.matchAt]=t,this.regexes.push([t,e]), +this.matchAt+=(e=>RegExp(e.toString()+"|").exec("").length-1)(e)+1}compile(){ +0===this.regexes.length&&(this.exec=()=>null) +;const e=this.regexes.map((e=>e[1]));this.matcherRe=n(((e,t="|")=>{let n=0 +;return e.map((e=>{n+=1;const t=n;let i=g(e),s="";for(;i.length>0;){ +const e=u.exec(i);if(!e){s+=i;break} +s+=i.substring(0,e.index),i=i.substring(e.index+e[0].length), +"\\"===e[0][0]&&e[1]?s+="\\"+(Number(e[1])+t):(s+=e[0],"("===e[0]&&n++)}return s +})).map((e=>`(${e})`)).join(t)})(e),!0),this.lastIndex=0}exec(e){ +this.matcherRe.lastIndex=this.lastIndex;const t=this.matcherRe.exec(e) +;if(!t)return null +;const n=t.findIndex(((e,t)=>t>0&&void 0!==e)),i=this.matchIndexes[n] +;return t.splice(0,n),Object.assign(t,i)}}class s{constructor(){ +this.rules=[],this.multiRegexes=[], +this.count=0,this.lastIndex=0,this.regexIndex=0}getMatcher(e){ +if(this.multiRegexes[e])return this.multiRegexes[e];const t=new i +;return this.rules.slice(e).forEach((([e,n])=>t.addRule(e,n))), +t.compile(),this.multiRegexes[e]=t,t}resumingScanAtSamePosition(){ +return 0!==this.regexIndex}considerAll(){this.regexIndex=0}addRule(e,t){ +this.rules.push([e,t]),"begin"===t.type&&this.count++}exec(e){ +const t=this.getMatcher(this.regexIndex);t.lastIndex=this.lastIndex +;let n=t.exec(e) +;if(this.resumingScanAtSamePosition())if(n&&n.index===this.lastIndex);else{ +const t=this.getMatcher(0);t.lastIndex=this.lastIndex+1,n=t.exec(e)} +return n&&(this.regexIndex+=n.position+1, +this.regexIndex===this.count&&this.considerAll()),n}} +if(e.compilerExtensions||(e.compilerExtensions=[]), +e.contains&&e.contains.includes("self"))throw Error("ERR: contains `self` is not supported at the top-level of a language. See documentation.") +;return e.classNameAliases=a(e.classNameAliases||{}),function t(i,r){const l=i +;if(i.isCompiled)return l +;[A].forEach((e=>e(i,r))),e.compilerExtensions.forEach((e=>e(i,r))), +i.__beforeBegin=null,[M,O,L].forEach((e=>e(i,r))),i.isCompiled=!0;let o=null +;if("object"==typeof i.keywords&&(o=i.keywords.$pattern, +delete i.keywords.$pattern), +i.keywords&&(i.keywords=j(i.keywords,e.case_insensitive)), +i.lexemes&&o)throw Error("ERR: Prefer `keywords.$pattern` to `mode.lexemes`, BOTH are not allowed. (see mode reference) ") +;return o=o||i.lexemes||/\w+/, +l.keywordPatternRe=n(o,!0),r&&(i.begin||(i.begin=/\B|\b/), +l.beginRe=n(i.begin),i.endSameAsBegin&&(i.end=i.begin), +i.end||i.endsWithParent||(i.end=/\B|\b/), +i.end&&(l.endRe=n(i.end)),l.terminatorEnd=g(i.end)||"", +i.endsWithParent&&r.terminatorEnd&&(l.terminatorEnd+=(i.end?"|":"")+r.terminatorEnd)), +i.illegal&&(l.illegalRe=n(i.illegal)), +i.contains||(i.contains=[]),i.contains=[].concat(...i.contains.map((e=>(e=>(e.variants&&!e.cachedVariants&&(e.cachedVariants=e.variants.map((t=>a(e,{ +variants:null},t)))),e.cachedVariants?e.cachedVariants:S(e)?a(e,{ +starts:e.starts?a(e.starts):null +}):Object.isFrozen(e)?a(e):e))("self"===e?i:e)))),i.contains.forEach((e=>{t(e,l) +})),i.starts&&t(i.starts,r),l.matcher=(e=>{const t=new s +;return e.contains.forEach((e=>t.addRule(e.begin,{rule:e,type:"begin" +}))),e.terminatorEnd&&t.addRule(e.terminatorEnd,{type:"end" +}),e.illegal&&t.addRule(e.illegal,{type:"illegal"}),t})(l),l}(e)}function S(e){ +return!!e&&(e.endsWithParent||S(e.starts))}function P(e){const t={ +props:["language","code","autodetect"],data:()=>({detectedLanguage:"", +unknownLanguage:!1}),computed:{className(){ +return this.unknownLanguage?"":"hljs "+this.detectedLanguage},highlighted(){ +if(!this.autoDetect&&!e.getLanguage(this.language))return console.warn(`The language "${this.language}" you specified could not be found.`), +this.unknownLanguage=!0,s(this.code);let t={} +;return this.autoDetect?(t=e.highlightAuto(this.code), +this.detectedLanguage=t.language):(t=e.highlight(this.language,this.code,this.ignoreIllegals), +this.detectedLanguage=this.language),t.value},autoDetect(){ +return!(this.language&&(e=this.autodetect,!e&&""!==e));var e}, +ignoreIllegals:()=>!0},render(e){return e("pre",{},[e("code",{ +class:this.className,domProps:{innerHTML:this.highlighted}})])}};return{ +Component:t,VuePlugin:{install(e){e.component("highlightjs",t)}}}}const D={ +"after:highlightElement":({el:e,result:t,text:n})=>{const i=H(e) +;if(!i.length)return;const a=document.createElement("div") +;a.innerHTML=t.value,t.value=((e,t,n)=>{let i=0,a="";const r=[];function l(){ +return e.length&&t.length?e[0].offset!==t[0].offset?e[0].offset"}function c(e){ +a+=""}function g(e){("start"===e.event?o:c)(e.node)} +for(;e.length||t.length;){let t=l() +;if(a+=s(n.substring(i,t[0].offset)),i=t[0].offset,t===e){r.reverse().forEach(c) +;do{g(t.splice(0,1)[0]),t=l()}while(t===e&&t.length&&t[0].offset===i) +;r.reverse().forEach(o) +}else"start"===t[0].event?r.push(t[0].node):r.pop(),g(t.splice(0,1)[0])} +return a+s(n.substr(i))})(i,H(a),n)}};function C(e){ +return e.nodeName.toLowerCase()}function H(e){const t=[];return function e(n,i){ +for(let s=n.firstChild;s;s=s.nextSibling)3===s.nodeType?i+=s.nodeValue.length:1===s.nodeType&&(t.push({ +event:"start",offset:i,node:s}),i=e(s,i),C(s).match(/br|hr|img|input/)||t.push({ +event:"stop",offset:i,node:s}));return i}(e,0),t}const $=e=>{console.error(e) +},U=(e,...t)=>{console.log("WARN: "+e,...t)},z=(e,t)=>{ +console.log(`Deprecated as of ${e}. ${t}`)},K=s,G=a,V=Symbol("nomatch") +;return(e=>{const n=Object.create(null),s=Object.create(null),a=[];let r=!0 +;const l=/(^(<[^>]+>|\t|)+|\n)/gm,o="Could not find the language '{}', did you forget to load/include a language module?",g={ +disableAutodetect:!0,name:"Plain text",contains:[]};let u={ +noHighlightRe:/^(no-?highlight)$/i, +languageDetectRe:/\blang(?:uage)?-([\w-]+)\b/i,classPrefix:"hljs-", +tabReplace:null,useBR:!1,languages:null,__emitter:c};function h(e){ +return u.noHighlightRe.test(e)}function d(e,t,n,i){let s="",a="" +;"object"==typeof t?(s=e, +n=t.ignoreIllegals,a=t.language,i=void 0):(z("10.7.0","highlight(lang, code, ...args) has been deprecated."), +z("10.7.0","Please use highlight(code, options) instead.\nhttps://github.com/highlightjs/highlight.js/issues/2277"), +a=e,s=t);const r={code:s,language:a};M("before:highlight",r) +;const l=r.result?r.result:f(r.language,r.code,n,i) +;return l.code=r.code,M("after:highlight",l),l}function f(e,t,s,l){ +function c(e,t){const n=v.case_insensitive?t[0].toLowerCase():t[0] +;return Object.prototype.hasOwnProperty.call(e.keywords,n)&&e.keywords[n]} +function g(){null!=R.subLanguage?(()=>{if(""===M)return;let e=null +;if("string"==typeof R.subLanguage){ +if(!n[R.subLanguage])return void k.addText(M) +;e=f(R.subLanguage,M,!0,_[R.subLanguage]),_[R.subLanguage]=e.top +}else e=p(M,R.subLanguage.length?R.subLanguage:null) +;R.relevance>0&&(O+=e.relevance),k.addSublanguage(e.emitter,e.language) +})():(()=>{if(!R.keywords)return void k.addText(M);let e=0 +;R.keywordPatternRe.lastIndex=0;let t=R.keywordPatternRe.exec(M),n="";for(;t;){ +n+=M.substring(e,t.index);const i=c(R,t);if(i){const[e,s]=i +;if(k.addText(n),n="",O+=s,e.startsWith("_"))n+=t[0];else{ +const n=v.classNameAliases[e]||e;k.addKeyword(t[0],n)}}else n+=t[0] +;e=R.keywordPatternRe.lastIndex,t=R.keywordPatternRe.exec(M)} +n+=M.substr(e),k.addText(n)})(),M=""}function h(e){ +return e.className&&k.openNode(v.classNameAliases[e.className]||e.className), +R=Object.create(e,{parent:{value:R}}),R}function d(e,t,n){let s=((e,t)=>{ +const n=e&&e.exec(t);return n&&0===n.index})(e.endRe,n);if(s){if(e["on:end"]){ +const n=new i(e);e["on:end"](t,n),n.isMatchIgnored&&(s=!1)}if(s){ +for(;e.endsParent&&e.parent;)e=e.parent;return e}} +if(e.endsWithParent)return d(e.parent,t,n)}function m(e){ +return 0===R.matcher.regexIndex?(M+=e[0],1):(I=!0,0)}function b(e){ +const n=e[0],i=t.substr(e.index),s=d(R,e,i);if(!s)return V;const a=R +;a.skip?M+=n:(a.returnEnd||a.excludeEnd||(M+=n),g(),a.excludeEnd&&(M=n));do{ +R.className&&k.closeNode(),R.skip||R.subLanguage||(O+=R.relevance),R=R.parent +}while(R!==s.parent) +;return s.starts&&(s.endSameAsBegin&&(s.starts.endRe=s.endRe), +h(s.starts)),a.returnEnd?0:n.length}let E={};function x(n,a){const l=a&&a[0] +;if(M+=n,null==l)return g(),0 +;if("begin"===E.type&&"end"===a.type&&E.index===a.index&&""===l){ +if(M+=t.slice(a.index,a.index+1),!r){const t=Error("0 width match regex") +;throw t.languageName=e,t.badRule=E.rule,t}return 1} +if(E=a,"begin"===a.type)return function(e){ +const t=e[0],n=e.rule,s=new i(n),a=[n.__beforeBegin,n["on:begin"]] +;for(const n of a)if(n&&(n(e,s),s.isMatchIgnored))return m(t) +;return n&&n.endSameAsBegin&&(n.endRe=RegExp(t.replace(/[-/\\^$*+?.()|[\]{}]/g,"\\$&"),"m")), +n.skip?M+=t:(n.excludeBegin&&(M+=t), +g(),n.returnBegin||n.excludeBegin||(M=t)),h(n),n.returnBegin?0:t.length}(a) +;if("illegal"===a.type&&!s){ +const e=Error('Illegal lexeme "'+l+'" for mode "'+(R.className||"")+'"') +;throw e.mode=R,e}if("end"===a.type){const e=b(a);if(e!==V)return e} +if("illegal"===a.type&&""===l)return 1 +;if(L>1e5&&L>3*a.index)throw Error("potential infinite loop, way more iterations than matches") +;return M+=l,l.length}const v=N(e) +;if(!v)throw $(o.replace("{}",e)),Error('Unknown language: "'+e+'"') +;const w=T(v,{plugins:a});let y="",R=l||w;const _={},k=new u.__emitter(u);(()=>{ +const e=[];for(let t=R;t!==v;t=t.parent)t.className&&e.unshift(t.className) +;e.forEach((e=>k.openNode(e)))})();let M="",O=0,A=0,L=0,I=!1;try{ +for(R.matcher.considerAll();;){ +L++,I?I=!1:R.matcher.considerAll(),R.matcher.lastIndex=A +;const e=R.matcher.exec(t);if(!e)break;const n=x(t.substring(A,e.index),e) +;A=e.index+n}return x(t.substr(A)),k.closeAllNodes(),k.finalize(),y=k.toHTML(),{ +relevance:Math.floor(O),value:y,language:e,illegal:!1,emitter:k,top:R}}catch(n){ +if(n.message&&n.message.includes("Illegal"))return{illegal:!0,illegalBy:{ +msg:n.message,context:t.slice(A-100,A+100),mode:n.mode},sofar:y,relevance:0, +value:K(t),emitter:k};if(r)return{illegal:!1,relevance:0,value:K(t),emitter:k, +language:e,top:R,errorRaised:n};throw n}}function p(e,t){ +t=t||u.languages||Object.keys(n);const i=(e=>{const t={relevance:0, +emitter:new u.__emitter(u),value:K(e),illegal:!1,top:g} +;return t.emitter.addText(e),t})(e),s=t.filter(N).filter(k).map((t=>f(t,e,!1))) +;s.unshift(i);const a=s.sort(((e,t)=>{ +if(e.relevance!==t.relevance)return t.relevance-e.relevance +;if(e.language&&t.language){if(N(e.language).supersetOf===t.language)return 1 +;if(N(t.language).supersetOf===e.language)return-1}return 0})),[r,l]=a,o=r +;return o.second_best=l,o}const m={"before:highlightElement":({el:e})=>{ +u.useBR&&(e.innerHTML=e.innerHTML.replace(/\n/g,"").replace(//g,"\n")) +},"after:highlightElement":({result:e})=>{ +u.useBR&&(e.value=e.value.replace(/\n/g,"
"))}},b=/^(<[^>]+>|\t)+/gm,E={ +"after:highlightElement":({result:e})=>{ +u.tabReplace&&(e.value=e.value.replace(b,(e=>e.replace(/\t/g,u.tabReplace))))}} +;function x(e){let t=null;const n=(e=>{let t=e.className+" " +;t+=e.parentNode?e.parentNode.className:"";const n=u.languageDetectRe.exec(t) +;if(n){const t=N(n[1]) +;return t||(U(o.replace("{}",n[1])),U("Falling back to no-highlight mode for this block.",e)), +t?n[1]:"no-highlight"}return t.split(/\s+/).find((e=>h(e)||N(e)))})(e) +;if(h(n))return;M("before:highlightElement",{el:e,language:n}),t=e +;const i=t.textContent,a=n?d(i,{language:n,ignoreIllegals:!0}):p(i) +;M("after:highlightElement",{el:e,result:a,text:i +}),e.innerHTML=a.value,((e,t,n)=>{const i=t?s[t]:n +;e.classList.add("hljs"),i&&e.classList.add(i)})(e,n,a.language),e.result={ +language:a.language,re:a.relevance,relavance:a.relevance +},a.second_best&&(e.second_best={language:a.second_best.language, +re:a.second_best.relevance,relavance:a.second_best.relevance})}const v=()=>{ +v.called||(v.called=!0, +z("10.6.0","initHighlighting() is deprecated. Use highlightAll() instead."), +document.querySelectorAll("pre code").forEach(x))};let w=!1;function y(){ +"loading"!==document.readyState?document.querySelectorAll("pre code").forEach(x):w=!0 +}function N(e){return e=(e||"").toLowerCase(),n[e]||n[s[e]]} +function R(e,{languageName:t}){"string"==typeof e&&(e=[e]),e.forEach((e=>{ +s[e.toLowerCase()]=t}))}function k(e){const t=N(e) +;return t&&!t.disableAutodetect}function M(e,t){const n=e;a.forEach((e=>{ +e[n]&&e[n](t)}))} +"undefined"!=typeof window&&window.addEventListener&&window.addEventListener("DOMContentLoaded",(()=>{ +w&&y()}),!1),Object.assign(e,{highlight:d,highlightAuto:p,highlightAll:y, +fixMarkup:e=>{ +return z("10.2.0","fixMarkup will be removed entirely in v11.0"),z("10.2.0","Please see https://github.com/highlightjs/highlight.js/issues/2534"), +t=e, +u.tabReplace||u.useBR?t.replace(l,(e=>"\n"===e?u.useBR?"
":e:u.tabReplace?e.replace(/\t/g,u.tabReplace):e)):t +;var t},highlightElement:x, +highlightBlock:e=>(z("10.7.0","highlightBlock will be removed entirely in v12.0"), +z("10.7.0","Please use highlightElement now."),x(e)),configure:e=>{ +e.useBR&&(z("10.3.0","'useBR' will be removed entirely in v11.0"), +z("10.3.0","Please see https://github.com/highlightjs/highlight.js/issues/2559")), +u=G(u,e)},initHighlighting:v,initHighlightingOnLoad:()=>{ +z("10.6.0","initHighlightingOnLoad() is deprecated. Use highlightAll() instead."), +w=!0},registerLanguage:(t,i)=>{let s=null;try{s=i(e)}catch(e){ +if($("Language definition for '{}' could not be registered.".replace("{}",t)), +!r)throw e;$(e),s=g} +s.name||(s.name=t),n[t]=s,s.rawDefinition=i.bind(null,e),s.aliases&&R(s.aliases,{ +languageName:t})},unregisterLanguage:e=>{delete n[e] +;for(const t of Object.keys(s))s[t]===e&&delete s[t]}, +listLanguages:()=>Object.keys(n),getLanguage:N,registerAliases:R, +requireLanguage:e=>{ +z("10.4.0","requireLanguage will be removed entirely in v11."), +z("10.4.0","Please see https://github.com/highlightjs/highlight.js/pull/2844") +;const t=N(e);if(t)return t +;throw Error("The '{}' language is required, but not loaded.".replace("{}",e))}, +autoDetection:k,inherit:G,addPlugin:e=>{(e=>{ +e["before:highlightBlock"]&&!e["before:highlightElement"]&&(e["before:highlightElement"]=t=>{ +e["before:highlightBlock"](Object.assign({block:t.el},t)) +}),e["after:highlightBlock"]&&!e["after:highlightElement"]&&(e["after:highlightElement"]=t=>{ +e["after:highlightBlock"](Object.assign({block:t.el},t))})})(e),a.push(e)}, +vuePlugin:P(e).VuePlugin}),e.debugMode=()=>{r=!1},e.safeMode=()=>{r=!0 +},e.versionString="10.7.1";for(const e in _)"object"==typeof _[e]&&t(_[e]) +;return Object.assign(e,_),e.addPlugin(m),e.addPlugin(D),e.addPlugin(E),e})({}) +}();"object"==typeof exports&&"undefined"!=typeof module&&(module.exports=hljs);hljs.registerLanguage("bash",(()=>{"use strict";function e(...e){ +return e.map((e=>{return(s=e)?"string"==typeof s?s:s.source:null;var s +})).join("")}return s=>{const n={},t={begin:/\$\{/,end:/\}/,contains:["self",{ +begin:/:-/,contains:[n]}]};Object.assign(n,{className:"variable",variants:[{ +begin:e(/\$[\w\d#@][\w\d_]*/,"(?![\\w\\d])(?![$])")},t]});const a={ +className:"subst",begin:/\$\(/,end:/\)/,contains:[s.BACKSLASH_ESCAPE]},i={ +begin:/<<-?\s*(?=\w+)/,starts:{contains:[s.END_SAME_AS_BEGIN({begin:/(\w+)/, +end:/(\w+)/,className:"string"})]}},c={className:"string",begin:/"/,end:/"/, +contains:[s.BACKSLASH_ESCAPE,n,a]};a.contains.push(c);const o={begin:/\$\(\(/, +end:/\)\)/,contains:[{begin:/\d+#[0-9a-f]+/,className:"number"},s.NUMBER_MODE,n] +},r=s.SHEBANG({binary:"(fish|bash|zsh|sh|csh|ksh|tcsh|dash|scsh)",relevance:10 +}),l={className:"function",begin:/\w[\w\d_]*\s*\(\s*\)\s*\{/,returnBegin:!0, +contains:[s.inherit(s.TITLE_MODE,{begin:/\w[\w\d_]*/})],relevance:0};return{ +name:"Bash",aliases:["sh","zsh"],keywords:{$pattern:/\b[a-z._-]+\b/, +keyword:"if then else elif fi for while in do done case esac function", +literal:"true false", +built_in:"break cd continue eval exec exit export getopts hash pwd readonly return shift test times trap umask unset alias bind builtin caller command declare echo enable help let local logout mapfile printf read readarray source type typeset ulimit unalias set shopt autoload bg bindkey bye cap chdir clone comparguments compcall compctl compdescribe compfiles compgroups compquote comptags comptry compvalues dirs disable disown echotc echoti emulate fc fg float functions getcap getln history integer jobs kill limit log noglob popd print pushd pushln rehash sched setcap setopt stat suspend ttyctl unfunction unhash unlimit unsetopt vared wait whence where which zcompile zformat zftp zle zmodload zparseopts zprof zpty zregexparse zsocket zstyle ztcp" +},contains:[r,s.SHEBANG(),l,o,s.HASH_COMMENT_MODE,i,c,{className:"",begin:/\\"/ +},{className:"string",begin:/'/,end:/'/},n]}}})());hljs.registerLanguage("plaintext",(()=>{"use strict";return t=>({ +name:"Plain text",aliases:["text","txt"],disableAutodetect:!0})})());hljs.registerLanguage("ini",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(...n){ +return n.map((n=>e(n))).join("")}return s=>{const a={className:"number", +relevance:0,variants:[{begin:/([+-]+)?[\d]+_[\d_]+/},{begin:s.NUMBER_RE}] +},i=s.COMMENT();i.variants=[{begin:/;/,end:/$/},{begin:/#/,end:/$/}];const t={ +className:"variable",variants:[{begin:/\$[\w\d"][\w\d_]*/},{begin:/\$\{(.*?)\}/ +}]},r={className:"literal",begin:/\bon|off|true|false|yes|no\b/},l={ +className:"string",contains:[s.BACKSLASH_ESCAPE],variants:[{begin:"'''", +end:"'''",relevance:10},{begin:'"""',end:'"""',relevance:10},{begin:'"',end:'"' +},{begin:"'",end:"'"}]},c={begin:/\[/,end:/\]/,contains:[i,r,t,l,a,"self"], +relevance:0 +},g="("+[/[A-Za-z0-9_-]+/,/"(\\"|[^"])*"/,/'[^']*'/].map((n=>e(n))).join("|")+")" +;return{name:"TOML, also INI",aliases:["toml"],case_insensitive:!0,illegal:/\S/, +contains:[i,{className:"section",begin:/\[+/,end:/\]+/},{ +begin:n(g,"(\\s*\\.\\s*",g,")*",n("(?=",/\s*=\s*[^#\s]/,")")),className:"attr", +starts:{end:/$/,contains:[i,c,r,t,l,a]}}]}}})());hljs.registerLanguage("julia",(()=>{"use strict";return e=>{ +var r="[A-Za-z_\\u00A1-\\uFFFF][A-Za-z_0-9\\u00A1-\\uFFFF]*",t={$pattern:r, +keyword:["baremodule","begin","break","catch","ccall","const","continue","do","else","elseif","end","export","false","finally","for","function","global","if","import","in","isa","let","local","macro","module","quote","return","true","try","using","where","while"], +literal:["ARGS","C_NULL","DEPOT_PATH","ENDIAN_BOM","ENV","Inf","Inf16","Inf32","Inf64","InsertionSort","LOAD_PATH","MergeSort","NaN","NaN16","NaN32","NaN64","PROGRAM_FILE","QuickSort","RoundDown","RoundFromZero","RoundNearest","RoundNearestTiesAway","RoundNearestTiesUp","RoundToZero","RoundUp","VERSION|0","devnull","false","im","missing","nothing","pi","stderr","stdin","stdout","true","undef","\u03c0","\u212f"], +built_in:["AbstractArray","AbstractChannel","AbstractChar","AbstractDict","AbstractDisplay","AbstractFloat","AbstractIrrational","AbstractMatrix","AbstractRange","AbstractSet","AbstractString","AbstractUnitRange","AbstractVecOrMat","AbstractVector","Any","ArgumentError","Array","AssertionError","BigFloat","BigInt","BitArray","BitMatrix","BitSet","BitVector","Bool","BoundsError","CapturedException","CartesianIndex","CartesianIndices","Cchar","Cdouble","Cfloat","Channel","Char","Cint","Cintmax_t","Clong","Clonglong","Cmd","Colon","Complex","ComplexF16","ComplexF32","ComplexF64","CompositeException","Condition","Cptrdiff_t","Cshort","Csize_t","Cssize_t","Cstring","Cuchar","Cuint","Cuintmax_t","Culong","Culonglong","Cushort","Cvoid","Cwchar_t","Cwstring","DataType","DenseArray","DenseMatrix","DenseVecOrMat","DenseVector","Dict","DimensionMismatch","Dims","DivideError","DomainError","EOFError","Enum","ErrorException","Exception","ExponentialBackOff","Expr","Float16","Float32","Float64","Function","GlobalRef","HTML","IO","IOBuffer","IOContext","IOStream","IdDict","IndexCartesian","IndexLinear","IndexStyle","InexactError","InitError","Int","Int128","Int16","Int32","Int64","Int8","Integer","InterruptException","InvalidStateException","Irrational","KeyError","LinRange","LineNumberNode","LinearIndices","LoadError","MIME","Matrix","Method","MethodError","Missing","MissingException","Module","NTuple","NamedTuple","Nothing","Number","OrdinalRange","OutOfMemoryError","OverflowError","Pair","PartialQuickSort","PermutedDimsArray","Pipe","ProcessFailedException","Ptr","QuoteNode","Rational","RawFD","ReadOnlyMemoryError","Real","ReentrantLock","Ref","Regex","RegexMatch","RoundingMode","SegmentationFault","Set","Signed","Some","StackOverflowError","StepRange","StepRangeLen","StridedArray","StridedMatrix","StridedVecOrMat","StridedVector","String","StringIndexError","SubArray","SubString","SubstitutionString","Symbol","SystemError","Task","TaskFailedException","Text","TextDisplay","Timer","Tuple","Type","TypeError","TypeVar","UInt","UInt128","UInt16","UInt32","UInt64","UInt8","UndefInitializer","UndefKeywordError","UndefRefError","UndefVarError","Union","UnionAll","UnitRange","Unsigned","Val","Vararg","VecElement","VecOrMat","Vector","VersionNumber","WeakKeyDict","WeakRef"] +},n={keywords:t,illegal:/<\//},a={className:"subst",begin:/\$\(/,end:/\)/, +keywords:t},i={className:"variable",begin:"\\$"+r},o={className:"string", +contains:[e.BACKSLASH_ESCAPE,a,i],variants:[{begin:/\w*"""/,end:/"""\w*/, +relevance:10},{begin:/\w*"/,end:/"\w*/}]},s={className:"string", +contains:[e.BACKSLASH_ESCAPE,a,i],begin:"`",end:"`"},l={className:"meta", +begin:"@"+r};return n.name="Julia",n.contains=[{className:"number", +begin:/(\b0x[\d_]*(\.[\d_]*)?|0x\.\d[\d_]*)p[-+]?\d+|\b0[box][a-fA-F0-9][a-fA-F0-9_]*|(\b\d[\d_]*(\.[\d_]*)?|\.\d[\d_]*)([eEfF][-+]?\d+)?/, +relevance:0},{className:"string",begin:/'(.|\\[xXuU][a-zA-Z0-9]+)'/},o,s,l,{ +className:"comment",variants:[{begin:"#=",end:"=#",relevance:10},{begin:"#", +end:"$"}]},e.HASH_COMMENT_MODE,{className:"keyword", +begin:"\\b(((abstract|primitive)\\s+)type|(mutable\\s+)?struct)\\b"},{begin:/<:/ +}],a.contains=n.contains,n}})());hljs.registerLanguage("julia-repl",(()=>{"use strict";return a=>({ +name:"Julia REPL",contains:[{className:"meta",begin:/^julia>/,relevance:10,starts:{end:/^(?![ ]{6})/,subLanguage:"julia"},aliases:["jldoctest"]}, +{className:"metas",begin:/^shell>/,relevance:10,starts:{end:/^(?![ ]{6})/,subLanguage:"bash"}}, +{className:"metap",begin:/^\(.*\)\spkg>/,relevance:10,starts:{end:/^(?![ ]{6})/,subLanguage:"julia"}}]})})()); +hljs.registerLanguage("css",(()=>{"use strict" +;const e=["a","abbr","address","article","aside","audio","b","blockquote","body","button","canvas","caption","cite","code","dd","del","details","dfn","div","dl","dt","em","fieldset","figcaption","figure","footer","form","h1","h2","h3","h4","h5","h6","header","hgroup","html","i","iframe","img","input","ins","kbd","label","legend","li","main","mark","menu","nav","object","ol","p","q","quote","samp","section","span","strong","summary","sup","table","tbody","td","textarea","tfoot","th","thead","time","tr","ul","var","video"],t=["any-hover","any-pointer","aspect-ratio","color","color-gamut","color-index","device-aspect-ratio","device-height","device-width","display-mode","forced-colors","grid","height","hover","inverted-colors","monochrome","orientation","overflow-block","overflow-inline","pointer","prefers-color-scheme","prefers-contrast","prefers-reduced-motion","prefers-reduced-transparency","resolution","scan","scripting","update","width","min-width","max-width","min-height","max-height"],i=["active","any-link","blank","checked","current","default","defined","dir","disabled","drop","empty","enabled","first","first-child","first-of-type","fullscreen","future","focus","focus-visible","focus-within","has","host","host-context","hover","indeterminate","in-range","invalid","is","lang","last-child","last-of-type","left","link","local-link","not","nth-child","nth-col","nth-last-child","nth-last-col","nth-last-of-type","nth-of-type","only-child","only-of-type","optional","out-of-range","past","placeholder-shown","read-only","read-write","required","right","root","scope","target","target-within","user-invalid","valid","visited","where"],o=["after","backdrop","before","cue","cue-region","first-letter","first-line","grammar-error","marker","part","placeholder","selection","slotted","spelling-error"],r=["align-content","align-items","align-self","animation","animation-delay","animation-direction","animation-duration","animation-fill-mode","animation-iteration-count","animation-name","animation-play-state","animation-timing-function","auto","backface-visibility","background","background-attachment","background-clip","background-color","background-image","background-origin","background-position","background-repeat","background-size","border","border-bottom","border-bottom-color","border-bottom-left-radius","border-bottom-right-radius","border-bottom-style","border-bottom-width","border-collapse","border-color","border-image","border-image-outset","border-image-repeat","border-image-slice","border-image-source","border-image-width","border-left","border-left-color","border-left-style","border-left-width","border-radius","border-right","border-right-color","border-right-style","border-right-width","border-spacing","border-style","border-top","border-top-color","border-top-left-radius","border-top-right-radius","border-top-style","border-top-width","border-width","bottom","box-decoration-break","box-shadow","box-sizing","break-after","break-before","break-inside","caption-side","clear","clip","clip-path","color","column-count","column-fill","column-gap","column-rule","column-rule-color","column-rule-style","column-rule-width","column-span","column-width","columns","content","counter-increment","counter-reset","cursor","direction","display","empty-cells","filter","flex","flex-basis","flex-direction","flex-flow","flex-grow","flex-shrink","flex-wrap","float","font","font-display","font-family","font-feature-settings","font-kerning","font-language-override","font-size","font-size-adjust","font-smoothing","font-stretch","font-style","font-variant","font-variant-ligatures","font-variation-settings","font-weight","height","hyphens","icon","image-orientation","image-rendering","image-resolution","ime-mode","inherit","initial","justify-content","left","letter-spacing","line-height","list-style","list-style-image","list-style-position","list-style-type","margin","margin-bottom","margin-left","margin-right","margin-top","marks","mask","max-height","max-width","min-height","min-width","nav-down","nav-index","nav-left","nav-right","nav-up","none","normal","object-fit","object-position","opacity","order","orphans","outline","outline-color","outline-offset","outline-style","outline-width","overflow","overflow-wrap","overflow-x","overflow-y","padding","padding-bottom","padding-left","padding-right","padding-top","page-break-after","page-break-before","page-break-inside","perspective","perspective-origin","pointer-events","position","quotes","resize","right","src","tab-size","table-layout","text-align","text-align-last","text-decoration","text-decoration-color","text-decoration-line","text-decoration-style","text-indent","text-overflow","text-rendering","text-shadow","text-transform","text-underline-position","top","transform","transform-origin","transform-style","transition","transition-delay","transition-duration","transition-property","transition-timing-function","unicode-bidi","vertical-align","visibility","white-space","widows","width","word-break","word-spacing","word-wrap","z-index"].reverse() +;return n=>{const a=(e=>({IMPORTANT:{className:"meta",begin:"!important"}, +HEXCOLOR:{className:"number",begin:"#([a-fA-F0-9]{6}|[a-fA-F0-9]{3})"}, +ATTRIBUTE_SELECTOR_MODE:{className:"selector-attr",begin:/\[/,end:/\]/, +illegal:"$",contains:[e.APOS_STRING_MODE,e.QUOTE_STRING_MODE]} +}))(n),l=[n.APOS_STRING_MODE,n.QUOTE_STRING_MODE];return{name:"CSS", +case_insensitive:!0,illegal:/[=|'\$]/,keywords:{keyframePosition:"from to"}, +classNameAliases:{keyframePosition:"selector-tag"}, +contains:[n.C_BLOCK_COMMENT_MODE,{begin:/-(webkit|moz|ms|o)-(?=[a-z])/ +},n.CSS_NUMBER_MODE,{className:"selector-id",begin:/#[A-Za-z0-9_-]+/,relevance:0 +},{className:"selector-class",begin:"\\.[a-zA-Z-][a-zA-Z0-9_-]*",relevance:0 +},a.ATTRIBUTE_SELECTOR_MODE,{className:"selector-pseudo",variants:[{ +begin:":("+i.join("|")+")"},{begin:"::("+o.join("|")+")"}]},{ +className:"attribute",begin:"\\b("+r.join("|")+")\\b"},{begin:":",end:"[;}]", +contains:[a.HEXCOLOR,a.IMPORTANT,n.CSS_NUMBER_MODE,...l,{ +begin:/(url|data-uri)\(/,end:/\)/,relevance:0,keywords:{built_in:"url data-uri" +},contains:[{className:"string",begin:/[^)]/,endsWithParent:!0,excludeEnd:!0}] +},{className:"built_in",begin:/[\w-]+(?=\()/}]},{ +begin:(s=/@/,((...e)=>e.map((e=>(e=>e?"string"==typeof e?e:e.source:null)(e))).join(""))("(?=",s,")")), +end:"[{;]",relevance:0,illegal:/:/,contains:[{className:"keyword", +begin:/@-?\w[\w]*(-\w+)*/},{begin:/\s/,endsWithParent:!0,excludeEnd:!0, +relevance:0,keywords:{$pattern:/[a-z-]+/,keyword:"and or not only", +attribute:t.join(" ")},contains:[{begin:/[a-z-]+(?=:)/,className:"attribute" +},...l,n.CSS_NUMBER_MODE]}]},{className:"selector-tag", +begin:"\\b("+e.join("|")+")\\b"}]};var s}})());hljs.registerLanguage("cpp",(()=>{"use strict";function e(e){ +return t("(",e,")?")}function t(...e){return e.map((e=>{ +return(t=e)?"string"==typeof t?t:t.source:null;var t})).join("")}return n=>{ +const r=n.COMMENT("//","$",{contains:[{begin:/\\\n/}] +}),a="[a-zA-Z_]\\w*::",i="(decltype\\(auto\\)|"+e(a)+"[a-zA-Z_]\\w*"+e("<[^<>]+>")+")",s={ +className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},c={className:"string", +variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n", +contains:[n.BACKSLASH_ESCAPE]},{ +begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)", +end:"'",illegal:"."},n.END_SAME_AS_BEGIN({ +begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},o={ +className:"number",variants:[{begin:"\\b(0b[01']+)"},{ +begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)" +},{ +begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)" +}],relevance:0},l={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{ +"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include" +},contains:[{begin:/\\\n/,relevance:0},n.inherit(c,{className:"meta-string"}),{ +className:"meta-string",begin:/<.*?>/},r,n.C_BLOCK_COMMENT_MODE]},d={ +className:"title",begin:e(a)+n.IDENT_RE,relevance:0 +},u=e(a)+n.IDENT_RE+"\\s*\\(",m={ +keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq", +built_in:"_Bool _Complex _Imaginary", +_relevance_hints:["asin","atan2","atan","calloc","ceil","cosh","cos","exit","exp","fabs","floor","fmod","fprintf","fputs","free","frexp","auto_ptr","deque","list","queue","stack","vector","map","set","pair","bitset","multiset","multimap","unordered_set","fscanf","future","isalnum","isalpha","iscntrl","isdigit","isgraph","islower","isprint","ispunct","isspace","isupper","isxdigit","tolower","toupper","labs","ldexp","log10","log","malloc","realloc","memchr","memcmp","memcpy","memset","modf","pow","printf","putchar","puts","scanf","sinh","sin","snprintf","sprintf","sqrt","sscanf","strcat","strchr","strcmp","strcpy","strcspn","strlen","strncat","strncmp","strncpy","strpbrk","strrchr","strspn","strstr","tanh","tan","unordered_map","unordered_multiset","unordered_multimap","priority_queue","make_pair","array","shared_ptr","abort","terminate","abs","acos","vfprintf","vprintf","vsprintf","endl","initializer_list","unique_ptr","complex","imaginary","std","string","wstring","cin","cout","cerr","clog","stdin","stdout","stderr","stringstream","istringstream","ostringstream"], +literal:"true false nullptr NULL"},p={className:"function.dispatch",relevance:0, +keywords:m, +begin:t(/\b/,/(?!decltype)/,/(?!if)/,/(?!for)/,/(?!while)/,n.IDENT_RE,(_=/\s*\(/, +t("(?=",_,")")))};var _;const g=[p,l,s,r,n.C_BLOCK_COMMENT_MODE,o,c],b={ +variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{ +beginKeywords:"new throw return else",end:/;/}],keywords:m,contains:g.concat([{ +begin:/\(/,end:/\)/,keywords:m,contains:g.concat(["self"]),relevance:0}]), +relevance:0},f={className:"function",begin:"("+i+"[\\*&\\s]+)+"+u, +returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:m,illegal:/[^\w\s\*&:<>.]/, +contains:[{begin:"decltype\\(auto\\)",keywords:m,relevance:0},{begin:u, +returnBegin:!0,contains:[d],relevance:0},{begin:/::/,relevance:0},{begin:/:/, +endsWithParent:!0,contains:[c,o]},{className:"params",begin:/\(/,end:/\)/, +keywords:m,relevance:0,contains:[r,n.C_BLOCK_COMMENT_MODE,c,o,s,{begin:/\(/, +end:/\)/,keywords:m,relevance:0,contains:["self",r,n.C_BLOCK_COMMENT_MODE,c,o,s] +}]},s,r,n.C_BLOCK_COMMENT_MODE,l]};return{name:"C++", +aliases:["cc","c++","h++","hpp","hh","hxx","cxx"],keywords:m,illegal:"",keywords:m,contains:["self",s]},{begin:n.IDENT_RE+"::",keywords:m},{ +className:"class",beginKeywords:"enum class struct union",end:/[{;:<>=]/, +contains:[{beginKeywords:"final class struct"},n.TITLE_MODE]}]),exports:{ +preprocessor:l,strings:c,keywords:m}}}})());hljs.registerLanguage("xml",(()=>{"use strict";function e(e){ +return e?"string"==typeof e?e:e.source:null}function n(e){return a("(?=",e,")")} +function a(...n){return n.map((n=>e(n))).join("")}function s(...n){ +return"("+n.map((n=>e(n))).join("|")+")"}return e=>{ +const t=a(/[A-Z_]/,a("(",/[A-Z0-9_.-]*:/,")?"),/[A-Z0-9_.-]*/),i={ +className:"symbol",begin:/&[a-z]+;|&#[0-9]+;|&#x[a-f0-9]+;/},r={begin:/\s/, +contains:[{className:"meta-keyword",begin:/#?[a-z_][a-z1-9_-]+/,illegal:/\n/}] +},c=e.inherit(r,{begin:/\(/,end:/\)/}),l=e.inherit(e.APOS_STRING_MODE,{ +className:"meta-string"}),g=e.inherit(e.QUOTE_STRING_MODE,{ +className:"meta-string"}),m={endsWithParent:!0,illegal:/`]+/}]}] +}]};return{name:"HTML, XML", +aliases:["html","xhtml","rss","atom","xjb","xsd","xsl","plist","wsf","svg"], +case_insensitive:!0,contains:[{className:"meta",begin://, +relevance:10,contains:[r,g,l,c,{begin:/\[/,end:/\]/,contains:[{className:"meta", +begin://,contains:[r,c,g,l]}]}]},e.COMMENT(//,{ +relevance:10}),{begin://,relevance:10},i,{ +className:"meta",begin:/<\?xml/,end:/\?>/,relevance:10},{className:"tag", +begin:/)/,end:/>/,keywords:{name:"style"},contains:[m],starts:{ +end:/<\/style>/,returnEnd:!0,subLanguage:["css","xml"]}},{className:"tag", +begin:/)/,end:/>/,keywords:{name:"script"},contains:[m],starts:{ +end:/<\/script>/,returnEnd:!0,subLanguage:["javascript","handlebars","xml"]}},{ +className:"tag",begin:/<>|<\/>/},{className:"tag", +begin:a(//,/>/,/\s/)))),end:/\/?>/,contains:[{className:"name", +begin:t,relevance:0,starts:m}]},{className:"tag",begin:a(/<\//,n(a(t,/>/))), +contains:[{className:"name",begin:t,relevance:0},{begin:/>/,relevance:0, +endsParent:!0}]}]}}})());hljs.registerLanguage("r",(()=>{"use strict";function e(...e){return e.map((e=>{ +return(a=e)?"string"==typeof a?a:a.source:null;var a})).join("")}return a=>{ +const n=/(?:(?:[a-zA-Z]|\.[._a-zA-Z])[._a-zA-Z0-9]*)|\.(?!\d)/;return{name:"R", +illegal:/->/,keywords:{$pattern:n, +keyword:"function if in break next repeat else for while", +literal:"NULL NA TRUE FALSE Inf NaN NA_integer_|10 NA_real_|10 NA_character_|10 NA_complex_|10", +built_in:"LETTERS letters month.abb month.name pi T F abs acos acosh all any anyNA Arg as.call as.character as.complex as.double as.environment as.integer as.logical as.null.default as.numeric as.raw asin asinh atan atanh attr attributes baseenv browser c call ceiling class Conj cos cosh cospi cummax cummin cumprod cumsum digamma dim dimnames emptyenv exp expression floor forceAndCall gamma gc.time globalenv Im interactive invisible is.array is.atomic is.call is.character is.complex is.double is.environment is.expression is.finite is.function is.infinite is.integer is.language is.list is.logical is.matrix is.na is.name is.nan is.null is.numeric is.object is.pairlist is.raw is.recursive is.single is.symbol lazyLoadDBfetch length lgamma list log max min missing Mod names nargs nzchar oldClass on.exit pos.to.env proc.time prod quote range Re rep retracemem return round seq_along seq_len seq.int sign signif sin sinh sinpi sqrt standardGeneric substitute sum switch tan tanh tanpi tracemem trigamma trunc unclass untracemem UseMethod xtfrm" +},compilerExtensions:[(a,n)=>{if(!a.beforeMatch)return +;if(a.starts)throw Error("beforeMatch cannot be used with starts") +;const i=Object.assign({},a);Object.keys(a).forEach((e=>{delete a[e] +})),a.begin=e(i.beforeMatch,e("(?=",i.begin,")")),a.starts={relevance:0, +contains:[Object.assign(i,{endsParent:!0})]},a.relevance=0,delete i.beforeMatch +}],contains:[a.COMMENT(/#'/,/$/,{contains:[{className:"doctag", +begin:"@examples",starts:{contains:[{begin:/\n/},{begin:/#'\s*(?=@[a-zA-Z]+)/, +endsParent:!0},{begin:/#'/,end:/$/,excludeBegin:!0}]}},{className:"doctag", +begin:"@param",end:/$/,contains:[{className:"variable",variants:[{begin:n},{ +begin:/`(?:\\.|[^`\\])+`/}],endsParent:!0}]},{className:"doctag", +begin:/@[a-zA-Z]+/},{className:"meta-keyword",begin:/\\[a-zA-Z]+/}] +}),a.HASH_COMMENT_MODE,{className:"string",contains:[a.BACKSLASH_ESCAPE], +variants:[a.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\(/,end:/\)(-*)"/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\{/,end:/\}(-*)"/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]"(-*)\[/,end:/\](-*)"/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\(/,end:/\)(-*)'/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\{/,end:/\}(-*)'/ +}),a.END_SAME_AS_BEGIN({begin:/[rR]'(-*)\[/,end:/\](-*)'/}),{begin:'"',end:'"', +relevance:0},{begin:"'",end:"'",relevance:0}]},{className:"number",relevance:0, +beforeMatch:/([^a-zA-Z0-9._])/,variants:[{ +match:/0[xX][0-9a-fA-F]+\.[0-9a-fA-F]*[pP][+-]?\d+i?/},{ +match:/0[xX][0-9a-fA-F]+([pP][+-]?\d+)?[Li]?/},{ +match:/(\d+(\.\d*)?|\.\d+)([eE][+-]?\d+)?[Li]?/}]},{begin:"%",end:"%"},{ +begin:e(/[a-zA-Z][a-zA-Z_0-9]*/,"\\s+<-\\s+")},{begin:"`",end:"`",contains:[{ +begin:/\\./}]}]}}})());hljs.registerLanguage("ruby",(()=>{"use strict";function e(...e){ +return e.map((e=>{return(n=e)?"string"==typeof n?n:n.source:null;var n +})).join("")}return n=>{ +const a="([a-zA-Z_]\\w*[!?=]?|[-+~]@|<<|>>|=~|===?|<=>|[<>]=?|\\*\\*|[-/+%^&*~`|]|\\[\\]=?)",i={ +keyword:"and then defined module in return redo if BEGIN retry end for self when next until do begin unless END rescue else break undef not super class case require yield alias while ensure elsif or include attr_reader attr_writer attr_accessor __FILE__", +built_in:"proc lambda",literal:"true false nil"},s={className:"doctag", +begin:"@[A-Za-z]+"},r={begin:"#<",end:">"},b=[n.COMMENT("#","$",{contains:[s] +}),n.COMMENT("^=begin","^=end",{contains:[s],relevance:10 +}),n.COMMENT("^__END__","\\n$")],c={className:"subst",begin:/#\{/,end:/\}/, +keywords:i},t={className:"string",contains:[n.BACKSLASH_ESCAPE,c],variants:[{ +begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/`/,end:/`/},{begin:/%[qQwWx]?\(/, +end:/\)/},{begin:/%[qQwWx]?\[/,end:/\]/},{begin:/%[qQwWx]?\{/,end:/\}/},{ +begin:/%[qQwWx]?/},{begin:/%[qQwWx]?\//,end:/\//},{begin:/%[qQwWx]?%/, +end:/%/},{begin:/%[qQwWx]?-/,end:/-/},{begin:/%[qQwWx]?\|/,end:/\|/},{ +begin:/\B\?(\\\d{1,3})/},{begin:/\B\?(\\x[A-Fa-f0-9]{1,2})/},{ +begin:/\B\?(\\u\{?[A-Fa-f0-9]{1,6}\}?)/},{ +begin:/\B\?(\\M-\\C-|\\M-\\c|\\c\\M-|\\M-|\\C-\\M-)[\x20-\x7e]/},{ +begin:/\B\?\\(c|C-)[\x20-\x7e]/},{begin:/\B\?\\?\S/},{ +begin:/<<[-~]?'?(\w+)\n(?:[^\n]*\n)*?\s*\1\b/,returnBegin:!0,contains:[{ +begin:/<<[-~]?'?/},n.END_SAME_AS_BEGIN({begin:/(\w+)/,end:/(\w+)/, +contains:[n.BACKSLASH_ESCAPE,c]})]}]},g="[0-9](_?[0-9])*",d={className:"number", +relevance:0,variants:[{ +begin:`\\b([1-9](_?[0-9])*|0)(\\.(${g}))?([eE][+-]?(${g})|r)?i?\\b`},{ +begin:"\\b0[dD][0-9](_?[0-9])*r?i?\\b"},{begin:"\\b0[bB][0-1](_?[0-1])*r?i?\\b" +},{begin:"\\b0[oO][0-7](_?[0-7])*r?i?\\b"},{ +begin:"\\b0[xX][0-9a-fA-F](_?[0-9a-fA-F])*r?i?\\b"},{ +begin:"\\b0(_?[0-7])+r?i?\\b"}]},l={className:"params",begin:"\\(",end:"\\)", +endsParent:!0,keywords:i},o=[t,{className:"class",beginKeywords:"class module", +end:"$|;",illegal:/=/,contains:[n.inherit(n.TITLE_MODE,{ +begin:"[A-Za-z_]\\w*(::\\w+)*(\\?|!)?"}),{begin:"<\\s*",contains:[{ +begin:"("+n.IDENT_RE+"::)?"+n.IDENT_RE,relevance:0}]}].concat(b)},{ +className:"function",begin:e(/def\s+/,(_=a+"\\s*(\\(|;|$)",e("(?=",_,")"))), +relevance:0,keywords:"def",end:"$|;",contains:[n.inherit(n.TITLE_MODE,{begin:a +}),l].concat(b)},{begin:n.IDENT_RE+"::"},{className:"symbol", +begin:n.UNDERSCORE_IDENT_RE+"(!|\\?)?:",relevance:0},{className:"symbol", +begin:":(?!\\s)",contains:[t,{begin:a}],relevance:0},d,{className:"variable", +begin:"(\\$\\W)|((\\$|@@?)(\\w+))(?=[^@$?])(?![A-Za-z])(?![@$?'])"},{ +className:"params",begin:/\|/,end:/\|/,relevance:0,keywords:i},{ +begin:"("+n.RE_STARTERS_RE+"|unless)\\s*",keywords:"unless",contains:[{ +className:"regexp",contains:[n.BACKSLASH_ESCAPE,c],illegal:/\n/,variants:[{ +begin:"/",end:"/[a-z]*"},{begin:/%r\{/,end:/\}[a-z]*/},{begin:"%r\\(", +end:"\\)[a-z]*"},{begin:"%r!",end:"![a-z]*"},{begin:"%r\\[",end:"\\][a-z]*"}] +}].concat(r,b),relevance:0}].concat(r,b);var _;c.contains=o,l.contains=o +;const E=[{begin:/^\s*=>/,starts:{end:"$",contains:o}},{className:"meta", +begin:"^([>?]>|[\\w#]+\\(\\w+\\):\\d+:\\d+>|(\\w+-)?\\d+\\.\\d+\\.\\d+(p\\d+)?[^\\d][^>]+>)(?=[ ])", +starts:{end:"$",contains:o}}];return b.unshift(r),{name:"Ruby", +aliases:["rb","gemspec","podspec","thor","irb"],keywords:i,illegal:/\/\*/, +contains:[n.SHEBANG({binary:"ruby"})].concat(E).concat(b).concat(o)}}})());hljs.registerLanguage("yaml",(()=>{"use strict";return e=>{ +var n="true false yes no null",a="[\\w#;/?:@&=+$,.~*'()[\\]]+",s={ +className:"string",relevance:0,variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/ +},{begin:/\S+/}],contains:[e.BACKSLASH_ESCAPE,{className:"template-variable", +variants:[{begin:/\{\{/,end:/\}\}/},{begin:/%\{/,end:/\}/}]}]},i=e.inherit(s,{ +variants:[{begin:/'/,end:/'/},{begin:/"/,end:/"/},{begin:/[^\s,{}[\]]+/}]}),l={ +end:",",endsWithParent:!0,excludeEnd:!0,keywords:n,relevance:0},t={begin:/\{/, +end:/\}/,contains:[l],illegal:"\\n",relevance:0},g={begin:"\\[",end:"\\]", +contains:[l],illegal:"\\n",relevance:0},b=[{className:"attr",variants:[{ +begin:"\\w[\\w :\\/.-]*:(?=[ \t]|$)"},{begin:'"\\w[\\w :\\/.-]*":(?=[ \t]|$)'},{ +begin:"'\\w[\\w :\\/.-]*':(?=[ \t]|$)"}]},{className:"meta",begin:"^---\\s*$", +relevance:10},{className:"string", +begin:"[\\|>]([1-9]?[+-])?[ ]*\\n( +)[^ ][^\\n]*\\n(\\2[^\\n]+\\n?)*"},{ +begin:"<%[%=-]?",end:"[%-]?%>",subLanguage:"ruby",excludeBegin:!0,excludeEnd:!0, +relevance:0},{className:"type",begin:"!\\w+!"+a},{className:"type", +begin:"!<"+a+">"},{className:"type",begin:"!"+a},{className:"type",begin:"!!"+a +},{className:"meta",begin:"&"+e.UNDERSCORE_IDENT_RE+"$"},{className:"meta", +begin:"\\*"+e.UNDERSCORE_IDENT_RE+"$"},{className:"bullet",begin:"-(?=[ ]|$)", +relevance:0},e.HASH_COMMENT_MODE,{beginKeywords:n,keywords:{literal:n}},{ +className:"number", +begin:"\\b[0-9]{4}(-[0-9][0-9]){0,2}([Tt \\t][0-9][0-9]?(:[0-9][0-9]){2})?(\\.[0-9]*)?([ \\t])*(Z|[-+][0-9][0-9]?(:[0-9][0-9])?)?\\b" +},{className:"number",begin:e.C_NUMBER_RE+"\\b",relevance:0},t,g,s],r=[...b] +;return r.pop(),r.push(i),l.contains=r,{name:"YAML",case_insensitive:!0, +aliases:["yml"],contains:b}}})());hljs.registerLanguage("python",(()=>{"use strict";return e=>{const n={ +$pattern:/[A-Za-z]\w+|__\w+__/, +keyword:["and","as","assert","async","await","break","class","continue","def","del","elif","else","except","finally","for","from","global","if","import","in","is","lambda","nonlocal|10","not","or","pass","raise","return","try","while","with","yield"], +built_in:["__import__","abs","all","any","ascii","bin","bool","breakpoint","bytearray","bytes","callable","chr","classmethod","compile","complex","delattr","dict","dir","divmod","enumerate","eval","exec","filter","float","format","frozenset","getattr","globals","hasattr","hash","help","hex","id","input","int","isinstance","issubclass","iter","len","list","locals","map","max","memoryview","min","next","object","oct","open","ord","pow","print","property","range","repr","reversed","round","set","setattr","slice","sorted","staticmethod","str","sum","super","tuple","type","vars","zip"], +literal:["__debug__","Ellipsis","False","None","NotImplemented","True"], +type:["Any","Callable","Coroutine","Dict","List","Literal","Generic","Optional","Sequence","Set","Tuple","Type","Union"] +},a={className:"meta",begin:/^(>>>|\.\.\.) /},i={className:"subst",begin:/\{/, +end:/\}/,keywords:n,illegal:/#/},s={begin:/\{\{/,relevance:0},t={ +className:"string",contains:[e.BACKSLASH_ESCAPE],variants:[{ +begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?'''/,end:/'''/, +contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{ +begin:/([uU]|[bB]|[rR]|[bB][rR]|[rR][bB])?"""/,end:/"""/, +contains:[e.BACKSLASH_ESCAPE,a],relevance:10},{ +begin:/([fF][rR]|[rR][fF]|[fF])'''/,end:/'''/, +contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/([fF][rR]|[rR][fF]|[fF])"""/, +end:/"""/,contains:[e.BACKSLASH_ESCAPE,a,s,i]},{begin:/([uU]|[rR])'/,end:/'/, +relevance:10},{begin:/([uU]|[rR])"/,end:/"/,relevance:10},{ +begin:/([bB]|[bB][rR]|[rR][bB])'/,end:/'/},{begin:/([bB]|[bB][rR]|[rR][bB])"/, +end:/"/},{begin:/([fF][rR]|[rR][fF]|[fF])'/,end:/'/, +contains:[e.BACKSLASH_ESCAPE,s,i]},{begin:/([fF][rR]|[rR][fF]|[fF])"/,end:/"/, +contains:[e.BACKSLASH_ESCAPE,s,i]},e.APOS_STRING_MODE,e.QUOTE_STRING_MODE] +},r="[0-9](_?[0-9])*",l=`(\\b(${r}))?\\.(${r})|\\b(${r})\\.`,b={ +className:"number",relevance:0,variants:[{ +begin:`(\\b(${r})|(${l}))[eE][+-]?(${r})[jJ]?\\b`},{begin:`(${l})[jJ]?`},{ +begin:"\\b([1-9](_?[0-9])*|0+(_?0)*)[lLjJ]?\\b"},{ +begin:"\\b0[bB](_?[01])+[lL]?\\b"},{begin:"\\b0[oO](_?[0-7])+[lL]?\\b"},{ +begin:"\\b0[xX](_?[0-9a-fA-F])+[lL]?\\b"},{begin:`\\b(${r})[jJ]\\b`}]},o={ +className:"comment", +begin:(d=/# type:/,((...e)=>e.map((e=>(e=>e?"string"==typeof e?e:e.source:null)(e))).join(""))("(?=",d,")")), +end:/$/,keywords:n,contains:[{begin:/# type:/},{begin:/#/,end:/\b\B/, +endsWithParent:!0}]},c={className:"params",variants:[{className:"", +begin:/\(\s*\)/,skip:!0},{begin:/\(/,end:/\)/,excludeBegin:!0,excludeEnd:!0, +keywords:n,contains:["self",a,b,t,e.HASH_COMMENT_MODE]}]};var d +;return i.contains=[t,b,a],{name:"Python",aliases:["py","gyp","ipython"], +keywords:n,illegal:/(<\/|->|\?)|=>/,contains:[a,b,{begin:/\bself\b/},{ +beginKeywords:"if",relevance:0},t,o,e.HASH_COMMENT_MODE,{variants:[{ +className:"function",beginKeywords:"def"},{className:"class", +beginKeywords:"class"}],end:/:/,illegal:/[${=;\n,]/, +contains:[e.UNDERSCORE_TITLE_MODE,c,{begin:/->/,endsWithParent:!0,keywords:n}] +},{className:"meta",begin:/^[\t ]*@/,end:/(?=#)|$/,contains:[b,c,t]}]}}})());hljs.registerLanguage("c",(()=>{"use strict";function e(e){ +return((...e)=>e.map((e=>(e=>e?"string"==typeof e?e:e.source:null)(e))).join(""))("(",e,")?") +}return t=>{const n=t.COMMENT("//","$",{contains:[{begin:/\\\n/}] +}),r="[a-zA-Z_]\\w*::",a="(decltype\\(auto\\)|"+e(r)+"[a-zA-Z_]\\w*"+e("<[^<>]+>")+")",i={ +className:"keyword",begin:"\\b[a-z\\d_]*_t\\b"},s={className:"string", +variants:[{begin:'(u8?|U|L)?"',end:'"',illegal:"\\n", +contains:[t.BACKSLASH_ESCAPE]},{ +begin:"(u8?|U|L)?'(\\\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4,8}|[0-7]{3}|\\S)|.)", +end:"'",illegal:"."},t.END_SAME_AS_BEGIN({ +begin:/(?:u8?|U|L)?R"([^()\\ ]{0,16})\(/,end:/\)([^()\\ ]{0,16})"/})]},o={ +className:"number",variants:[{begin:"\\b(0b[01']+)"},{ +begin:"(-?)\\b([\\d']+(\\.[\\d']*)?|\\.[\\d']+)((ll|LL|l|L)(u|U)?|(u|U)(ll|LL|l|L)?|f|F|b|B)" +},{ +begin:"(-?)(\\b0[xX][a-fA-F0-9']+|(\\b[\\d']+(\\.[\\d']*)?|\\.[\\d']+)([eE][-+]?[\\d']+)?)" +}],relevance:0},c={className:"meta",begin:/#\s*[a-z]+\b/,end:/$/,keywords:{ +"meta-keyword":"if else elif endif define undef warning error line pragma _Pragma ifdef ifndef include" +},contains:[{begin:/\\\n/,relevance:0},t.inherit(s,{className:"meta-string"}),{ +className:"meta-string",begin:/<.*?>/},n,t.C_BLOCK_COMMENT_MODE]},l={ +className:"title",begin:e(r)+t.IDENT_RE,relevance:0 +},d=e(r)+t.IDENT_RE+"\\s*\\(",u={ +keyword:"int float while private char char8_t char16_t char32_t catch import module export virtual operator sizeof dynamic_cast|10 typedef const_cast|10 const for static_cast|10 union namespace unsigned long volatile static protected bool template mutable if public friend do goto auto void enum else break extern using asm case typeid wchar_t short reinterpret_cast|10 default double register explicit signed typename try this switch continue inline delete alignas alignof constexpr consteval constinit decltype concept co_await co_return co_yield requires noexcept static_assert thread_local restrict final override atomic_bool atomic_char atomic_schar atomic_uchar atomic_short atomic_ushort atomic_int atomic_uint atomic_long atomic_ulong atomic_llong atomic_ullong new throw return and and_eq bitand bitor compl not not_eq or or_eq xor xor_eq", +built_in:"std string wstring cin cout cerr clog stdin stdout stderr stringstream istringstream ostringstream auto_ptr deque list queue stack vector map set pair bitset multiset multimap unordered_set unordered_map unordered_multiset unordered_multimap priority_queue make_pair array shared_ptr abort terminate abs acos asin atan2 atan calloc ceil cosh cos exit exp fabs floor fmod fprintf fputs free frexp fscanf future isalnum isalpha iscntrl isdigit isgraph islower isprint ispunct isspace isupper isxdigit tolower toupper labs ldexp log10 log malloc realloc memchr memcmp memcpy memset modf pow printf putchar puts scanf sinh sin snprintf sprintf sqrt sscanf strcat strchr strcmp strcpy strcspn strlen strncat strncmp strncpy strpbrk strrchr strspn strstr tanh tan vfprintf vprintf vsprintf endl initializer_list unique_ptr _Bool complex _Complex imaginary _Imaginary", +literal:"true false nullptr NULL"},m=[c,i,n,t.C_BLOCK_COMMENT_MODE,o,s],p={ +variants:[{begin:/=/,end:/;/},{begin:/\(/,end:/\)/},{ +beginKeywords:"new throw return else",end:/;/}],keywords:u,contains:m.concat([{ +begin:/\(/,end:/\)/,keywords:u,contains:m.concat(["self"]),relevance:0}]), +relevance:0},_={className:"function",begin:"("+a+"[\\*&\\s]+)+"+d, +returnBegin:!0,end:/[{;=]/,excludeEnd:!0,keywords:u,illegal:/[^\w\s\*&:<>.]/, +contains:[{begin:"decltype\\(auto\\)",keywords:u,relevance:0},{begin:d, +returnBegin:!0,contains:[l],relevance:0},{className:"params",begin:/\(/, +end:/\)/,keywords:u,relevance:0,contains:[n,t.C_BLOCK_COMMENT_MODE,s,o,i,{ +begin:/\(/,end:/\)/,keywords:u,relevance:0, +contains:["self",n,t.C_BLOCK_COMMENT_MODE,s,o,i]}] +},i,n,t.C_BLOCK_COMMENT_MODE,c]};return{name:"C",aliases:["h"],keywords:u, +disableAutodetect:!0,illegal:"",keywords:u,contains:["self",i]},{begin:t.IDENT_RE+"::",keywords:u},{ +className:"class",beginKeywords:"enum class struct union",end:/[{;:<>=]/, +contains:[{beginKeywords:"final class struct"},t.TITLE_MODE]}]),exports:{ +preprocessor:c,strings:s,keywords:u}}}})());hljs.registerLanguage("markdown",(()=>{"use strict";function n(...n){ +return n.map((n=>{return(e=n)?"string"==typeof e?e:e.source:null;var e +})).join("")}return e=>{const a={begin:/<\/?[A-Za-z_]/,end:">", +subLanguage:"xml",relevance:0},i={variants:[{begin:/\[.+?\]\[.*?\]/,relevance:0 +},{begin:/\[.+?\]\(((data|javascript|mailto):|(?:http|ftp)s?:\/\/).*?\)/, +relevance:2},{begin:n(/\[.+?\]\(/,/[A-Za-z][A-Za-z0-9+.-]*/,/:\/\/.*?\)/), +relevance:2},{begin:/\[.+?\]\([./?&#].*?\)/,relevance:1},{ +begin:/\[.+?\]\(.*?\)/,relevance:0}],returnBegin:!0,contains:[{ +className:"string",relevance:0,begin:"\\[",end:"\\]",excludeBegin:!0, +returnEnd:!0},{className:"link",relevance:0,begin:"\\]\\(",end:"\\)", +excludeBegin:!0,excludeEnd:!0},{className:"symbol",relevance:0,begin:"\\]\\[", +end:"\\]",excludeBegin:!0,excludeEnd:!0}]},s={className:"strong",contains:[], +variants:[{begin:/_{2}/,end:/_{2}/},{begin:/\*{2}/,end:/\*{2}/}]},c={ +className:"emphasis",contains:[],variants:[{begin:/\*(?!\*)/,end:/\*/},{ +begin:/_(?!_)/,end:/_/,relevance:0}]};s.contains.push(c),c.contains.push(s) +;let t=[a,i] +;return s.contains=s.contains.concat(t),c.contains=c.contains.concat(t), +t=t.concat(s,c),{name:"Markdown",aliases:["md","mkdown","mkd"],contains:[{ +className:"section",variants:[{begin:"^#{1,6}",end:"$",contains:t},{ +begin:"(?=^.+?\\n[=-]{2,}$)",contains:[{begin:"^[=-]*$"},{begin:"^",end:"\\n", +contains:t}]}]},a,{className:"bullet",begin:"^[ \t]*([*+-]|(\\d+\\.))(?=\\s+)", +end:"\\s+",excludeEnd:!0},s,c,{className:"quote",begin:"^>\\s+",contains:t, +end:"$"},{className:"code",variants:[{begin:"(`{3,})[^`](.|\\n)*?\\1`*[ ]*"},{ +begin:"(~{3,})[^~](.|\\n)*?\\1~*[ ]*"},{begin:"```",end:"```+[ ]*$"},{ +begin:"~~~",end:"~~~+[ ]*$"},{begin:"`.+?`"},{begin:"(?=^( {4}|\\t))", +contains:[{begin:"^( {4}|\\t)",end:"(\\n)$"}],relevance:0}]},{ +begin:"^[-\\*]{3,}",end:"$"},i,{begin:/^\[[^\n]+\]:/,returnBegin:!0,contains:[{ +className:"symbol",begin:/\[/,end:/\]/,excludeBegin:!0,excludeEnd:!0},{ +className:"link",begin:/:\s*/,end:/$/,excludeBegin:!0}]}]}}})()); \ No newline at end of file diff --git a/libs/katex/auto-render.min.js b/libs/katex/auto-render.min.js new file mode 100644 index 00000000..1647be8b --- /dev/null +++ b/libs/katex/auto-render.min.js @@ -0,0 +1 @@ +!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t(require("katex")):"function"==typeof define&&define.amd?define(["katex"],t):"object"==typeof exports?exports.renderMathInElement=t(require("katex")):e.renderMathInElement=t(e.katex)}("undefined"!=typeof self?self:this,(function(e){return function(){"use strict";var t={974:function(t){t.exports=e}},r={};function n(e){if(r[e])return r[e].exports;var a=r[e]={exports:{}};return t[e](a,a.exports,n),a.exports}n.n=function(e){var t=e&&e.__esModule?function(){return e.default}:function(){return e};return n.d(t,{a:t}),t},n.d=function(e,t){for(var r in t)n.o(t,r)&&!n.o(e,r)&&Object.defineProperty(e,r,{enumerable:!0,get:t[r]})},n.o=function(e,t){return Object.prototype.hasOwnProperty.call(e,t)};var a={};return function(){n.d(a,{default:function(){return s}});var e=n(974),t=n.n(e),r=function(e,t,r){for(var n=r,a=0,i=e.length;n0&&(a.push({type:"text",data:e.slice(0,n)}),e=e.slice(n));var l=t.findIndex((function(t){return e.startsWith(t.left)}));if(-1===(n=r(t[l].right,e,t[l].left.length)))break;var d=e.slice(0,n+t[l].right.length),s=i.test(d)?d:e.slice(t[l].left.length,n);a.push({type:"math",data:s,rawData:d,display:t[l].display}),e=e.slice(n+t[l].right.length)}return""!==e&&a.push({type:"text",data:e}),a},l=function(e,r){var n=o(e,r.delimiters);if(1===n.length&&"text"===n[0].type)return null;for(var a=document.createDocumentFragment(),i=0;i.newline{display:block}.katex .base{position:relative;white-space:nowrap;width:-webkit-min-content;width:-moz-min-content;width:min-content}.katex .base,.katex .strut{display:inline-block}.katex .textbf{font-weight:700}.katex .textit{font-style:italic}.katex .textrm{font-family:KaTeX_Main}.katex .textsf{font-family:KaTeX_SansSerif}.katex .texttt{font-family:KaTeX_Typewriter}.katex .mathnormal{font-family:KaTeX_Math;font-style:italic}.katex .mathit{font-family:KaTeX_Main;font-style:italic}.katex .mathrm{font-style:normal}.katex .mathbf{font-family:KaTeX_Main;font-weight:700}.katex .boldsymbol{font-family:KaTeX_Math;font-weight:700;font-style:italic}.katex .amsrm,.katex .mathbb,.katex .textbb{font-family:KaTeX_AMS}.katex .mathcal{font-family:KaTeX_Caligraphic}.katex .mathfrak,.katex .textfrak{font-family:KaTeX_Fraktur}.katex .mathtt{font-family:KaTeX_Typewriter}.katex .mathscr,.katex .textscr{font-family:KaTeX_Script}.katex .mathsf,.katex .textsf{font-family:KaTeX_SansSerif}.katex .mathboldsf,.katex .textboldsf{font-family:KaTeX_SansSerif;font-weight:700}.katex .mathitsf,.katex .textitsf{font-family:KaTeX_SansSerif;font-style:italic}.katex .mainrm{font-family:KaTeX_Main;font-style:normal}.katex .vlist-t{display:inline-table;table-layout:fixed;border-collapse:collapse}.katex .vlist-r{display:table-row}.katex .vlist{display:table-cell;vertical-align:bottom;position:relative}.katex .vlist>span{display:block;height:0;position:relative}.katex .vlist>span>span{display:inline-block}.katex .vlist>span>.pstrut{overflow:hidden;width:0}.katex .vlist-t2{margin-right:-2px}.katex .vlist-s{display:table-cell;vertical-align:bottom;font-size:1px;width:2px;min-width:2px}.katex .vbox{display:inline-flex;flex-direction:column;align-items:baseline}.katex .hbox{width:100%}.katex .hbox,.katex .thinbox{display:inline-flex;flex-direction:row}.katex .thinbox{width:0;max-width:0}.katex .msupsub{text-align:left}.katex .mfrac>span>span{text-align:center}.katex .mfrac .frac-line{display:inline-block;width:100%;border-bottom-style:solid}.katex .hdashline,.katex .hline,.katex .mfrac .frac-line,.katex .overline .overline-line,.katex .rule,.katex .underline .underline-line{min-height:1px}.katex .mspace{display:inline-block}.katex .clap,.katex .llap,.katex .rlap{width:0;position:relative}.katex .clap>.inner,.katex .llap>.inner,.katex .rlap>.inner{position:absolute}.katex .clap>.fix,.katex .llap>.fix,.katex .rlap>.fix{display:inline-block}.katex .llap>.inner{right:0}.katex .clap>.inner,.katex .rlap>.inner{left:0}.katex .clap>.inner>span{margin-left:-50%;margin-right:50%}.katex .rule{display:inline-block;border:0 solid;position:relative}.katex .hline,.katex .overline .overline-line,.katex .underline .underline-line{display:inline-block;width:100%;border-bottom-style:solid}.katex .hdashline{display:inline-block;width:100%;border-bottom-style:dashed}.katex .sqrt>.root{margin-left:.27777778em;margin-right:-.55555556em}.katex .fontsize-ensurer.reset-size1.size1,.katex .sizing.reset-size1.size1{font-size:1em}.katex .fontsize-ensurer.reset-size1.size2,.katex .sizing.reset-size1.size2{font-size:1.2em}.katex .fontsize-ensurer.reset-size1.size3,.katex .sizing.reset-size1.size3{font-size:1.4em}.katex .fontsize-ensurer.reset-size1.size4,.katex .sizing.reset-size1.size4{font-size:1.6em}.katex .fontsize-ensurer.reset-size1.size5,.katex .sizing.reset-size1.size5{font-size:1.8em}.katex .fontsize-ensurer.reset-size1.size6,.katex .sizing.reset-size1.size6{font-size:2em}.katex .fontsize-ensurer.reset-size1.size7,.katex .sizing.reset-size1.size7{font-size:2.4em}.katex .fontsize-ensurer.reset-size1.size8,.katex .sizing.reset-size1.size8{font-size:2.88em}.katex .fontsize-ensurer.reset-size1.size9,.katex .sizing.reset-size1.size9{font-size:3.456em}.katex .fontsize-ensurer.reset-size1.size10,.katex .sizing.reset-size1.size10{font-size:4.148em}.katex .fontsize-ensurer.reset-size1.size11,.katex .sizing.reset-size1.size11{font-size:4.976em}.katex .fontsize-ensurer.reset-size2.size1,.katex .sizing.reset-size2.size1{font-size:.83333333em}.katex .fontsize-ensurer.reset-size2.size2,.katex .sizing.reset-size2.size2{font-size:1em}.katex .fontsize-ensurer.reset-size2.size3,.katex .sizing.reset-size2.size3{font-size:1.16666667em}.katex .fontsize-ensurer.reset-size2.size4,.katex .sizing.reset-size2.size4{font-size:1.33333333em}.katex .fontsize-ensurer.reset-size2.size5,.katex .sizing.reset-size2.size5{font-size:1.5em}.katex .fontsize-ensurer.reset-size2.size6,.katex .sizing.reset-size2.size6{font-size:1.66666667em}.katex .fontsize-ensurer.reset-size2.size7,.katex .sizing.reset-size2.size7{font-size:2em}.katex .fontsize-ensurer.reset-size2.size8,.katex .sizing.reset-size2.size8{font-size:2.4em}.katex .fontsize-ensurer.reset-size2.size9,.katex .sizing.reset-size2.size9{font-size:2.88em}.katex .fontsize-ensurer.reset-size2.size10,.katex .sizing.reset-size2.size10{font-size:3.45666667em}.katex .fontsize-ensurer.reset-size2.size11,.katex .sizing.reset-size2.size11{font-size:4.14666667em}.katex .fontsize-ensurer.reset-size3.size1,.katex .sizing.reset-size3.size1{font-size:.71428571em}.katex .fontsize-ensurer.reset-size3.size2,.katex .sizing.reset-size3.size2{font-size:.85714286em}.katex .fontsize-ensurer.reset-size3.size3,.katex .sizing.reset-size3.size3{font-size:1em}.katex .fontsize-ensurer.reset-size3.size4,.katex .sizing.reset-size3.size4{font-size:1.14285714em}.katex .fontsize-ensurer.reset-size3.size5,.katex .sizing.reset-size3.size5{font-size:1.28571429em}.katex .fontsize-ensurer.reset-size3.size6,.katex .sizing.reset-size3.size6{font-size:1.42857143em}.katex .fontsize-ensurer.reset-size3.size7,.katex .sizing.reset-size3.size7{font-size:1.71428571em}.katex .fontsize-ensurer.reset-size3.size8,.katex .sizing.reset-size3.size8{font-size:2.05714286em}.katex .fontsize-ensurer.reset-size3.size9,.katex .sizing.reset-size3.size9{font-size:2.46857143em}.katex .fontsize-ensurer.reset-size3.size10,.katex .sizing.reset-size3.size10{font-size:2.96285714em}.katex .fontsize-ensurer.reset-size3.size11,.katex .sizing.reset-size3.size11{font-size:3.55428571em}.katex .fontsize-ensurer.reset-size4.size1,.katex .sizing.reset-size4.size1{font-size:.625em}.katex .fontsize-ensurer.reset-size4.size2,.katex .sizing.reset-size4.size2{font-size:.75em}.katex .fontsize-ensurer.reset-size4.size3,.katex .sizing.reset-size4.size3{font-size:.875em}.katex .fontsize-ensurer.reset-size4.size4,.katex .sizing.reset-size4.size4{font-size:1em}.katex .fontsize-ensurer.reset-size4.size5,.katex .sizing.reset-size4.size5{font-size:1.125em}.katex .fontsize-ensurer.reset-size4.size6,.katex .sizing.reset-size4.size6{font-size:1.25em}.katex .fontsize-ensurer.reset-size4.size7,.katex .sizing.reset-size4.size7{font-size:1.5em}.katex .fontsize-ensurer.reset-size4.size8,.katex .sizing.reset-size4.size8{font-size:1.8em}.katex .fontsize-ensurer.reset-size4.size9,.katex .sizing.reset-size4.size9{font-size:2.16em}.katex .fontsize-ensurer.reset-size4.size10,.katex .sizing.reset-size4.size10{font-size:2.5925em}.katex .fontsize-ensurer.reset-size4.size11,.katex .sizing.reset-size4.size11{font-size:3.11em}.katex .fontsize-ensurer.reset-size5.size1,.katex .sizing.reset-size5.size1{font-size:.55555556em}.katex .fontsize-ensurer.reset-size5.size2,.katex .sizing.reset-size5.size2{font-size:.66666667em}.katex .fontsize-ensurer.reset-size5.size3,.katex .sizing.reset-size5.size3{font-size:.77777778em}.katex .fontsize-ensurer.reset-size5.size4,.katex .sizing.reset-size5.size4{font-size:.88888889em}.katex .fontsize-ensurer.reset-size5.size5,.katex .sizing.reset-size5.size5{font-size:1em}.katex .fontsize-ensurer.reset-size5.size6,.katex .sizing.reset-size5.size6{font-size:1.11111111em}.katex .fontsize-ensurer.reset-size5.size7,.katex .sizing.reset-size5.size7{font-size:1.33333333em}.katex .fontsize-ensurer.reset-size5.size8,.katex .sizing.reset-size5.size8{font-size:1.6em}.katex .fontsize-ensurer.reset-size5.size9,.katex .sizing.reset-size5.size9{font-size:1.92em}.katex .fontsize-ensurer.reset-size5.size10,.katex .sizing.reset-size5.size10{font-size:2.30444444em}.katex .fontsize-ensurer.reset-size5.size11,.katex .sizing.reset-size5.size11{font-size:2.76444444em}.katex .fontsize-ensurer.reset-size6.size1,.katex .sizing.reset-size6.size1{font-size:.5em}.katex .fontsize-ensurer.reset-size6.size2,.katex .sizing.reset-size6.size2{font-size:.6em}.katex .fontsize-ensurer.reset-size6.size3,.katex .sizing.reset-size6.size3{font-size:.7em}.katex .fontsize-ensurer.reset-size6.size4,.katex .sizing.reset-size6.size4{font-size:.8em}.katex .fontsize-ensurer.reset-size6.size5,.katex .sizing.reset-size6.size5{font-size:.9em}.katex .fontsize-ensurer.reset-size6.size6,.katex .sizing.reset-size6.size6{font-size:1em}.katex .fontsize-ensurer.reset-size6.size7,.katex .sizing.reset-size6.size7{font-size:1.2em}.katex .fontsize-ensurer.reset-size6.size8,.katex .sizing.reset-size6.size8{font-size:1.44em}.katex .fontsize-ensurer.reset-size6.size9,.katex .sizing.reset-size6.size9{font-size:1.728em}.katex .fontsize-ensurer.reset-size6.size10,.katex .sizing.reset-size6.size10{font-size:2.074em}.katex .fontsize-ensurer.reset-size6.size11,.katex .sizing.reset-size6.size11{font-size:2.488em}.katex .fontsize-ensurer.reset-size7.size1,.katex .sizing.reset-size7.size1{font-size:.41666667em}.katex .fontsize-ensurer.reset-size7.size2,.katex .sizing.reset-size7.size2{font-size:.5em}.katex .fontsize-ensurer.reset-size7.size3,.katex .sizing.reset-size7.size3{font-size:.58333333em}.katex .fontsize-ensurer.reset-size7.size4,.katex .sizing.reset-size7.size4{font-size:.66666667em}.katex .fontsize-ensurer.reset-size7.size5,.katex .sizing.reset-size7.size5{font-size:.75em}.katex .fontsize-ensurer.reset-size7.size6,.katex .sizing.reset-size7.size6{font-size:.83333333em}.katex .fontsize-ensurer.reset-size7.size7,.katex .sizing.reset-size7.size7{font-size:1em}.katex .fontsize-ensurer.reset-size7.size8,.katex .sizing.reset-size7.size8{font-size:1.2em}.katex .fontsize-ensurer.reset-size7.size9,.katex .sizing.reset-size7.size9{font-size:1.44em}.katex .fontsize-ensurer.reset-size7.size10,.katex .sizing.reset-size7.size10{font-size:1.72833333em}.katex .fontsize-ensurer.reset-size7.size11,.katex .sizing.reset-size7.size11{font-size:2.07333333em}.katex .fontsize-ensurer.reset-size8.size1,.katex .sizing.reset-size8.size1{font-size:.34722222em}.katex .fontsize-ensurer.reset-size8.size2,.katex .sizing.reset-size8.size2{font-size:.41666667em}.katex .fontsize-ensurer.reset-size8.size3,.katex .sizing.reset-size8.size3{font-size:.48611111em}.katex .fontsize-ensurer.reset-size8.size4,.katex .sizing.reset-size8.size4{font-size:.55555556em}.katex .fontsize-ensurer.reset-size8.size5,.katex .sizing.reset-size8.size5{font-size:.625em}.katex .fontsize-ensurer.reset-size8.size6,.katex .sizing.reset-size8.size6{font-size:.69444444em}.katex .fontsize-ensurer.reset-size8.size7,.katex .sizing.reset-size8.size7{font-size:.83333333em}.katex .fontsize-ensurer.reset-size8.size8,.katex .sizing.reset-size8.size8{font-size:1em}.katex .fontsize-ensurer.reset-size8.size9,.katex .sizing.reset-size8.size9{font-size:1.2em}.katex .fontsize-ensurer.reset-size8.size10,.katex .sizing.reset-size8.size10{font-size:1.44027778em}.katex .fontsize-ensurer.reset-size8.size11,.katex .sizing.reset-size8.size11{font-size:1.72777778em}.katex .fontsize-ensurer.reset-size9.size1,.katex .sizing.reset-size9.size1{font-size:.28935185em}.katex .fontsize-ensurer.reset-size9.size2,.katex .sizing.reset-size9.size2{font-size:.34722222em}.katex .fontsize-ensurer.reset-size9.size3,.katex .sizing.reset-size9.size3{font-size:.40509259em}.katex .fontsize-ensurer.reset-size9.size4,.katex .sizing.reset-size9.size4{font-size:.46296296em}.katex .fontsize-ensurer.reset-size9.size5,.katex .sizing.reset-size9.size5{font-size:.52083333em}.katex .fontsize-ensurer.reset-size9.size6,.katex .sizing.reset-size9.size6{font-size:.5787037em}.katex .fontsize-ensurer.reset-size9.size7,.katex .sizing.reset-size9.size7{font-size:.69444444em}.katex .fontsize-ensurer.reset-size9.size8,.katex .sizing.reset-size9.size8{font-size:.83333333em}.katex .fontsize-ensurer.reset-size9.size9,.katex .sizing.reset-size9.size9{font-size:1em}.katex .fontsize-ensurer.reset-size9.size10,.katex .sizing.reset-size9.size10{font-size:1.20023148em}.katex .fontsize-ensurer.reset-size9.size11,.katex .sizing.reset-size9.size11{font-size:1.43981481em}.katex .fontsize-ensurer.reset-size10.size1,.katex .sizing.reset-size10.size1{font-size:.24108004em}.katex .fontsize-ensurer.reset-size10.size2,.katex .sizing.reset-size10.size2{font-size:.28929605em}.katex .fontsize-ensurer.reset-size10.size3,.katex .sizing.reset-size10.size3{font-size:.33751205em}.katex .fontsize-ensurer.reset-size10.size4,.katex .sizing.reset-size10.size4{font-size:.38572806em}.katex .fontsize-ensurer.reset-size10.size5,.katex .sizing.reset-size10.size5{font-size:.43394407em}.katex .fontsize-ensurer.reset-size10.size6,.katex .sizing.reset-size10.size6{font-size:.48216008em}.katex .fontsize-ensurer.reset-size10.size7,.katex .sizing.reset-size10.size7{font-size:.57859209em}.katex .fontsize-ensurer.reset-size10.size8,.katex .sizing.reset-size10.size8{font-size:.69431051em}.katex .fontsize-ensurer.reset-size10.size9,.katex .sizing.reset-size10.size9{font-size:.83317261em}.katex .fontsize-ensurer.reset-size10.size10,.katex .sizing.reset-size10.size10{font-size:1em}.katex .fontsize-ensurer.reset-size10.size11,.katex .sizing.reset-size10.size11{font-size:1.19961427em}.katex .fontsize-ensurer.reset-size11.size1,.katex .sizing.reset-size11.size1{font-size:.20096463em}.katex .fontsize-ensurer.reset-size11.size2,.katex .sizing.reset-size11.size2{font-size:.24115756em}.katex .fontsize-ensurer.reset-size11.size3,.katex .sizing.reset-size11.size3{font-size:.28135048em}.katex .fontsize-ensurer.reset-size11.size4,.katex .sizing.reset-size11.size4{font-size:.32154341em}.katex .fontsize-ensurer.reset-size11.size5,.katex .sizing.reset-size11.size5{font-size:.36173633em}.katex .fontsize-ensurer.reset-size11.size6,.katex .sizing.reset-size11.size6{font-size:.40192926em}.katex .fontsize-ensurer.reset-size11.size7,.katex .sizing.reset-size11.size7{font-size:.48231511em}.katex .fontsize-ensurer.reset-size11.size8,.katex .sizing.reset-size11.size8{font-size:.57877814em}.katex .fontsize-ensurer.reset-size11.size9,.katex .sizing.reset-size11.size9{font-size:.69453376em}.katex .fontsize-ensurer.reset-size11.size10,.katex .sizing.reset-size11.size10{font-size:.83360129em}.katex .fontsize-ensurer.reset-size11.size11,.katex .sizing.reset-size11.size11{font-size:1em}.katex .delimsizing.size1{font-family:KaTeX_Size1}.katex .delimsizing.size2{font-family:KaTeX_Size2}.katex .delimsizing.size3{font-family:KaTeX_Size3}.katex .delimsizing.size4{font-family:KaTeX_Size4}.katex .delimsizing.mult .delim-size1>span{font-family:KaTeX_Size1}.katex .delimsizing.mult .delim-size4>span{font-family:KaTeX_Size4}.katex .nulldelimiter{display:inline-block;width:.12em}.katex .delimcenter,.katex .op-symbol{position:relative}.katex .op-symbol.small-op{font-family:KaTeX_Size1}.katex .op-symbol.large-op{font-family:KaTeX_Size2}.katex .accent>.vlist-t,.katex .op-limits>.vlist-t{text-align:center}.katex .accent .accent-body{position:relative}.katex .accent .accent-body:not(.accent-full){width:0}.katex .overlay{display:block}.katex .mtable .vertical-separator{display:inline-block;min-width:1px}.katex .mtable .arraycolsep{display:inline-block}.katex .mtable .col-align-c>.vlist-t{text-align:center}.katex .mtable .col-align-l>.vlist-t{text-align:left}.katex .mtable .col-align-r>.vlist-t{text-align:right}.katex .svg-align{text-align:left}.katex svg{display:block;position:absolute;width:100%;height:inherit;fill:currentColor;stroke:currentColor;fill-rule:nonzero;fill-opacity:1;stroke-width:1;stroke-linecap:butt;stroke-linejoin:miter;stroke-miterlimit:4;stroke-dasharray:none;stroke-dashoffset:0;stroke-opacity:1}.katex svg path{stroke:none}.katex img{border-style:none;min-width:0;min-height:0;max-width:none;max-height:none}.katex .stretchy{width:100%;display:block;position:relative;overflow:hidden}.katex .stretchy:after,.katex .stretchy:before{content:""}.katex .hide-tail{width:100%;position:relative;overflow:hidden}.katex .halfarrow-left{position:absolute;left:0;width:50.2%;overflow:hidden}.katex .halfarrow-right{position:absolute;right:0;width:50.2%;overflow:hidden}.katex .brace-left{position:absolute;left:0;width:25.1%;overflow:hidden}.katex .brace-center{position:absolute;left:25%;width:50%;overflow:hidden}.katex .brace-right{position:absolute;right:0;width:25.1%;overflow:hidden}.katex .x-arrow-pad{padding:0 .5em}.katex .cd-arrow-pad{padding:0 .55556em 0 .27778em}.katex .mover,.katex .munder,.katex .x-arrow{text-align:center}.katex .boxpad{padding:0 .3em}.katex .fbox,.katex .fcolorbox{box-sizing:border-box;border:.04em solid}.katex .cancel-pad{padding:0 .2em}.katex .cancel-lap{margin-left:-.2em;margin-right:-.2em}.katex .sout{border-bottom-style:solid;border-bottom-width:.08em}.katex .angl{box-sizing:border-content;border-top:.049em solid;border-right:.049em solid;margin-right:.03889em}.katex .anglpad{padding:0 .03889em}.katex .eqn-num:before{counter-increment:katexEqnNo;content:"(" counter(katexEqnNo) ")"}.katex .mml-eqn-num:before{counter-increment:mmlEqnNo;content:"(" counter(mmlEqnNo) ")"}.katex .mtr-glue{width:50%}.katex .cd-vert-arrow{display:inline-block;position:relative}.katex .cd-label-left{display:inline-block;position:absolute;right:calc(50% + .3em);text-align:left}.katex .cd-label-right{display:inline-block;position:absolute;left:calc(50% + .3em);text-align:right}.katex-display{display:block;margin:1em 0;text-align:center}.katex-display>.katex{display:block;text-align:center;white-space:nowrap}.katex-display>.katex>.katex-html{display:block;position:relative}.katex-display>.katex>.katex-html>.tag{position:absolute;right:0}.katex-display.leqno>.katex>.katex-html>.tag{left:0;right:auto}.katex-display.fleqn>.katex{text-align:left;padding-left:2em}body{counter-reset:katexEqnNo mmlEqnNo} diff --git a/libs/katex/katex.min.js b/libs/katex/katex.min.js new file mode 100644 index 00000000..6b1f525b --- /dev/null +++ b/libs/katex/katex.min.js @@ -0,0 +1 @@ +!function(e,t){"object"==typeof exports&&"object"==typeof module?module.exports=t():"function"==typeof define&&define.amd?define([],t):"object"==typeof exports?exports.katex=t():e.katex=t()}("undefined"!=typeof self?self:this,(function(){return function(){"use strict";var e={d:function(t,r){for(var n in r)e.o(r,n)&&!e.o(t,n)&&Object.defineProperty(t,n,{enumerable:!0,get:r[n]})},o:function(e,t){return Object.prototype.hasOwnProperty.call(e,t)}},t={};e.d(t,{default:function(){return Kn}});var r=function e(t,r){this.position=void 0;var n,a="KaTeX parse error: "+t,i=r&&r.loc;if(i&&i.start<=i.end){var o=i.lexer.input;n=i.start;var s=i.end;n===o.length?a+=" at end of input: ":a+=" at position "+(n+1)+": ";var l=o.slice(n,s).replace(/[^]/g,"$&\u0332");a+=(n>15?"\u2026"+o.slice(n-15,n):o.slice(0,n))+l+(s+15":">","<":"<",'"':""","'":"'"},o=/[&><"']/g;var s=function e(t){return"ordgroup"===t.type||"color"===t.type?1===t.body.length?e(t.body[0]):t:"font"===t.type?e(t.body):t},l={contains:function(e,t){return-1!==e.indexOf(t)},deflt:function(e,t){return void 0===e?t:e},escape:function(e){return String(e).replace(o,(function(e){return i[e]}))},hyphenate:function(e){return e.replace(a,"-$1").toLowerCase()},getBaseElem:s,isCharacterBox:function(e){var t=s(e);return"mathord"===t.type||"textord"===t.type||"atom"===t.type},protocolFromUrl:function(e){var t=/^\s*([^\\/#]*?)(?::|�*58|�*3a)/i.exec(e);return null!=t?t[1]:"_relative"}},h=function(){function e(e){this.displayMode=void 0,this.output=void 0,this.leqno=void 0,this.fleqn=void 0,this.throwOnError=void 0,this.errorColor=void 0,this.macros=void 0,this.minRuleThickness=void 0,this.colorIsTextColor=void 0,this.strict=void 0,this.trust=void 0,this.maxSize=void 0,this.maxExpand=void 0,this.globalGroup=void 0,e=e||{},this.displayMode=l.deflt(e.displayMode,!1),this.output=l.deflt(e.output,"htmlAndMathml"),this.leqno=l.deflt(e.leqno,!1),this.fleqn=l.deflt(e.fleqn,!1),this.throwOnError=l.deflt(e.throwOnError,!0),this.errorColor=l.deflt(e.errorColor,"#cc0000"),this.macros=e.macros||{},this.minRuleThickness=Math.max(0,l.deflt(e.minRuleThickness,0)),this.colorIsTextColor=l.deflt(e.colorIsTextColor,!1),this.strict=l.deflt(e.strict,"warn"),this.trust=l.deflt(e.trust,!1),this.maxSize=Math.max(0,l.deflt(e.maxSize,1/0)),this.maxExpand=Math.max(0,l.deflt(e.maxExpand,1e3)),this.globalGroup=l.deflt(e.globalGroup,!1)}var t=e.prototype;return t.reportNonstrict=function(e,t,r){var a=this.strict;if("function"==typeof a&&(a=a(e,t,r)),a&&"ignore"!==a){if(!0===a||"error"===a)throw new n("LaTeX-incompatible input and strict mode is set to 'error': "+t+" ["+e+"]",r);"warn"===a?"undefined"!=typeof console&&console.warn("LaTeX-incompatible input and strict mode is set to 'warn': "+t+" ["+e+"]"):"undefined"!=typeof console&&console.warn("LaTeX-incompatible input and strict mode is set to unrecognized '"+a+"': "+t+" ["+e+"]")}},t.useStrictBehavior=function(e,t,r){var n=this.strict;if("function"==typeof n)try{n=n(e,t,r)}catch(e){n="error"}return!(!n||"ignore"===n)&&(!0===n||"error"===n||("warn"===n?("undefined"!=typeof console&&console.warn("LaTeX-incompatible input and strict mode is set to 'warn': "+t+" ["+e+"]"),!1):("undefined"!=typeof console&&console.warn("LaTeX-incompatible input and strict mode is set to unrecognized '"+n+"': "+t+" ["+e+"]"),!1)))},t.isTrusted=function(e){e.url&&!e.protocol&&(e.protocol=l.protocolFromUrl(e.url));var t="function"==typeof this.trust?this.trust(e):this.trust;return Boolean(t)},e}(),m=function(){function e(e,t,r){this.id=void 0,this.size=void 0,this.cramped=void 0,this.id=e,this.size=t,this.cramped=r}var t=e.prototype;return t.sup=function(){return c[u[this.id]]},t.sub=function(){return c[p[this.id]]},t.fracNum=function(){return c[d[this.id]]},t.fracDen=function(){return c[f[this.id]]},t.cramp=function(){return c[g[this.id]]},t.text=function(){return c[v[this.id]]},t.isTight=function(){return this.size>=2},e}(),c=[new m(0,0,!1),new m(1,0,!0),new m(2,1,!1),new m(3,1,!0),new m(4,2,!1),new m(5,2,!0),new m(6,3,!1),new m(7,3,!0)],u=[4,5,4,5,6,7,6,7],p=[5,5,5,5,7,7,7,7],d=[2,3,4,5,6,7,6,7],f=[3,3,5,5,7,7,7,7],g=[1,1,3,3,5,5,7,7],v=[0,1,2,3,2,3,2,3],b={DISPLAY:c[0],TEXT:c[2],SCRIPT:c[4],SCRIPTSCRIPT:c[6]},y=[{name:"latin",blocks:[[256,591],[768,879]]},{name:"cyrillic",blocks:[[1024,1279]]},{name:"armenian",blocks:[[1328,1423]]},{name:"brahmic",blocks:[[2304,4255]]},{name:"georgian",blocks:[[4256,4351]]},{name:"cjk",blocks:[[12288,12543],[19968,40879],[65280,65376]]},{name:"hangul",blocks:[[44032,55215]]}];var x=[];function w(e){for(var t=0;t=x[t]&&e<=x[t+1])return!0;return!1}y.forEach((function(e){return e.blocks.forEach((function(e){return x.push.apply(x,e)}))}));var k=80,S={doubleleftarrow:"M262 157\nl10-10c34-36 62.7-77 86-123 3.3-8 5-13.3 5-16 0-5.3-6.7-8-20-8-7.3\n 0-12.2.5-14.5 1.5-2.3 1-4.8 4.5-7.5 10.5-49.3 97.3-121.7 169.3-217 216-28\n 14-57.3 25-88 33-6.7 2-11 3.8-13 5.5-2 1.7-3 4.2-3 7.5s1 5.8 3 7.5\nc2 1.7 6.3 3.5 13 5.5 68 17.3 128.2 47.8 180.5 91.5 52.3 43.7 93.8 96.2 124.5\n 157.5 9.3 8 15.3 12.3 18 13h6c12-.7 18-4 18-10 0-2-1.7-7-5-15-23.3-46-52-87\n-86-123l-10-10h399738v-40H218c328 0 0 0 0 0l-10-8c-26.7-20-65.7-43-117-69 2.7\n-2 6-3.7 10-5 36.7-16 72.3-37.3 107-64l10-8h399782v-40z\nm8 0v40h399730v-40zm0 194v40h399730v-40z",doublerightarrow:"M399738 392l\n-10 10c-34 36-62.7 77-86 123-3.3 8-5 13.3-5 16 0 5.3 6.7 8 20 8 7.3 0 12.2-.5\n 14.5-1.5 2.3-1 4.8-4.5 7.5-10.5 49.3-97.3 121.7-169.3 217-216 28-14 57.3-25 88\n-33 6.7-2 11-3.8 13-5.5 2-1.7 3-4.2 3-7.5s-1-5.8-3-7.5c-2-1.7-6.3-3.5-13-5.5-68\n-17.3-128.2-47.8-180.5-91.5-52.3-43.7-93.8-96.2-124.5-157.5-9.3-8-15.3-12.3-18\n-13h-6c-12 .7-18 4-18 10 0 2 1.7 7 5 15 23.3 46 52 87 86 123l10 10H0v40h399782\nc-328 0 0 0 0 0l10 8c26.7 20 65.7 43 117 69-2.7 2-6 3.7-10 5-36.7 16-72.3 37.3\n-107 64l-10 8H0v40zM0 157v40h399730v-40zm0 194v40h399730v-40z",leftarrow:"M400000 241H110l3-3c68.7-52.7 113.7-120\n 135-202 4-14.7 6-23 6-25 0-7.3-7-11-21-11-8 0-13.2.8-15.5 2.5-2.3 1.7-4.2 5.8\n-5.5 12.5-1.3 4.7-2.7 10.3-4 17-12 48.7-34.8 92-68.5 130S65.3 228.3 18 247\nc-10 4-16 7.7-18 11 0 8.7 6 14.3 18 17 47.3 18.7 87.8 47 121.5 85S196 441.3 208\n 490c.7 2 1.3 5 2 9s1.2 6.7 1.5 8c.3 1.3 1 3.3 2 6s2.2 4.5 3.5 5.5c1.3 1 3.3\n 1.8 6 2.5s6 1 10 1c14 0 21-3.7 21-11 0-2-2-10.3-6-25-20-79.3-65-146.7-135-202\n l-3-3h399890zM100 241v40h399900v-40z",leftbrace:"M6 548l-6-6v-35l6-11c56-104 135.3-181.3 238-232 57.3-28.7 117\n-45 179-50h399577v120H403c-43.3 7-81 15-113 26-100.7 33-179.7 91-237 174-2.7\n 5-6 9-10 13-.7 1-7.3 1-20 1H6z",leftbraceunder:"M0 6l6-6h17c12.688 0 19.313.3 20 1 4 4 7.313 8.3 10 13\n 35.313 51.3 80.813 93.8 136.5 127.5 55.688 33.7 117.188 55.8 184.5 66.5.688\n 0 2 .3 4 1 18.688 2.7 76 4.3 172 5h399450v120H429l-6-1c-124.688-8-235-61.7\n-331-161C60.687 138.7 32.312 99.3 7 54L0 41V6z",leftgroup:"M400000 80\nH435C64 80 168.3 229.4 21 260c-5.9 1.2-18 0-18 0-2 0-3-1-3-3v-38C76 61 257 0\n 435 0h399565z",leftgroupunder:"M400000 262\nH435C64 262 168.3 112.6 21 82c-5.9-1.2-18 0-18 0-2 0-3 1-3 3v38c76 158 257 219\n 435 219h399565z",leftharpoon:"M0 267c.7 5.3 3 10 7 14h399993v-40H93c3.3\n-3.3 10.2-9.5 20.5-18.5s17.8-15.8 22.5-20.5c50.7-52 88-110.3 112-175 4-11.3 5\n-18.3 3-21-1.3-4-7.3-6-18-6-8 0-13 .7-15 2s-4.7 6.7-8 16c-42 98.7-107.3 174.7\n-196 228-6.7 4.7-10.7 8-12 10-1.3 2-2 5.7-2 11zm100-26v40h399900v-40z",leftharpoonplus:"M0 267c.7 5.3 3 10 7 14h399993v-40H93c3.3-3.3 10.2-9.5\n 20.5-18.5s17.8-15.8 22.5-20.5c50.7-52 88-110.3 112-175 4-11.3 5-18.3 3-21-1.3\n-4-7.3-6-18-6-8 0-13 .7-15 2s-4.7 6.7-8 16c-42 98.7-107.3 174.7-196 228-6.7 4.7\n-10.7 8-12 10-1.3 2-2 5.7-2 11zm100-26v40h399900v-40zM0 435v40h400000v-40z\nm0 0v40h400000v-40z",leftharpoondown:"M7 241c-4 4-6.333 8.667-7 14 0 5.333.667 9 2 11s5.333\n 5.333 12 10c90.667 54 156 130 196 228 3.333 10.667 6.333 16.333 9 17 2 .667 5\n 1 9 1h5c10.667 0 16.667-2 18-6 2-2.667 1-9.667-3-21-32-87.333-82.667-157.667\n-152-211l-3-3h399907v-40zM93 281 H400000 v-40L7 241z",leftharpoondownplus:"M7 435c-4 4-6.3 8.7-7 14 0 5.3.7 9 2 11s5.3 5.3 12\n 10c90.7 54 156 130 196 228 3.3 10.7 6.3 16.3 9 17 2 .7 5 1 9 1h5c10.7 0 16.7\n-2 18-6 2-2.7 1-9.7-3-21-32-87.3-82.7-157.7-152-211l-3-3h399907v-40H7zm93 0\nv40h399900v-40zM0 241v40h399900v-40zm0 0v40h399900v-40z",lefthook:"M400000 281 H103s-33-11.2-61-33.5S0 197.3 0 164s14.2-61.2 42.5\n-83.5C70.8 58.2 104 47 142 47 c16.7 0 25 6.7 25 20 0 12-8.7 18.7-26 20-40 3.3\n-68.7 15.7-86 37-10 12-15 25.3-15 40 0 22.7 9.8 40.7 29.5 54 19.7 13.3 43.5 21\n 71.5 23h399859zM103 281v-40h399897v40z",leftlinesegment:"M40 281 V428 H0 V94 H40 V241 H400000 v40z\nM40 281 V428 H0 V94 H40 V241 H400000 v40z",leftmapsto:"M40 281 V448H0V74H40V241H400000v40z\nM40 281 V448H0V74H40V241H400000v40z",leftToFrom:"M0 147h400000v40H0zm0 214c68 40 115.7 95.7 143 167h22c15.3 0 23\n-.3 23-1 0-1.3-5.3-13.7-16-37-18-35.3-41.3-69-70-101l-7-8h399905v-40H95l7-8\nc28.7-32 52-65.7 70-101 10.7-23.3 16-35.7 16-37 0-.7-7.7-1-23-1h-22C115.7 265.3\n 68 321 0 361zm0-174v-40h399900v40zm100 154v40h399900v-40z",longequal:"M0 50 h400000 v40H0z m0 194h40000v40H0z\nM0 50 h400000 v40H0z m0 194h40000v40H0z",midbrace:"M200428 334\nc-100.7-8.3-195.3-44-280-108-55.3-42-101.7-93-139-153l-9-14c-2.7 4-5.7 8.7-9 14\n-53.3 86.7-123.7 153-211 199-66.7 36-137.3 56.3-212 62H0V214h199568c178.3-11.7\n 311.7-78.3 403-201 6-8 9.7-12 11-12 .7-.7 6.7-1 18-1s17.3.3 18 1c1.3 0 5 4 11\n 12 44.7 59.3 101.3 106.3 170 141s145.3 54.3 229 60h199572v120z",midbraceunder:"M199572 214\nc100.7 8.3 195.3 44 280 108 55.3 42 101.7 93 139 153l9 14c2.7-4 5.7-8.7 9-14\n 53.3-86.7 123.7-153 211-199 66.7-36 137.3-56.3 212-62h199568v120H200432c-178.3\n 11.7-311.7 78.3-403 201-6 8-9.7 12-11 12-.7.7-6.7 1-18 1s-17.3-.3-18-1c-1.3 0\n-5-4-11-12-44.7-59.3-101.3-106.3-170-141s-145.3-54.3-229-60H0V214z",oiintSize1:"M512.6 71.6c272.6 0 320.3 106.8 320.3 178.2 0 70.8-47.7 177.6\n-320.3 177.6S193.1 320.6 193.1 249.8c0-71.4 46.9-178.2 319.5-178.2z\nm368.1 178.2c0-86.4-60.9-215.4-368.1-215.4-306.4 0-367.3 129-367.3 215.4 0 85.8\n60.9 214.8 367.3 214.8 307.2 0 368.1-129 368.1-214.8z",oiintSize2:"M757.8 100.1c384.7 0 451.1 137.6 451.1 230 0 91.3-66.4 228.8\n-451.1 228.8-386.3 0-452.7-137.5-452.7-228.8 0-92.4 66.4-230 452.7-230z\nm502.4 230c0-111.2-82.4-277.2-502.4-277.2s-504 166-504 277.2\nc0 110 84 276 504 276s502.4-166 502.4-276z",oiiintSize1:"M681.4 71.6c408.9 0 480.5 106.8 480.5 178.2 0 70.8-71.6 177.6\n-480.5 177.6S202.1 320.6 202.1 249.8c0-71.4 70.5-178.2 479.3-178.2z\nm525.8 178.2c0-86.4-86.8-215.4-525.7-215.4-437.9 0-524.7 129-524.7 215.4 0\n85.8 86.8 214.8 524.7 214.8 438.9 0 525.7-129 525.7-214.8z",oiiintSize2:"M1021.2 53c603.6 0 707.8 165.8 707.8 277.2 0 110-104.2 275.8\n-707.8 275.8-606 0-710.2-165.8-710.2-275.8C311 218.8 415.2 53 1021.2 53z\nm770.4 277.1c0-131.2-126.4-327.6-770.5-327.6S248.4 198.9 248.4 330.1\nc0 130 128.8 326.4 772.7 326.4s770.5-196.4 770.5-326.4z",rightarrow:"M0 241v40h399891c-47.3 35.3-84 78-110 128\n-16.7 32-27.7 63.7-33 95 0 1.3-.2 2.7-.5 4-.3 1.3-.5 2.3-.5 3 0 7.3 6.7 11 20\n 11 8 0 13.2-.8 15.5-2.5 2.3-1.7 4.2-5.5 5.5-11.5 2-13.3 5.7-27 11-41 14.7-44.7\n 39-84.5 73-119.5s73.7-60.2 119-75.5c6-2 9-5.7 9-11s-3-9-9-11c-45.3-15.3-85\n-40.5-119-75.5s-58.3-74.8-73-119.5c-4.7-14-8.3-27.3-11-40-1.3-6.7-3.2-10.8-5.5\n-12.5-2.3-1.7-7.5-2.5-15.5-2.5-14 0-21 3.7-21 11 0 2 2 10.3 6 25 20.7 83.3 67\n 151.7 139 205zm0 0v40h399900v-40z",rightbrace:"M400000 542l\n-6 6h-17c-12.7 0-19.3-.3-20-1-4-4-7.3-8.3-10-13-35.3-51.3-80.8-93.8-136.5-127.5\ns-117.2-55.8-184.5-66.5c-.7 0-2-.3-4-1-18.7-2.7-76-4.3-172-5H0V214h399571l6 1\nc124.7 8 235 61.7 331 161 31.3 33.3 59.7 72.7 85 118l7 13v35z",rightbraceunder:"M399994 0l6 6v35l-6 11c-56 104-135.3 181.3-238 232-57.3\n 28.7-117 45-179 50H-300V214h399897c43.3-7 81-15 113-26 100.7-33 179.7-91 237\n-174 2.7-5 6-9 10-13 .7-1 7.3-1 20-1h17z",rightgroup:"M0 80h399565c371 0 266.7 149.4 414 180 5.9 1.2 18 0 18 0 2 0\n 3-1 3-3v-38c-76-158-257-219-435-219H0z",rightgroupunder:"M0 262h399565c371 0 266.7-149.4 414-180 5.9-1.2 18 0 18\n 0 2 0 3 1 3 3v38c-76 158-257 219-435 219H0z",rightharpoon:"M0 241v40h399993c4.7-4.7 7-9.3 7-14 0-9.3\n-3.7-15.3-11-18-92.7-56.7-159-133.7-199-231-3.3-9.3-6-14.7-8-16-2-1.3-7-2-15-2\n-10.7 0-16.7 2-18 6-2 2.7-1 9.7 3 21 15.3 42 36.7 81.8 64 119.5 27.3 37.7 58\n 69.2 92 94.5zm0 0v40h399900v-40z",rightharpoonplus:"M0 241v40h399993c4.7-4.7 7-9.3 7-14 0-9.3-3.7-15.3-11\n-18-92.7-56.7-159-133.7-199-231-3.3-9.3-6-14.7-8-16-2-1.3-7-2-15-2-10.7 0-16.7\n 2-18 6-2 2.7-1 9.7 3 21 15.3 42 36.7 81.8 64 119.5 27.3 37.7 58 69.2 92 94.5z\nm0 0v40h399900v-40z m100 194v40h399900v-40zm0 0v40h399900v-40z",rightharpoondown:"M399747 511c0 7.3 6.7 11 20 11 8 0 13-.8 15-2.5s4.7-6.8\n 8-15.5c40-94 99.3-166.3 178-217 13.3-8 20.3-12.3 21-13 5.3-3.3 8.5-5.8 9.5\n-7.5 1-1.7 1.5-5.2 1.5-10.5s-2.3-10.3-7-15H0v40h399908c-34 25.3-64.7 57-92 95\n-27.3 38-48.7 77.7-64 119-3.3 8.7-5 14-5 16zM0 241v40h399900v-40z",rightharpoondownplus:"M399747 705c0 7.3 6.7 11 20 11 8 0 13-.8\n 15-2.5s4.7-6.8 8-15.5c40-94 99.3-166.3 178-217 13.3-8 20.3-12.3 21-13 5.3-3.3\n 8.5-5.8 9.5-7.5 1-1.7 1.5-5.2 1.5-10.5s-2.3-10.3-7-15H0v40h399908c-34 25.3\n-64.7 57-92 95-27.3 38-48.7 77.7-64 119-3.3 8.7-5 14-5 16zM0 435v40h399900v-40z\nm0-194v40h400000v-40zm0 0v40h400000v-40z",righthook:"M399859 241c-764 0 0 0 0 0 40-3.3 68.7-15.7 86-37 10-12 15-25.3\n 15-40 0-22.7-9.8-40.7-29.5-54-19.7-13.3-43.5-21-71.5-23-17.3-1.3-26-8-26-20 0\n-13.3 8.7-20 26-20 38 0 71 11.2 99 33.5 0 0 7 5.6 21 16.7 14 11.2 21 33.5 21\n 66.8s-14 61.2-42 83.5c-28 22.3-61 33.5-99 33.5L0 241z M0 281v-40h399859v40z",rightlinesegment:"M399960 241 V94 h40 V428 h-40 V281 H0 v-40z\nM399960 241 V94 h40 V428 h-40 V281 H0 v-40z",rightToFrom:"M400000 167c-70.7-42-118-97.7-142-167h-23c-15.3 0-23 .3-23\n 1 0 1.3 5.3 13.7 16 37 18 35.3 41.3 69 70 101l7 8H0v40h399905l-7 8c-28.7 32\n-52 65.7-70 101-10.7 23.3-16 35.7-16 37 0 .7 7.7 1 23 1h23c24-69.3 71.3-125 142\n-167z M100 147v40h399900v-40zM0 341v40h399900v-40z",twoheadleftarrow:"M0 167c68 40\n 115.7 95.7 143 167h22c15.3 0 23-.3 23-1 0-1.3-5.3-13.7-16-37-18-35.3-41.3-69\n-70-101l-7-8h125l9 7c50.7 39.3 85 86 103 140h46c0-4.7-6.3-18.7-19-42-18-35.3\n-40-67.3-66-96l-9-9h399716v-40H284l9-9c26-28.7 48-60.7 66-96 12.7-23.333 19\n-37.333 19-42h-46c-18 54-52.3 100.7-103 140l-9 7H95l7-8c28.7-32 52-65.7 70-101\n 10.7-23.333 16-35.7 16-37 0-.7-7.7-1-23-1h-22C115.7 71.3 68 127 0 167z",twoheadrightarrow:"M400000 167\nc-68-40-115.7-95.7-143-167h-22c-15.3 0-23 .3-23 1 0 1.3 5.3 13.7 16 37 18 35.3\n 41.3 69 70 101l7 8h-125l-9-7c-50.7-39.3-85-86-103-140h-46c0 4.7 6.3 18.7 19 42\n 18 35.3 40 67.3 66 96l9 9H0v40h399716l-9 9c-26 28.7-48 60.7-66 96-12.7 23.333\n-19 37.333-19 42h46c18-54 52.3-100.7 103-140l9-7h125l-7 8c-28.7 32-52 65.7-70\n 101-10.7 23.333-16 35.7-16 37 0 .7 7.7 1 23 1h22c27.3-71.3 75-127 143-167z",tilde1:"M200 55.538c-77 0-168 73.953-177 73.953-3 0-7\n-2.175-9-5.437L2 97c-1-2-2-4-2-6 0-4 2-7 5-9l20-12C116 12 171 0 207 0c86 0\n 114 68 191 68 78 0 168-68 177-68 4 0 7 2 9 5l12 19c1 2.175 2 4.35 2 6.525 0\n 4.35-2 7.613-5 9.788l-19 13.05c-92 63.077-116.937 75.308-183 76.128\n-68.267.847-113-73.952-191-73.952z",tilde2:"M344 55.266c-142 0-300.638 81.316-311.5 86.418\n-8.01 3.762-22.5 10.91-23.5 5.562L1 120c-1-2-1-3-1-4 0-5 3-9 8-10l18.4-9C160.9\n 31.9 283 0 358 0c148 0 188 122 331 122s314-97 326-97c4 0 8 2 10 7l7 21.114\nc1 2.14 1 3.21 1 4.28 0 5.347-3 9.626-7 10.696l-22.3 12.622C852.6 158.372 751\n 181.476 676 181.476c-149 0-189-126.21-332-126.21z",tilde3:"M786 59C457 59 32 175.242 13 175.242c-6 0-10-3.457\n-11-10.37L.15 138c-1-7 3-12 10-13l19.2-6.4C378.4 40.7 634.3 0 804.3 0c337 0\n 411.8 157 746.8 157 328 0 754-112 773-112 5 0 10 3 11 9l1 14.075c1 8.066-.697\n 16.595-6.697 17.492l-21.052 7.31c-367.9 98.146-609.15 122.696-778.15 122.696\n -338 0-409-156.573-744-156.573z",tilde4:"M786 58C457 58 32 177.487 13 177.487c-6 0-10-3.345\n-11-10.035L.15 143c-1-7 3-12 10-13l22-6.7C381.2 35 637.15 0 807.15 0c337 0 409\n 177 744 177 328 0 754-127 773-127 5 0 10 3 11 9l1 14.794c1 7.805-3 13.38-9\n 14.495l-20.7 5.574c-366.85 99.79-607.3 139.372-776.3 139.372-338 0-409\n -175.236-744-175.236z",vec:"M377 20c0-5.333 1.833-10 5.5-14S391 0 397 0c4.667 0 8.667 1.667 12 5\n3.333 2.667 6.667 9 10 19 6.667 24.667 20.333 43.667 41 57 7.333 4.667 11\n10.667 11 18 0 6-1 10-3 12s-6.667 5-14 9c-28.667 14.667-53.667 35.667-75 63\n-1.333 1.333-3.167 3.5-5.5 6.5s-4 4.833-5 5.5c-1 .667-2.5 1.333-4.5 2s-4.333 1\n-7 1c-4.667 0-9.167-1.833-13.5-5.5S337 184 337 178c0-12.667 15.667-32.333 47-59\nH213l-171-1c-8.667-6-13-12.333-13-19 0-4.667 4.333-11.333 13-20h359\nc-16-25.333-24-45-24-59z",widehat1:"M529 0h5l519 115c5 1 9 5 9 10 0 1-1 2-1 3l-4 22\nc-1 5-5 9-11 9h-2L532 67 19 159h-2c-5 0-9-4-11-9l-5-22c-1-6 2-12 8-13z",widehat2:"M1181 0h2l1171 176c6 0 10 5 10 11l-2 23c-1 6-5 10\n-11 10h-1L1182 67 15 220h-1c-6 0-10-4-11-10l-2-23c-1-6 4-11 10-11z",widehat3:"M1181 0h2l1171 236c6 0 10 5 10 11l-2 23c-1 6-5 10\n-11 10h-1L1182 67 15 280h-1c-6 0-10-4-11-10l-2-23c-1-6 4-11 10-11z",widehat4:"M1181 0h2l1171 296c6 0 10 5 10 11l-2 23c-1 6-5 10\n-11 10h-1L1182 67 15 340h-1c-6 0-10-4-11-10l-2-23c-1-6 4-11 10-11z",widecheck1:"M529,159h5l519,-115c5,-1,9,-5,9,-10c0,-1,-1,-2,-1,-3l-4,-22c-1,\n-5,-5,-9,-11,-9h-2l-512,92l-513,-92h-2c-5,0,-9,4,-11,9l-5,22c-1,6,2,12,8,13z",widecheck2:"M1181,220h2l1171,-176c6,0,10,-5,10,-11l-2,-23c-1,-6,-5,-10,\n-11,-10h-1l-1168,153l-1167,-153h-1c-6,0,-10,4,-11,10l-2,23c-1,6,4,11,10,11z",widecheck3:"M1181,280h2l1171,-236c6,0,10,-5,10,-11l-2,-23c-1,-6,-5,-10,\n-11,-10h-1l-1168,213l-1167,-213h-1c-6,0,-10,4,-11,10l-2,23c-1,6,4,11,10,11z",widecheck4:"M1181,340h2l1171,-296c6,0,10,-5,10,-11l-2,-23c-1,-6,-5,-10,\n-11,-10h-1l-1168,273l-1167,-273h-1c-6,0,-10,4,-11,10l-2,23c-1,6,4,11,10,11z",baraboveleftarrow:"M400000 620h-399890l3 -3c68.7 -52.7 113.7 -120 135 -202\nc4 -14.7 6 -23 6 -25c0 -7.3 -7 -11 -21 -11c-8 0 -13.2 0.8 -15.5 2.5\nc-2.3 1.7 -4.2 5.8 -5.5 12.5c-1.3 4.7 -2.7 10.3 -4 17c-12 48.7 -34.8 92 -68.5 130\ns-74.2 66.3 -121.5 85c-10 4 -16 7.7 -18 11c0 8.7 6 14.3 18 17c47.3 18.7 87.8 47\n121.5 85s56.5 81.3 68.5 130c0.7 2 1.3 5 2 9s1.2 6.7 1.5 8c0.3 1.3 1 3.3 2 6\ns2.2 4.5 3.5 5.5c1.3 1 3.3 1.8 6 2.5s6 1 10 1c14 0 21 -3.7 21 -11\nc0 -2 -2 -10.3 -6 -25c-20 -79.3 -65 -146.7 -135 -202l-3 -3h399890z\nM100 620v40h399900v-40z M0 241v40h399900v-40zM0 241v40h399900v-40z",rightarrowabovebar:"M0 241v40h399891c-47.3 35.3-84 78-110 128-16.7 32\n-27.7 63.7-33 95 0 1.3-.2 2.7-.5 4-.3 1.3-.5 2.3-.5 3 0 7.3 6.7 11 20 11 8 0\n13.2-.8 15.5-2.5 2.3-1.7 4.2-5.5 5.5-11.5 2-13.3 5.7-27 11-41 14.7-44.7 39\n-84.5 73-119.5s73.7-60.2 119-75.5c6-2 9-5.7 9-11s-3-9-9-11c-45.3-15.3-85-40.5\n-119-75.5s-58.3-74.8-73-119.5c-4.7-14-8.3-27.3-11-40-1.3-6.7-3.2-10.8-5.5\n-12.5-2.3-1.7-7.5-2.5-15.5-2.5-14 0-21 3.7-21 11 0 2 2 10.3 6 25 20.7 83.3 67\n151.7 139 205zm96 379h399894v40H0zm0 0h399904v40H0z",baraboveshortleftharpoon:"M507,435c-4,4,-6.3,8.7,-7,14c0,5.3,0.7,9,2,11\nc1.3,2,5.3,5.3,12,10c90.7,54,156,130,196,228c3.3,10.7,6.3,16.3,9,17\nc2,0.7,5,1,9,1c0,0,5,0,5,0c10.7,0,16.7,-2,18,-6c2,-2.7,1,-9.7,-3,-21\nc-32,-87.3,-82.7,-157.7,-152,-211c0,0,-3,-3,-3,-3l399351,0l0,-40\nc-398570,0,-399437,0,-399437,0z M593 435 v40 H399500 v-40z\nM0 281 v-40 H399908 v40z M0 281 v-40 H399908 v40z",rightharpoonaboveshortbar:"M0,241 l0,40c399126,0,399993,0,399993,0\nc4.7,-4.7,7,-9.3,7,-14c0,-9.3,-3.7,-15.3,-11,-18c-92.7,-56.7,-159,-133.7,-199,\n-231c-3.3,-9.3,-6,-14.7,-8,-16c-2,-1.3,-7,-2,-15,-2c-10.7,0,-16.7,2,-18,6\nc-2,2.7,-1,9.7,3,21c15.3,42,36.7,81.8,64,119.5c27.3,37.7,58,69.2,92,94.5z\nM0 241 v40 H399908 v-40z M0 475 v-40 H399500 v40z M0 475 v-40 H399500 v40z",shortbaraboveleftharpoon:"M7,435c-4,4,-6.3,8.7,-7,14c0,5.3,0.7,9,2,11\nc1.3,2,5.3,5.3,12,10c90.7,54,156,130,196,228c3.3,10.7,6.3,16.3,9,17c2,0.7,5,1,9,\n1c0,0,5,0,5,0c10.7,0,16.7,-2,18,-6c2,-2.7,1,-9.7,-3,-21c-32,-87.3,-82.7,-157.7,\n-152,-211c0,0,-3,-3,-3,-3l399907,0l0,-40c-399126,0,-399993,0,-399993,0z\nM93 435 v40 H400000 v-40z M500 241 v40 H400000 v-40z M500 241 v40 H400000 v-40z",shortrightharpoonabovebar:"M53,241l0,40c398570,0,399437,0,399437,0\nc4.7,-4.7,7,-9.3,7,-14c0,-9.3,-3.7,-15.3,-11,-18c-92.7,-56.7,-159,-133.7,-199,\n-231c-3.3,-9.3,-6,-14.7,-8,-16c-2,-1.3,-7,-2,-15,-2c-10.7,0,-16.7,2,-18,6\nc-2,2.7,-1,9.7,3,21c15.3,42,36.7,81.8,64,119.5c27.3,37.7,58,69.2,92,94.5z\nM500 241 v40 H399408 v-40z M500 435 v40 H400000 v-40z"},M=function(){function e(e){this.children=void 0,this.classes=void 0,this.height=void 0,this.depth=void 0,this.maxFontSize=void 0,this.style=void 0,this.children=e,this.classes=[],this.height=0,this.depth=0,this.maxFontSize=0,this.style={}}var t=e.prototype;return t.hasClass=function(e){return l.contains(this.classes,e)},t.toNode=function(){for(var e=document.createDocumentFragment(),t=0;t"},q=function(){function e(e,t,r,n){this.children=void 0,this.attributes=void 0,this.classes=void 0,this.height=void 0,this.depth=void 0,this.width=void 0,this.maxFontSize=void 0,this.style=void 0,A.call(this,e,r,n),this.children=t||[]}var t=e.prototype;return t.setAttribute=function(e,t){this.attributes[e]=t},t.hasClass=function(e){return l.contains(this.classes,e)},t.toNode=function(){return T.call(this,"span")},t.toMarkup=function(){return B.call(this,"span")},e}(),N=function(){function e(e,t,r,n){this.children=void 0,this.attributes=void 0,this.classes=void 0,this.height=void 0,this.depth=void 0,this.maxFontSize=void 0,this.style=void 0,A.call(this,t,n),this.children=r||[],this.setAttribute("href",e)}var t=e.prototype;return t.setAttribute=function(e,t){this.attributes[e]=t},t.hasClass=function(e){return l.contains(this.classes,e)},t.toNode=function(){return T.call(this,"a")},t.toMarkup=function(){return B.call(this,"a")},e}(),C=function(){function e(e,t,r){this.src=void 0,this.alt=void 0,this.classes=void 0,this.height=void 0,this.depth=void 0,this.maxFontSize=void 0,this.style=void 0,this.alt=t,this.src=e,this.classes=["mord"],this.style=r}var t=e.prototype;return t.hasClass=function(e){return l.contains(this.classes,e)},t.toNode=function(){var e=document.createElement("img");for(var t in e.src=this.src,e.alt=this.alt,e.className="mord",this.style)this.style.hasOwnProperty(t)&&(e.style[t]=this.style[t]);return e},t.toMarkup=function(){var e=""+this.alt+"=a[0]&&e<=a[1])return r.name}return null}(this.text.charCodeAt(0));l&&this.classes.push(l+"_fallback"),/[\xee\xef\xed\xec]/.test(this.text)&&(this.text=I[this.text])}var t=e.prototype;return t.hasClass=function(e){return l.contains(this.classes,e)},t.toNode=function(){var e=document.createTextNode(this.text),t=null;for(var r in this.italic>0&&((t=document.createElement("span")).style.marginRight=this.italic+"em"),this.classes.length>0&&((t=t||document.createElement("span")).className=z(this.classes)),this.style)this.style.hasOwnProperty(r)&&((t=t||document.createElement("span")).style[r]=this.style[r]);return t?(t.appendChild(e),t):e},t.toMarkup=function(){var e=!1,t="0&&(r+="margin-right:"+this.italic+"em;"),this.style)this.style.hasOwnProperty(n)&&(r+=l.hyphenate(n)+":"+this.style[n]+";");r&&(e=!0,t+=' style="'+l.escape(r)+'"');var a=l.escape(this.text);return e?(t+=">",t+=a,t+=""):a},e}(),R=function(){function e(e,t){this.children=void 0,this.attributes=void 0,this.children=e||[],this.attributes=t||{}}var t=e.prototype;return t.toNode=function(){var e=document.createElementNS("http://www.w3.org/2000/svg","svg");for(var t in this.attributes)Object.prototype.hasOwnProperty.call(this.attributes,t)&&e.setAttribute(t,this.attributes[t]);for(var r=0;r":""},e}(),H=function(){function e(e){this.attributes=void 0,this.attributes=e||{}}var t=e.prototype;return t.toNode=function(){var e=document.createElementNS("http://www.w3.org/2000/svg","line");for(var t in this.attributes)Object.prototype.hasOwnProperty.call(this.attributes,t)&&e.setAttribute(t,this.attributes[t]);return e},t.toMarkup=function(){var e="","\\gt",!0),_(j,Z,oe,"\u2208","\\in",!0),_(j,Z,oe,"\ue020","\\@not"),_(j,Z,oe,"\u2282","\\subset",!0),_(j,Z,oe,"\u2283","\\supset",!0),_(j,Z,oe,"\u2286","\\subseteq",!0),_(j,Z,oe,"\u2287","\\supseteq",!0),_(j,K,oe,"\u2288","\\nsubseteq",!0),_(j,K,oe,"\u2289","\\nsupseteq",!0),_(j,Z,oe,"\u22a8","\\models"),_(j,Z,oe,"\u2190","\\leftarrow",!0),_(j,Z,oe,"\u2264","\\le"),_(j,Z,oe,"\u2264","\\leq",!0),_(j,Z,oe,"<","\\lt",!0),_(j,Z,oe,"\u2192","\\rightarrow",!0),_(j,Z,oe,"\u2192","\\to"),_(j,K,oe,"\u2271","\\ngeq",!0),_(j,K,oe,"\u2270","\\nleq",!0),_(j,Z,se,"\xa0","\\ "),_(j,Z,se,"\xa0","~"),_(j,Z,se,"\xa0","\\space"),_(j,Z,se,"\xa0","\\nobreakspace"),_($,Z,se,"\xa0","\\ "),_($,Z,se,"\xa0"," "),_($,Z,se,"\xa0","~"),_($,Z,se,"\xa0","\\space"),_($,Z,se,"\xa0","\\nobreakspace"),_(j,Z,se,null,"\\nobreak"),_(j,Z,se,null,"\\allowbreak"),_(j,Z,ie,",",","),_(j,Z,ie,";",";"),_(j,K,Q,"\u22bc","\\barwedge",!0),_(j,K,Q,"\u22bb","\\veebar",!0),_(j,Z,Q,"\u2299","\\odot",!0),_(j,Z,Q,"\u2295","\\oplus",!0),_(j,Z,Q,"\u2297","\\otimes",!0),_(j,Z,le,"\u2202","\\partial",!0),_(j,Z,Q,"\u2298","\\oslash",!0),_(j,K,Q,"\u229a","\\circledcirc",!0),_(j,K,Q,"\u22a1","\\boxdot",!0),_(j,Z,Q,"\u25b3","\\bigtriangleup"),_(j,Z,Q,"\u25bd","\\bigtriangledown"),_(j,Z,Q,"\u2020","\\dagger"),_(j,Z,Q,"\u22c4","\\diamond"),_(j,Z,Q,"\u22c6","\\star"),_(j,Z,Q,"\u25c3","\\triangleleft"),_(j,Z,Q,"\u25b9","\\triangleright"),_(j,Z,ae,"{","\\{"),_($,Z,le,"{","\\{"),_($,Z,le,"{","\\textbraceleft"),_(j,Z,ee,"}","\\}"),_($,Z,le,"}","\\}"),_($,Z,le,"}","\\textbraceright"),_(j,Z,ae,"{","\\lbrace"),_(j,Z,ee,"}","\\rbrace"),_(j,Z,ae,"[","\\lbrack",!0),_($,Z,le,"[","\\lbrack",!0),_(j,Z,ee,"]","\\rbrack",!0),_($,Z,le,"]","\\rbrack",!0),_(j,Z,ae,"(","\\lparen",!0),_(j,Z,ee,")","\\rparen",!0),_($,Z,le,"<","\\textless",!0),_($,Z,le,">","\\textgreater",!0),_(j,Z,ae,"\u230a","\\lfloor",!0),_(j,Z,ee,"\u230b","\\rfloor",!0),_(j,Z,ae,"\u2308","\\lceil",!0),_(j,Z,ee,"\u2309","\\rceil",!0),_(j,Z,le,"\\","\\backslash"),_(j,Z,le,"\u2223","|"),_(j,Z,le,"\u2223","\\vert"),_($,Z,le,"|","\\textbar",!0),_(j,Z,le,"\u2225","\\|"),_(j,Z,le,"\u2225","\\Vert"),_($,Z,le,"\u2225","\\textbardbl"),_($,Z,le,"~","\\textasciitilde"),_($,Z,le,"\\","\\textbackslash"),_($,Z,le,"^","\\textasciicircum"),_(j,Z,oe,"\u2191","\\uparrow",!0),_(j,Z,oe,"\u21d1","\\Uparrow",!0),_(j,Z,oe,"\u2193","\\downarrow",!0),_(j,Z,oe,"\u21d3","\\Downarrow",!0),_(j,Z,oe,"\u2195","\\updownarrow",!0),_(j,Z,oe,"\u21d5","\\Updownarrow",!0),_(j,Z,ne,"\u2210","\\coprod"),_(j,Z,ne,"\u22c1","\\bigvee"),_(j,Z,ne,"\u22c0","\\bigwedge"),_(j,Z,ne,"\u2a04","\\biguplus"),_(j,Z,ne,"\u22c2","\\bigcap"),_(j,Z,ne,"\u22c3","\\bigcup"),_(j,Z,ne,"\u222b","\\int"),_(j,Z,ne,"\u222b","\\intop"),_(j,Z,ne,"\u222c","\\iint"),_(j,Z,ne,"\u222d","\\iiint"),_(j,Z,ne,"\u220f","\\prod"),_(j,Z,ne,"\u2211","\\sum"),_(j,Z,ne,"\u2a02","\\bigotimes"),_(j,Z,ne,"\u2a01","\\bigoplus"),_(j,Z,ne,"\u2a00","\\bigodot"),_(j,Z,ne,"\u222e","\\oint"),_(j,Z,ne,"\u222f","\\oiint"),_(j,Z,ne,"\u2230","\\oiiint"),_(j,Z,ne,"\u2a06","\\bigsqcup"),_(j,Z,ne,"\u222b","\\smallint"),_($,Z,te,"\u2026","\\textellipsis"),_(j,Z,te,"\u2026","\\mathellipsis"),_($,Z,te,"\u2026","\\ldots",!0),_(j,Z,te,"\u2026","\\ldots",!0),_(j,Z,te,"\u22ef","\\@cdots",!0),_(j,Z,te,"\u22f1","\\ddots",!0),_(j,Z,le,"\u22ee","\\varvdots"),_(j,Z,J,"\u02ca","\\acute"),_(j,Z,J,"\u02cb","\\grave"),_(j,Z,J,"\xa8","\\ddot"),_(j,Z,J,"~","\\tilde"),_(j,Z,J,"\u02c9","\\bar"),_(j,Z,J,"\u02d8","\\breve"),_(j,Z,J,"\u02c7","\\check"),_(j,Z,J,"^","\\hat"),_(j,Z,J,"\u20d7","\\vec"),_(j,Z,J,"\u02d9","\\dot"),_(j,Z,J,"\u02da","\\mathring"),_(j,Z,re,"\ue131","\\@imath"),_(j,Z,re,"\ue237","\\@jmath"),_(j,Z,le,"\u0131","\u0131"),_(j,Z,le,"\u0237","\u0237"),_($,Z,le,"\u0131","\\i",!0),_($,Z,le,"\u0237","\\j",!0),_($,Z,le,"\xdf","\\ss",!0),_($,Z,le,"\xe6","\\ae",!0),_($,Z,le,"\u0153","\\oe",!0),_($,Z,le,"\xf8","\\o",!0),_($,Z,le,"\xc6","\\AE",!0),_($,Z,le,"\u0152","\\OE",!0),_($,Z,le,"\xd8","\\O",!0),_($,Z,J,"\u02ca","\\'"),_($,Z,J,"\u02cb","\\`"),_($,Z,J,"\u02c6","\\^"),_($,Z,J,"\u02dc","\\~"),_($,Z,J,"\u02c9","\\="),_($,Z,J,"\u02d8","\\u"),_($,Z,J,"\u02d9","\\."),_($,Z,J,"\u02da","\\r"),_($,Z,J,"\u02c7","\\v"),_($,Z,J,"\xa8",'\\"'),_($,Z,J,"\u02dd","\\H"),_($,Z,J,"\u25ef","\\textcircled");var he={"--":!0,"---":!0,"``":!0,"''":!0};_($,Z,le,"\u2013","--",!0),_($,Z,le,"\u2013","\\textendash"),_($,Z,le,"\u2014","---",!0),_($,Z,le,"\u2014","\\textemdash"),_($,Z,le,"\u2018","`",!0),_($,Z,le,"\u2018","\\textquoteleft"),_($,Z,le,"\u2019","'",!0),_($,Z,le,"\u2019","\\textquoteright"),_($,Z,le,"\u201c","``",!0),_($,Z,le,"\u201c","\\textquotedblleft"),_($,Z,le,"\u201d","''",!0),_($,Z,le,"\u201d","\\textquotedblright"),_(j,Z,le,"\xb0","\\degree",!0),_($,Z,le,"\xb0","\\degree"),_($,Z,le,"\xb0","\\textdegree",!0),_(j,Z,le,"\xa3","\\pounds"),_(j,Z,le,"\xa3","\\mathsterling",!0),_($,Z,le,"\xa3","\\pounds"),_($,Z,le,"\xa3","\\textsterling",!0),_(j,K,le,"\u2720","\\maltese"),_($,K,le,"\u2720","\\maltese");for(var me='0123456789/@."',ce=0;ce=5?0:e>=3?1:2]){var r=G[t]={cssEmPerMu:P.quad[t]/18};for(var n in P)P.hasOwnProperty(n)&&(r[n]=P[n][t])}return G[t]}(this.size)),this._fontMetrics},t.getColor=function(){return this.phantom?"transparent":this.color},e}();Ie.BASESIZE=6;var Oe=Ie,Re={pt:1,mm:7227/2540,cm:7227/254,in:72.27,bp:1.00375,pc:12,dd:1238/1157,cc:14856/1157,nd:685/642,nc:1370/107,sp:1/65536,px:1.00375},Ee={ex:!0,em:!0,mu:!0},He=function(e){return"string"!=typeof e&&(e=e.unit),e in Re||e in Ee||"ex"===e},Le=function(e,t){var r;if(e.unit in Re)r=Re[e.unit]/t.fontMetrics().ptPerEm/t.sizeMultiplier;else if("mu"===e.unit)r=t.fontMetrics().cssEmPerMu;else{var a;if(a=t.style.isTight()?t.havingStyle(t.style.text()):t,"ex"===e.unit)r=a.fontMetrics().xHeight;else{if("em"!==e.unit)throw new n("Invalid unit: '"+e.unit+"'");r=a.fontMetrics().quad}a!==t&&(r*=a.sizeMultiplier/t.sizeMultiplier)}return Math.min(e.number*r,t.maxSize)},De=function(e,t,r){return X[r][e]&&X[r][e].replace&&(e=X[r][e].replace),{value:e,metrics:V(e,t,r)}},Pe=function(e,t,r,n,a){var i,o=De(e,t,r),s=o.metrics;if(e=o.value,s){var l=s.italic;("text"===r||n&&"mathit"===n.font)&&(l=0),i=new O(e,s.height,s.depth,l,s.skew,s.width,a)}else"undefined"!=typeof console&&console.warn("No character metrics for '"+e+"' in style '"+t+"' and mode '"+r+"'"),i=new O(e,0,0,0,0,0,a);if(n){i.maxFontSize=n.sizeMultiplier,n.style.isTight()&&i.classes.push("mtight");var h=n.getColor();h&&(i.style.color=h)}return i},Fe=function(e,t){if(z(e.classes)!==z(t.classes)||e.skew!==t.skew||e.maxFontSize!==t.maxFontSize)return!1;if(1===e.classes.length){var r=e.classes[0];if("mbin"===r||"mord"===r)return!1}for(var n in e.style)if(e.style.hasOwnProperty(n)&&e.style[n]!==t.style[n])return!1;for(var a in t.style)if(t.style.hasOwnProperty(a)&&e.style[a]!==t.style[a])return!1;return!0},Ve=function(e){for(var t=0,r=0,n=0,a=0;at&&(t=i.height),i.depth>r&&(r=i.depth),i.maxFontSize>n&&(n=i.maxFontSize)}e.height=t,e.depth=r,e.maxFontSize=n},Ge=function(e,t,r,n){var a=new q(e,t,r,n);return Ve(a),a},Ue=function(e,t,r,n){return new q(e,t,r,n)},Ye=function(e){var t=new M(e);return Ve(t),t},We=function(e,t,r){var n="";switch(e){case"amsrm":n="AMS";break;case"textrm":n="Main";break;case"textsf":n="SansSerif";break;case"texttt":n="Typewriter";break;default:n=e}return n+"-"+("textbf"===t&&"textit"===r?"BoldItalic":"textbf"===t?"Bold":"textit"===t?"Italic":"Regular")},Xe={mathbf:{variant:"bold",fontName:"Main-Bold"},mathrm:{variant:"normal",fontName:"Main-Regular"},textit:{variant:"italic",fontName:"Main-Italic"},mathit:{variant:"italic",fontName:"Main-Italic"},mathnormal:{variant:"italic",fontName:"Math-Italic"},mathbb:{variant:"double-struck",fontName:"AMS-Regular"},mathcal:{variant:"script",fontName:"Caligraphic-Regular"},mathfrak:{variant:"fraktur",fontName:"Fraktur-Regular"},mathscr:{variant:"script",fontName:"Script-Regular"},mathsf:{variant:"sans-serif",fontName:"SansSerif-Regular"},mathtt:{variant:"monospace",fontName:"Typewriter-Regular"}},_e={vec:["vec",.471,.714],oiintSize1:["oiintSize1",.957,.499],oiintSize2:["oiintSize2",1.472,.659],oiiintSize1:["oiiintSize1",1.304,.499],oiiintSize2:["oiiintSize2",1.98,.659]},je={fontMap:Xe,makeSymbol:Pe,mathsym:function(e,t,r,n){return void 0===n&&(n=[]),"boldsymbol"===r.font&&De(e,"Main-Bold",t).metrics?Pe(e,"Main-Bold",t,r,n.concat(["mathbf"])):"\\"===e||"main"===X[t][e].font?Pe(e,"Main-Regular",t,r,n):Pe(e,"AMS-Regular",t,r,n.concat(["amsrm"]))},makeSpan:Ge,makeSvgSpan:Ue,makeLineSpan:function(e,t,r){var n=Ge([e],[],t);return n.height=Math.max(r||t.fontMetrics().defaultRuleThickness,t.minRuleThickness),n.style.borderBottomWidth=n.height+"em",n.maxFontSize=1,n},makeAnchor:function(e,t,r,n){var a=new N(e,t,r,n);return Ve(a),a},makeFragment:Ye,wrapFragment:function(e,t){return e instanceof M?Ge([],[e],t):e},makeVList:function(e,t){for(var r=function(e){if("individualShift"===e.positionType){for(var t=e.children,r=[t[0]],n=-t[0].shift-t[0].elem.depth,a=n,i=1;i0&&(o.push(yt(s,t)),s=[]),o.push(a[l]));s.length>0&&o.push(yt(s,t)),r?((i=yt(ut(r,t,!0))).classes=["tag"],o.push(i)):n&&o.push(n);var m=st(["katex-html"],o);if(m.setAttribute("aria-hidden","true"),i){var c=i.children[0];c.style.height=m.height+m.depth+"em",c.style.verticalAlign=-m.depth+"em"}return m}function wt(e){return new M(e)}var kt=function(){function e(e,t,r){this.type=void 0,this.attributes=void 0,this.children=void 0,this.classes=void 0,this.type=e,this.attributes={},this.children=t||[],this.classes=r||[]}var t=e.prototype;return t.setAttribute=function(e,t){this.attributes[e]=t},t.getAttribute=function(e){return this.attributes[e]},t.toNode=function(){var e=document.createElementNS("http://www.w3.org/1998/Math/MathML",this.type);for(var t in this.attributes)Object.prototype.hasOwnProperty.call(this.attributes,t)&&e.setAttribute(t,this.attributes[t]);this.classes.length>0&&(e.className=z(this.classes));for(var r=0;r0&&(e+=' class ="'+l.escape(z(this.classes))+'"'),e+=">";for(var r=0;r"},t.toText=function(){return this.children.map((function(e){return e.toText()})).join("")},e}(),St=function(){function e(e){this.text=void 0,this.text=e}var t=e.prototype;return t.toNode=function(){return document.createTextNode(this.text)},t.toMarkup=function(){return l.escape(this.toText())},t.toText=function(){return this.text},e}(),Mt={MathNode:kt,TextNode:St,SpaceNode:function(){function e(e){this.width=void 0,this.character=void 0,this.width=e,this.character=e>=.05555&&e<=.05556?"\u200a":e>=.1666&&e<=.1667?"\u2009":e>=.2222&&e<=.2223?"\u2005":e>=.2777&&e<=.2778?"\u2005\u200a":e>=-.05556&&e<=-.05555?"\u200a\u2063":e>=-.1667&&e<=-.1666?"\u2009\u2063":e>=-.2223&&e<=-.2222?"\u205f\u2063":e>=-.2778&&e<=-.2777?"\u2005\u2063":null}var t=e.prototype;return t.toNode=function(){if(this.character)return document.createTextNode(this.character);var e=document.createElementNS("http://www.w3.org/1998/Math/MathML","mspace");return e.setAttribute("width",this.width+"em"),e},t.toMarkup=function(){return this.character?""+this.character+"":''},t.toText=function(){return this.character?this.character:" "},e}(),newDocumentFragment:wt},zt=function(e,t,r){return!X[t][e]||!X[t][e].replace||55349===e.charCodeAt(0)||he.hasOwnProperty(e)&&r&&(r.fontFamily&&"tt"===r.fontFamily.substr(4,2)||r.font&&"tt"===r.font.substr(4,2))||(e=X[t][e].replace),new Mt.TextNode(e)},At=function(e){return 1===e.length?e[0]:new Mt.MathNode("mrow",e)},Tt=function(e,t){if("texttt"===t.fontFamily)return"monospace";if("textsf"===t.fontFamily)return"textit"===t.fontShape&&"textbf"===t.fontWeight?"sans-serif-bold-italic":"textit"===t.fontShape?"sans-serif-italic":"textbf"===t.fontWeight?"bold-sans-serif":"sans-serif";if("textit"===t.fontShape&&"textbf"===t.fontWeight)return"bold-italic";if("textit"===t.fontShape)return"italic";if("textbf"===t.fontWeight)return"bold";var r=t.font;if(!r||"mathnormal"===r)return null;var n=e.mode;if("mathit"===r)return"italic";if("boldsymbol"===r)return"textord"===e.type?"bold":"bold-italic";if("mathbf"===r)return"bold";if("mathbb"===r)return"double-struck";if("mathfrak"===r)return"fraktur";if("mathscr"===r||"mathcal"===r)return"script";if("mathsf"===r)return"sans-serif";if("mathtt"===r)return"monospace";var a=e.text;return l.contains(["\\imath","\\jmath"],a)?null:(X[n][a]&&X[n][a].replace&&(a=X[n][a].replace),V(a,je.fontMap[r].fontName,n)?je.fontMap[r].variant:null)},Bt=function(e,t,r){if(1===e.length){var n=Nt(e[0],t);return r&&n instanceof kt&&"mo"===n.type&&(n.setAttribute("lspace","0em"),n.setAttribute("rspace","0em")),[n]}for(var a,i=[],o=0;o0&&(p.text=p.text.slice(0,1)+"\u0338"+p.text.slice(1),i.pop())}}}i.push(s),a=s}return i},qt=function(e,t,r){return At(Bt(e,t,r))},Nt=function(e,t){if(!e)return new Mt.MathNode("mrow");if(rt[e.type])return rt[e.type](e,t);throw new n("Got group of unknown type: '"+e.type+"'")};function Ct(e,t,r,n,a){var i,o=Bt(e,r);i=1===o.length&&o[0]instanceof kt&&l.contains(["mrow","mtable"],o[0].type)?o[0]:new Mt.MathNode("mrow",o);var s=new Mt.MathNode("annotation",[new Mt.TextNode(t)]);s.setAttribute("encoding","application/x-tex");var h=new Mt.MathNode("semantics",[i,s]),m=new Mt.MathNode("math",[h]);m.setAttribute("xmlns","http://www.w3.org/1998/Math/MathML"),n&&m.setAttribute("display","block");var c=a?"katex":"katex-mathml";return je.makeSpan([c],[m])}var It=function(e){return new Oe({style:e.displayMode?b.DISPLAY:b.TEXT,maxSize:e.maxSize,minRuleThickness:e.minRuleThickness})},Ot=function(e,t){if(t.displayMode){var r=["katex-display"];t.leqno&&r.push("leqno"),t.fleqn&&r.push("fleqn"),e=je.makeSpan(r,[e])}return e},Rt=function(e,t,r){var n,a=It(r);if("mathml"===r.output)return Ct(e,t,a,r.displayMode,!0);if("html"===r.output){var i=xt(e,a);n=je.makeSpan(["katex"],[i])}else{var o=Ct(e,t,a,r.displayMode,!1),s=xt(e,a);n=je.makeSpan(["katex"],[o,s])}return Ot(n,r)},Et={widehat:"^",widecheck:"\u02c7",widetilde:"~",utilde:"~",overleftarrow:"\u2190",underleftarrow:"\u2190",xleftarrow:"\u2190",overrightarrow:"\u2192",underrightarrow:"\u2192",xrightarrow:"\u2192",underbrace:"\u23df",overbrace:"\u23de",overgroup:"\u23e0",undergroup:"\u23e1",overleftrightarrow:"\u2194",underleftrightarrow:"\u2194",xleftrightarrow:"\u2194",Overrightarrow:"\u21d2",xRightarrow:"\u21d2",overleftharpoon:"\u21bc",xleftharpoonup:"\u21bc",overrightharpoon:"\u21c0",xrightharpoonup:"\u21c0",xLeftarrow:"\u21d0",xLeftrightarrow:"\u21d4",xhookleftarrow:"\u21a9",xhookrightarrow:"\u21aa",xmapsto:"\u21a6",xrightharpoondown:"\u21c1",xleftharpoondown:"\u21bd",xrightleftharpoons:"\u21cc",xleftrightharpoons:"\u21cb",xtwoheadleftarrow:"\u219e",xtwoheadrightarrow:"\u21a0",xlongequal:"=",xtofrom:"\u21c4",xrightleftarrows:"\u21c4",xrightequilibrium:"\u21cc",xleftequilibrium:"\u21cb","\\\\cdrightarrow":"\u2192","\\\\cdleftarrow":"\u2190","\\\\cdlongequal":"="},Ht={overrightarrow:[["rightarrow"],.888,522,"xMaxYMin"],overleftarrow:[["leftarrow"],.888,522,"xMinYMin"],underrightarrow:[["rightarrow"],.888,522,"xMaxYMin"],underleftarrow:[["leftarrow"],.888,522,"xMinYMin"],xrightarrow:[["rightarrow"],1.469,522,"xMaxYMin"],"\\cdrightarrow":[["rightarrow"],3,522,"xMaxYMin"],xleftarrow:[["leftarrow"],1.469,522,"xMinYMin"],"\\cdleftarrow":[["leftarrow"],3,522,"xMinYMin"],Overrightarrow:[["doublerightarrow"],.888,560,"xMaxYMin"],xRightarrow:[["doublerightarrow"],1.526,560,"xMaxYMin"],xLeftarrow:[["doubleleftarrow"],1.526,560,"xMinYMin"],overleftharpoon:[["leftharpoon"],.888,522,"xMinYMin"],xleftharpoonup:[["leftharpoon"],.888,522,"xMinYMin"],xleftharpoondown:[["leftharpoondown"],.888,522,"xMinYMin"],overrightharpoon:[["rightharpoon"],.888,522,"xMaxYMin"],xrightharpoonup:[["rightharpoon"],.888,522,"xMaxYMin"],xrightharpoondown:[["rightharpoondown"],.888,522,"xMaxYMin"],xlongequal:[["longequal"],.888,334,"xMinYMin"],"\\cdlongequal":[["longequal"],3,334,"xMinYMin"],xtwoheadleftarrow:[["twoheadleftarrow"],.888,334,"xMinYMin"],xtwoheadrightarrow:[["twoheadrightarrow"],.888,334,"xMaxYMin"],overleftrightarrow:[["leftarrow","rightarrow"],.888,522],overbrace:[["leftbrace","midbrace","rightbrace"],1.6,548],underbrace:[["leftbraceunder","midbraceunder","rightbraceunder"],1.6,548],underleftrightarrow:[["leftarrow","rightarrow"],.888,522],xleftrightarrow:[["leftarrow","rightarrow"],1.75,522],xLeftrightarrow:[["doubleleftarrow","doublerightarrow"],1.75,560],xrightleftharpoons:[["leftharpoondownplus","rightharpoonplus"],1.75,716],xleftrightharpoons:[["leftharpoonplus","rightharpoondownplus"],1.75,716],xhookleftarrow:[["leftarrow","righthook"],1.08,522],xhookrightarrow:[["lefthook","rightarrow"],1.08,522],overlinesegment:[["leftlinesegment","rightlinesegment"],.888,522],underlinesegment:[["leftlinesegment","rightlinesegment"],.888,522],overgroup:[["leftgroup","rightgroup"],.888,342],undergroup:[["leftgroupunder","rightgroupunder"],.888,342],xmapsto:[["leftmapsto","rightarrow"],1.5,522],xtofrom:[["leftToFrom","rightToFrom"],1.75,528],xrightleftarrows:[["baraboveleftarrow","rightarrowabovebar"],1.75,901],xrightequilibrium:[["baraboveshortleftharpoon","rightharpoonaboveshortbar"],1.75,716],xleftequilibrium:[["shortbaraboveleftharpoon","shortrightharpoonabovebar"],1.75,716]},Lt=function(e,t,r,n,a){var i,o=e.height+e.depth+r+n;if(/fbox|color|angl/.test(t)){if(i=je.makeSpan(["stretchy",t],[],a),"fbox"===t){var s=a.color&&a.getColor();s&&(i.style.borderColor=s)}}else{var l=[];/^[bx]cancel$/.test(t)&&l.push(new H({x1:"0",y1:"0",x2:"100%",y2:"100%","stroke-width":"0.046em"})),/^x?cancel$/.test(t)&&l.push(new H({x1:"0",y1:"100%",x2:"100%",y2:"0","stroke-width":"0.046em"}));var h=new R(l,{width:"100%",height:o+"em"});i=je.makeSvgSpan([],[h],a)}return i.height=o,i.style.height=o+"em",i},Dt=function(e){var t=new Mt.MathNode("mo",[new Mt.TextNode(Et[e])]);return t.setAttribute("stretchy","true"),t},Pt=function(e,t){var r=function(){var r=4e5,n=e.label.substr(1);if(l.contains(["widehat","widecheck","widetilde","utilde"],n)){var a,i,o,s="ordgroup"===(d=e.base).type?d.body.length:1;if(s>5)"widehat"===n||"widecheck"===n?(a=420,r=2364,o=.42,i=n+"4"):(a=312,r=2340,o=.34,i="tilde4");else{var h=[1,1,2,2,3,3][s];"widehat"===n||"widecheck"===n?(r=[0,1062,2364,2364,2364][h],a=[0,239,300,360,420][h],o=[0,.24,.3,.3,.36,.42][h],i=n+h):(r=[0,600,1033,2339,2340][h],a=[0,260,286,306,312][h],o=[0,.26,.286,.3,.306,.34][h],i="tilde"+h)}var m=new E(i),c=new R([m],{width:"100%",height:o+"em",viewBox:"0 0 "+r+" "+a,preserveAspectRatio:"none"});return{span:je.makeSvgSpan([],[c],t),minWidth:0,height:o}}var u,p,d,f=[],g=Ht[n],v=g[0],b=g[1],y=g[2],x=y/1e3,w=v.length;if(1===w)u=["hide-tail"],p=[g[3]];else if(2===w)u=["halfarrow-left","halfarrow-right"],p=["xMinYMin","xMaxYMin"];else{if(3!==w)throw new Error("Correct katexImagesData or update code here to support\n "+w+" children.");u=["brace-left","brace-center","brace-right"],p=["xMinYMin","xMidYMin","xMaxYMin"]}for(var k=0;k0&&(n.style.minWidth=a+"em"),n};function Ft(e,t){if(!e||e.type!==t)throw new Error("Expected node of type "+t+", but got "+(e?"node of type "+e.type:String(e)));return e}function Vt(e){var t=Gt(e);if(!t)throw new Error("Expected node of symbol group type, but got "+(e?"node of type "+e.type:String(e)));return t}function Gt(e){return e&&("atom"===e.type||Y.hasOwnProperty(e.type))?e:null}var Ut=function(e,t){var r,n,a;e&&"supsub"===e.type?(r=(n=Ft(e.base,"accent")).base,e.base=r,a=function(e){if(e instanceof q)return e;throw new Error("Expected span but got "+String(e)+".")}(bt(e,t)),e.base=n):r=(n=Ft(e,"accent")).base;var i=bt(r,t.havingCrampedStyle()),o=0;if(n.isShifty&&l.isCharacterBox(r)){var s=l.getBaseElem(r);o=L(bt(s,t.havingCrampedStyle())).skew}var h,m=Math.min(i.height,t.fontMetrics().xHeight);if(n.isStretchy)h=Pt(n,t),h=je.makeVList({positionType:"firstBaseline",children:[{type:"elem",elem:i},{type:"elem",elem:h,wrapperClasses:["svg-align"],wrapperStyle:o>0?{width:"calc(100% - "+2*o+"em)",marginLeft:2*o+"em"}:void 0}]},t);else{var c,u;"\\vec"===n.label?(c=je.staticSvg("vec",t),u=je.svgData.vec[1]):((c=L(c=je.makeOrd({mode:n.mode,text:n.label},t,"textord"))).italic=0,u=c.width),h=je.makeSpan(["accent-body"],[c]);var p="\\textcircled"===n.label;p&&(h.classes.push("accent-full"),m=i.height);var d=o;p||(d-=u/2),h.style.left=d+"em","\\textcircled"===n.label&&(h.style.top=".2em"),h=je.makeVList({positionType:"firstBaseline",children:[{type:"elem",elem:i},{type:"kern",size:-m},{type:"elem",elem:h}]},t)}var f=je.makeSpan(["mord","accent"],[h],t);return a?(a.children[0]=f,a.height=Math.max(f.height,a.height),a.classes[0]="mord",a):f},Yt=function(e,t){var r=e.isStretchy?Dt(e.label):new Mt.MathNode("mo",[zt(e.label,e.mode)]),n=new Mt.MathNode("mover",[Nt(e.base,t),r]);return n.setAttribute("accent","true"),n},Wt=new RegExp(["\\acute","\\grave","\\ddot","\\tilde","\\bar","\\breve","\\check","\\hat","\\vec","\\dot","\\mathring"].map((function(e){return"\\"+e})).join("|"));nt({type:"accent",names:["\\acute","\\grave","\\ddot","\\tilde","\\bar","\\breve","\\check","\\hat","\\vec","\\dot","\\mathring","\\widecheck","\\widehat","\\widetilde","\\overrightarrow","\\overleftarrow","\\Overrightarrow","\\overleftrightarrow","\\overgroup","\\overlinesegment","\\overleftharpoon","\\overrightharpoon"],props:{numArgs:1},handler:function(e,t){var r=it(t[0]),n=!Wt.test(e.funcName),a=!n||"\\widehat"===e.funcName||"\\widetilde"===e.funcName||"\\widecheck"===e.funcName;return{type:"accent",mode:e.parser.mode,label:e.funcName,isStretchy:n,isShifty:a,base:r}},htmlBuilder:Ut,mathmlBuilder:Yt}),nt({type:"accent",names:["\\'","\\`","\\^","\\~","\\=","\\u","\\.",'\\"',"\\r","\\H","\\v","\\textcircled"],props:{numArgs:1,allowedInText:!0,allowedInMath:!1,argTypes:["primitive"]},handler:function(e,t){var r=t[0];return{type:"accent",mode:e.parser.mode,label:e.funcName,isStretchy:!1,isShifty:!0,base:r}},htmlBuilder:Ut,mathmlBuilder:Yt}),nt({type:"accentUnder",names:["\\underleftarrow","\\underrightarrow","\\underleftrightarrow","\\undergroup","\\underlinesegment","\\utilde"],props:{numArgs:1},handler:function(e,t){var r=e.parser,n=e.funcName,a=t[0];return{type:"accentUnder",mode:r.mode,label:n,base:a}},htmlBuilder:function(e,t){var r=bt(e.base,t),n=Pt(e,t),a="\\utilde"===e.label?.12:0,i=je.makeVList({positionType:"top",positionData:r.height,children:[{type:"elem",elem:n,wrapperClasses:["svg-align"]},{type:"kern",size:a},{type:"elem",elem:r}]},t);return je.makeSpan(["mord","accentunder"],[i],t)},mathmlBuilder:function(e,t){var r=Dt(e.label),n=new Mt.MathNode("munder",[Nt(e.base,t),r]);return n.setAttribute("accentunder","true"),n}});var Xt=function(e){var t=new Mt.MathNode("mpadded",e?[e]:[]);return t.setAttribute("width","+0.6em"),t.setAttribute("lspace","0.3em"),t};nt({type:"xArrow",names:["\\xleftarrow","\\xrightarrow","\\xLeftarrow","\\xRightarrow","\\xleftrightarrow","\\xLeftrightarrow","\\xhookleftarrow","\\xhookrightarrow","\\xmapsto","\\xrightharpoondown","\\xrightharpoonup","\\xleftharpoondown","\\xleftharpoonup","\\xrightleftharpoons","\\xleftrightharpoons","\\xlongequal","\\xtwoheadrightarrow","\\xtwoheadleftarrow","\\xtofrom","\\xrightleftarrows","\\xrightequilibrium","\\xleftequilibrium","\\\\cdrightarrow","\\\\cdleftarrow","\\\\cdlongequal"],props:{numArgs:1,numOptionalArgs:1},handler:function(e,t,r){var n=e.parser,a=e.funcName;return{type:"xArrow",mode:n.mode,label:a,body:t[0],below:r[0]}},htmlBuilder:function(e,t){var r,n=t.style,a=t.havingStyle(n.sup()),i=je.wrapFragment(bt(e.body,a,t),t),o="\\x"===e.label.slice(0,2)?"x":"cd";i.classes.push(o+"-arrow-pad"),e.below&&(a=t.havingStyle(n.sub()),(r=je.wrapFragment(bt(e.below,a,t),t)).classes.push(o+"-arrow-pad"));var s,l=Pt(e,t),h=-t.fontMetrics().axisHeight+.5*l.height,m=-t.fontMetrics().axisHeight-.5*l.height-.111;if((i.depth>.25||"\\xleftequilibrium"===e.label)&&(m-=i.depth),r){var c=-t.fontMetrics().axisHeight+r.height+.5*l.height+.111;s=je.makeVList({positionType:"individualShift",children:[{type:"elem",elem:i,shift:m},{type:"elem",elem:l,shift:h},{type:"elem",elem:r,shift:c}]},t)}else s=je.makeVList({positionType:"individualShift",children:[{type:"elem",elem:i,shift:m},{type:"elem",elem:l,shift:h}]},t);return s.children[0].children[0].children[1].classes.push("svg-align"),je.makeSpan(["mrel","x-arrow"],[s],t)},mathmlBuilder:function(e,t){var r,n=Dt(e.label);if(n.setAttribute("minsize","x"===e.label.charAt(0)?"1.75em":"3.0em"),e.body){var a=Xt(Nt(e.body,t));if(e.below){var i=Xt(Nt(e.below,t));r=new Mt.MathNode("munderover",[n,i,a])}else r=new Mt.MathNode("mover",[n,a])}else if(e.below){var o=Xt(Nt(e.below,t));r=new Mt.MathNode("munder",[n,o])}else r=Xt(),r=new Mt.MathNode("mover",[n,r]);return r}});var _t={">":"\\\\cdrightarrow","<":"\\\\cdleftarrow","=":"\\\\cdlongequal",A:"\\uparrow",V:"\\downarrow","|":"\\Vert",".":"no arrow"},jt=function(e){return"textord"===e.type&&"@"===e.text};function $t(e,t,r){var n=_t[e];switch(n){case"\\\\cdrightarrow":case"\\\\cdleftarrow":return r.callFunction(n,[t[0]],[t[1]]);case"\\uparrow":case"\\downarrow":var a={type:"atom",text:n,mode:"math",family:"rel"},i={type:"ordgroup",mode:"math",body:[r.callFunction("\\\\cdleft",[t[0]],[]),r.callFunction("\\Big",[a],[]),r.callFunction("\\\\cdright",[t[1]],[])]};return r.callFunction("\\\\cdparent",[i],[]);case"\\\\cdlongequal":return r.callFunction("\\\\cdlongequal",[],[]);case"\\Vert":return r.callFunction("\\Big",[{type:"textord",text:"\\Vert",mode:"math"}],[]);default:return{type:"textord",text:" ",mode:"math"}}}nt({type:"cdlabel",names:["\\\\cdleft","\\\\cdright"],props:{numArgs:1},handler:function(e,t){var r=e.parser,n=e.funcName;return{type:"cdlabel",mode:r.mode,side:n.slice(4),label:t[0]}},htmlBuilder:function(e,t){var r=t.havingStyle(t.style.sup()),n=je.wrapFragment(bt(e.label,r,t),t);return n.classes.push("cd-label-"+e.side),n.style.bottom=.8-n.depth+"em",n.height=0,n.depth=0,n},mathmlBuilder:function(e,t){var r=new Mt.MathNode("mrow",[Nt(e.label,t)]);return(r=new Mt.MathNode("mpadded",[r])).setAttribute("width","0"),"left"===e.side&&r.setAttribute("lspace","-1width"),r.setAttribute("voffset","0.7em"),(r=new Mt.MathNode("mstyle",[r])).setAttribute("displaystyle","false"),r.setAttribute("scriptlevel","1"),r}}),nt({type:"cdlabelparent",names:["\\\\cdparent"],props:{numArgs:1},handler:function(e,t){return{type:"cdlabelparent",mode:e.parser.mode,fragment:t[0]}},htmlBuilder:function(e,t){var r=je.wrapFragment(bt(e.fragment,t),t);return r.classes.push("cd-vert-arrow"),r},mathmlBuilder:function(e,t){return new Mt.MathNode("mrow",[Nt(e.fragment,t)])}}),nt({type:"textord",names:["\\@char"],props:{numArgs:1,allowedInText:!0},handler:function(e,t){for(var r=e.parser,a=Ft(t[0],"ordgroup").body,i="",o=0;o","\\langle","\\rangle","/","\\backslash","\\lt","\\gt"],gr=[0,1.2,1.8,2.4,3],vr=[{type:"small",style:b.SCRIPTSCRIPT},{type:"small",style:b.SCRIPT},{type:"small",style:b.TEXT},{type:"large",size:1},{type:"large",size:2},{type:"large",size:3},{type:"large",size:4}],br=[{type:"small",style:b.SCRIPTSCRIPT},{type:"small",style:b.SCRIPT},{type:"small",style:b.TEXT},{type:"stack"}],yr=[{type:"small",style:b.SCRIPTSCRIPT},{type:"small",style:b.SCRIPT},{type:"small",style:b.TEXT},{type:"large",size:1},{type:"large",size:2},{type:"large",size:3},{type:"large",size:4},{type:"stack"}],xr=function(e){if("small"===e.type)return"Main-Regular";if("large"===e.type)return"Size"+e.size+"-Regular";if("stack"===e.type)return"Size4-Regular";throw new Error("Add support for delim type '"+e.type+"' here.")},wr=function(e,t,r,n){for(var a=Math.min(2,3-n.style.size);at)return r[a]}return r[r.length-1]},kr=function(e,t,r,n,a,i){var o;"<"===e||"\\lt"===e||"\u27e8"===e?e="\\langle":">"!==e&&"\\gt"!==e&&"\u27e9"!==e||(e="\\rangle"),o=l.contains(fr,e)?vr:l.contains(pr,e)?yr:br;var s=wr(e,t,o,n);return"small"===s.type?function(e,t,r,n,a,i){var o=je.makeSymbol(e,"Main-Regular",a,n),s=rr(o,t,n,i);return r&&nr(s,n,t),s}(e,s.style,r,n,a,i):"large"===s.type?ar(e,s.size,r,n,a,i):mr(e,t,r,n,a,i)},Sr=function(e,t){var r,n,a=t.havingBaseSizing(),i=wr("\\surd",e*a.sizeMultiplier,yr,a),o=a.sizeMultiplier,s=Math.max(0,t.minRuleThickness-t.fontMetrics().sqrtRuleThickness),l=0,h=0,m=0;return"small"===i.type?(e<1?o=1:e<1.4&&(o=.7),h=(1+s)/o,(r=ur("sqrtMain",l=(1+s+cr)/o,m=1e3+1e3*s+80,s,t)).style.minWidth="0.853em",n=.833/o):"large"===i.type?(m=1080*gr[i.size],h=(gr[i.size]+s)/o,l=(gr[i.size]+s+cr)/o,(r=ur("sqrtSize"+i.size,l,m,s,t)).style.minWidth="1.02em",n=1/o):(l=e+s+cr,h=e+s,m=Math.floor(1e3*e+s)+80,(r=ur("sqrtTall",l,m,s,t)).style.minWidth="0.742em",n=1.056),r.height=h,r.style.height=l+"em",{span:r,advanceWidth:n,ruleWidth:(t.fontMetrics().sqrtRuleThickness+s)*o}},Mr=function(e,t,r,a,i){if("<"===e||"\\lt"===e||"\u27e8"===e?e="\\langle":">"!==e&&"\\gt"!==e&&"\u27e9"!==e||(e="\\rangle"),l.contains(pr,e)||l.contains(fr,e))return ar(e,t,!1,r,a,i);if(l.contains(dr,e))return mr(e,gr[t],!1,r,a,i);throw new n("Illegal delimiter: '"+e+"'")},zr=gr,Ar=kr,Tr=function(e,t,r,n,a,i){var o=n.fontMetrics().axisHeight*n.sizeMultiplier,s=5/n.fontMetrics().ptPerEm,l=Math.max(t-o,r+o),h=Math.max(l/500*901,2*l-s);return kr(e,h,!0,n,a,i)},Br={"\\bigl":{mclass:"mopen",size:1},"\\Bigl":{mclass:"mopen",size:2},"\\biggl":{mclass:"mopen",size:3},"\\Biggl":{mclass:"mopen",size:4},"\\bigr":{mclass:"mclose",size:1},"\\Bigr":{mclass:"mclose",size:2},"\\biggr":{mclass:"mclose",size:3},"\\Biggr":{mclass:"mclose",size:4},"\\bigm":{mclass:"mrel",size:1},"\\Bigm":{mclass:"mrel",size:2},"\\biggm":{mclass:"mrel",size:3},"\\Biggm":{mclass:"mrel",size:4},"\\big":{mclass:"mord",size:1},"\\Big":{mclass:"mord",size:2},"\\bigg":{mclass:"mord",size:3},"\\Bigg":{mclass:"mord",size:4}},qr=["(","\\lparen",")","\\rparen","[","\\lbrack","]","\\rbrack","\\{","\\lbrace","\\}","\\rbrace","\\lfloor","\\rfloor","\u230a","\u230b","\\lceil","\\rceil","\u2308","\u2309","<",">","\\langle","\u27e8","\\rangle","\u27e9","\\lt","\\gt","\\lvert","\\rvert","\\lVert","\\rVert","\\lgroup","\\rgroup","\u27ee","\u27ef","\\lmoustache","\\rmoustache","\u23b0","\u23b1","/","\\backslash","|","\\vert","\\|","\\Vert","\\uparrow","\\Uparrow","\\downarrow","\\Downarrow","\\updownarrow","\\Updownarrow","."];function Nr(e,t){var r=Gt(e);if(r&&l.contains(qr,r.text))return r;throw new n(r?"Invalid delimiter '"+r.text+"' after '"+t.funcName+"'":"Invalid delimiter type '"+e.type+"'",e)}function Cr(e){if(!e.body)throw new Error("Bug: The leftright ParseNode wasn't fully parsed.")}nt({type:"delimsizing",names:["\\bigl","\\Bigl","\\biggl","\\Biggl","\\bigr","\\Bigr","\\biggr","\\Biggr","\\bigm","\\Bigm","\\biggm","\\Biggm","\\big","\\Big","\\bigg","\\Bigg"],props:{numArgs:1,argTypes:["primitive"]},handler:function(e,t){var r=Nr(t[0],e);return{type:"delimsizing",mode:e.parser.mode,size:Br[e.funcName].size,mclass:Br[e.funcName].mclass,delim:r.text}},htmlBuilder:function(e,t){return"."===e.delim?je.makeSpan([e.mclass]):Mr(e.delim,e.size,t,e.mode,[e.mclass])},mathmlBuilder:function(e){var t=[];"."!==e.delim&&t.push(zt(e.delim,e.mode));var r=new Mt.MathNode("mo",t);return"mopen"===e.mclass||"mclose"===e.mclass?r.setAttribute("fence","true"):r.setAttribute("fence","false"),r.setAttribute("stretchy","true"),r.setAttribute("minsize",zr[e.size]+"em"),r.setAttribute("maxsize",zr[e.size]+"em"),r}}),nt({type:"leftright-right",names:["\\right"],props:{numArgs:1,primitive:!0},handler:function(e,t){var r=e.parser.gullet.macros.get("\\current@color");if(r&&"string"!=typeof r)throw new n("\\current@color set to non-string in \\right");return{type:"leftright-right",mode:e.parser.mode,delim:Nr(t[0],e).text,color:r}}}),nt({type:"leftright",names:["\\left"],props:{numArgs:1,primitive:!0},handler:function(e,t){var r=Nr(t[0],e),n=e.parser;++n.leftrightDepth;var a=n.parseExpression(!1);--n.leftrightDepth,n.expect("\\right",!1);var i=Ft(n.parseFunction(),"leftright-right");return{type:"leftright",mode:n.mode,body:a,left:r.text,right:i.delim,rightColor:i.color}},htmlBuilder:function(e,t){Cr(e);for(var r,n,a=ut(e.body,t,!0,["mopen","mclose"]),i=0,o=0,s=!1,l=0;l-1?"mpadded":"menclose",[Nt(e.body,t)]);switch(e.label){case"\\cancel":n.setAttribute("notation","updiagonalstrike");break;case"\\bcancel":n.setAttribute("notation","downdiagonalstrike");break;case"\\phase":n.setAttribute("notation","phasorangle");break;case"\\sout":n.setAttribute("notation","horizontalstrike");break;case"\\fbox":n.setAttribute("notation","box");break;case"\\angl":n.setAttribute("notation","actuarial");break;case"\\fcolorbox":case"\\colorbox":if(r=t.fontMetrics().fboxsep*t.fontMetrics().ptPerEm,n.setAttribute("width","+"+2*r+"pt"),n.setAttribute("height","+"+2*r+"pt"),n.setAttribute("lspace",r+"pt"),n.setAttribute("voffset",r+"pt"),"\\fcolorbox"===e.label){var a=Math.max(t.fontMetrics().fboxrule,t.minRuleThickness);n.setAttribute("style","border: "+a+"em solid "+String(e.borderColor))}break;case"\\xcancel":n.setAttribute("notation","updiagonalstrike downdiagonalstrike")}return e.backgroundColor&&n.setAttribute("mathbackground",e.backgroundColor),n};nt({type:"enclose",names:["\\colorbox"],props:{numArgs:2,allowedInText:!0,argTypes:["color","text"]},handler:function(e,t,r){var n=e.parser,a=e.funcName,i=Ft(t[0],"color-token").color,o=t[1];return{type:"enclose",mode:n.mode,label:a,backgroundColor:i,body:o}},htmlBuilder:Ir,mathmlBuilder:Or}),nt({type:"enclose",names:["\\fcolorbox"],props:{numArgs:3,allowedInText:!0,argTypes:["color","color","text"]},handler:function(e,t,r){var n=e.parser,a=e.funcName,i=Ft(t[0],"color-token").color,o=Ft(t[1],"color-token").color,s=t[2];return{type:"enclose",mode:n.mode,label:a,backgroundColor:o,borderColor:i,body:s}},htmlBuilder:Ir,mathmlBuilder:Or}),nt({type:"enclose",names:["\\fbox"],props:{numArgs:1,argTypes:["hbox"],allowedInText:!0},handler:function(e,t){return{type:"enclose",mode:e.parser.mode,label:"\\fbox",body:t[0]}}}),nt({type:"enclose",names:["\\cancel","\\bcancel","\\xcancel","\\sout","\\phase"],props:{numArgs:1},handler:function(e,t){var r=e.parser,n=e.funcName,a=t[0];return{type:"enclose",mode:r.mode,label:n,body:a}},htmlBuilder:Ir,mathmlBuilder:Or}),nt({type:"enclose",names:["\\angl"],props:{numArgs:1,argTypes:["hbox"],allowedInText:!1},handler:function(e,t){return{type:"enclose",mode:e.parser.mode,label:"\\angl",body:t[0]}}});var Rr={};function Er(e){for(var t=e.type,r=e.names,n=e.props,a=e.handler,i=e.htmlBuilder,o=e.mathmlBuilder,s={type:t,numArgs:n.numArgs||0,allowedInText:!1,numOptionalArgs:0,handler:a},l=0;l0&&(x+=.25),m.push({pos:x,isDashed:e[t]})}for(w(o[0]),r=0;r0&&(M<(B+=y)&&(M=B),B=0),e.addJot&&(M+=f),z.height=S,z.depth=M,x+=S,z.pos=x,x+=M+B,h[r]=z,w(o[r+1])}var q,N,C=x/2+t.fontMetrics().axisHeight,I=e.cols||[],O=[],R=[];if(e.addEqnNum)for(r=0;r=s)){var G=void 0;(a>0||e.hskipBeforeAndAfter)&&0!==(G=l.deflt(D.pregap,p))&&((q=je.makeSpan(["arraycolsep"],[])).style.width=G+"em",O.push(q));var U=[];for(r=0;r0){for(var _=je.makeLineSpan("hline",t,c),j=je.makeLineSpan("hdashline",t,c),$=[{type:"elem",elem:h,shift:0}];m.length>0;){var Z=m.pop(),K=Z.pos-C;Z.isDashed?$.push({type:"elem",elem:j,shift:K}):$.push({type:"elem",elem:_,shift:K})}h=je.makeVList({positionType:"individualShift",children:$},t)}if(e.addEqnNum){var J=je.makeVList({positionType:"individualShift",children:R},t);return J=je.makeSpan(["tag"],[J],t),je.makeFragment([h,J])}return je.makeSpan(["mord"],[h],t)},Vr={c:"center ",l:"left ",r:"right "},Gr=function(e,t){for(var r=[],n=new Mt.MathNode("mtd",[],["mtr-glue"]),a=new Mt.MathNode("mtd",[],["mml-eqn-num"]),i=0;i0){var p=e.cols,d="",f=!1,g=0,v=p.length;"separator"===p[0].type&&(c+="top ",g=1),"separator"===p[p.length-1].type&&(c+="bottom ",v-=1);for(var b=g;b0?"left ":"",c+=S[S.length-1].length>0?"right ":"";for(var M=1;M-1?"alignat":"align",o=Dr(e.parser,{cols:a,addJot:!0,addEqnNum:"align"===e.envName||"alignat"===e.envName,colSeparationType:i,maxNumCols:"split"===e.envName?2:void 0,leqno:e.parser.settings.leqno},"display"),s=0,l={type:"ordgroup",mode:e.mode,body:[]};if(t[0]&&"ordgroup"===t[0].type){for(var h="",m=0;m0&&c&&(d=1),a[u]={type:"align",align:p,pregap:d,postgap:0}}return o.colSeparationType=c?"align":"alignat",o};Er({type:"array",names:["array","darray"],props:{numArgs:1},handler:function(e,t){var r=(Gt(t[0])?[t[0]]:Ft(t[0],"ordgroup").body).map((function(e){var t=Vt(e).text;if(-1!=="lcr".indexOf(t))return{type:"align",align:t};if("|"===t)return{type:"separator",separator:"|"};if(":"===t)return{type:"separator",separator:":"};throw new n("Unknown column alignment: "+t,e)})),a={cols:r,hskipBeforeAndAfter:!0,maxNumCols:r.length};return Dr(e.parser,a,Pr(e.envName))},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["matrix","pmatrix","bmatrix","Bmatrix","vmatrix","Vmatrix","matrix*","pmatrix*","bmatrix*","Bmatrix*","vmatrix*","Vmatrix*"],props:{numArgs:0},handler:function(e){var t={matrix:null,pmatrix:["(",")"],bmatrix:["[","]"],Bmatrix:["\\{","\\}"],vmatrix:["|","|"],Vmatrix:["\\Vert","\\Vert"]}[e.envName.replace("*","")],r="c",a={hskipBeforeAndAfter:!1,cols:[{type:"align",align:r}]};if("*"===e.envName.charAt(e.envName.length-1)){var i=e.parser;if(i.consumeSpaces(),"["===i.fetch().text){if(i.consume(),i.consumeSpaces(),r=i.fetch().text,-1==="lcr".indexOf(r))throw new n("Expected l or c or r",i.nextToken);i.consume(),i.consumeSpaces(),i.expect("]"),i.consume(),a.cols=[{type:"align",align:r}]}}var o=Dr(e.parser,a,Pr(e.envName));return o.cols=new Array(o.body[0].length).fill({type:"align",align:r}),t?{type:"leftright",mode:e.mode,body:[o],left:t[0],right:t[1],rightColor:void 0}:o},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["smallmatrix"],props:{numArgs:0},handler:function(e){var t=Dr(e.parser,{arraystretch:.5},"script");return t.colSeparationType="small",t},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["subarray"],props:{numArgs:1},handler:function(e,t){var r=(Gt(t[0])?[t[0]]:Ft(t[0],"ordgroup").body).map((function(e){var t=Vt(e).text;if(-1!=="lc".indexOf(t))return{type:"align",align:t};throw new n("Unknown column alignment: "+t,e)}));if(r.length>1)throw new n("{subarray} can contain only one column");var a={cols:r,hskipBeforeAndAfter:!1,arraystretch:.5};if((a=Dr(e.parser,a,"script")).body.length>0&&a.body[0].length>1)throw new n("{subarray} can contain only one column");return a},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["cases","dcases","rcases","drcases"],props:{numArgs:0},handler:function(e){var t=Dr(e.parser,{arraystretch:1.2,cols:[{type:"align",align:"l",pregap:0,postgap:1},{type:"align",align:"l",pregap:0,postgap:0}]},Pr(e.envName));return{type:"leftright",mode:e.mode,body:[t],left:e.envName.indexOf("r")>-1?".":"\\{",right:e.envName.indexOf("r")>-1?"\\}":".",rightColor:void 0}},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["align","align*","aligned","split"],props:{numArgs:0},handler:Ur,htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["gathered","gather","gather*"],props:{numArgs:0},handler:function(e){l.contains(["gather","gather*"],e.envName)&&Lr(e);var t={cols:[{type:"align",align:"c"}],addJot:!0,colSeparationType:"gather",addEqnNum:"gather"===e.envName,leqno:e.parser.settings.leqno};return Dr(e.parser,t,"display")},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["alignat","alignat*","alignedat"],props:{numArgs:1},handler:Ur,htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["equation","equation*"],props:{numArgs:0},handler:function(e){Lr(e);var t={addEqnNum:"equation"===e.envName,singleRow:!0,maxNumCols:1,leqno:e.parser.settings.leqno};return Dr(e.parser,t,"display")},htmlBuilder:Fr,mathmlBuilder:Gr}),Er({type:"array",names:["CD"],props:{numArgs:0},handler:function(e){return Lr(e),function(e){var t=[];for(e.gullet.beginGroup(),e.gullet.macros.set("\\cr","\\\\\\relax"),e.gullet.beginGroup();;){t.push(e.parseExpression(!1,"\\\\")),e.gullet.endGroup(),e.gullet.beginGroup();var r=e.fetch().text;if("&"!==r&&"\\\\"!==r){if("\\end"===r){0===t[t.length-1].length&&t.pop();break}throw new n("Expected \\\\ or \\cr or \\end",e.nextToken)}e.consume()}for(var a,i,o=[],s=[o],l=0;l-1);else{if(!("<>AV".indexOf(u)>-1))throw new n('Expected one of "<>AV=|." after @',h[c]);for(var d=0;d<2;d++){for(var f=!0,g=c+1;g=b.SCRIPT.id?r.text():b.DISPLAY:"text"===e&&r.size===b.DISPLAY.size?r=b.TEXT:"script"===e?r=b.SCRIPT:"scriptscript"===e&&(r=b.SCRIPTSCRIPT),r},Qr=function(e,t){var r,n=Jr(e.size,t.style),a=n.fracNum(),i=n.fracDen();r=t.havingStyle(a);var o=bt(e.numer,r,t);if(e.continued){var s=8.5/t.fontMetrics().ptPerEm,l=3.5/t.fontMetrics().ptPerEm;o.height=o.height0?3*c:7*c,d=t.fontMetrics().denom1):(m>0?(u=t.fontMetrics().num2,p=c):(u=t.fontMetrics().num3,p=3*c),d=t.fontMetrics().denom2),h){var w=t.fontMetrics().axisHeight;u-o.depth-(w+.5*m)0&&(t="."===(t=e)?null:t),t};nt({type:"genfrac",names:["\\genfrac"],props:{numArgs:6,allowedInArgument:!0,argTypes:["math","math","size","text","math","math"]},handler:function(e,t){var r,n=e.parser,a=t[4],i=t[5],o=it(t[0]),s="atom"===o.type&&"open"===o.family?rn(o.text):null,l=it(t[1]),h="atom"===l.type&&"close"===l.family?rn(l.text):null,m=Ft(t[2],"size"),c=null;r=!!m.isBlank||(c=m.value).number>0;var u="auto",p=t[3];if("ordgroup"===p.type){if(p.body.length>0){var d=Ft(p.body[0],"textord");u=tn[Number(d.text)]}}else p=Ft(p,"textord"),u=tn[Number(p.text)];return{type:"genfrac",mode:n.mode,numer:a,denom:i,continued:!1,hasBarLine:r,barSize:c,leftDelim:s,rightDelim:h,size:u}},htmlBuilder:Qr,mathmlBuilder:en}),nt({type:"infix",names:["\\above"],props:{numArgs:1,argTypes:["size"],infix:!0},handler:function(e,t){var r=e.parser,n=(e.funcName,e.token);return{type:"infix",mode:r.mode,replaceWith:"\\\\abovefrac",size:Ft(t[0],"size").value,token:n}}}),nt({type:"genfrac",names:["\\\\abovefrac"],props:{numArgs:3,argTypes:["math","size","math"]},handler:function(e,t){var r=e.parser,n=(e.funcName,t[0]),a=function(e){if(!e)throw new Error("Expected non-null, but got "+String(e));return e}(Ft(t[1],"infix").size),i=t[2],o=a.number>0;return{type:"genfrac",mode:r.mode,numer:n,denom:i,continued:!1,hasBarLine:o,barSize:a,leftDelim:null,rightDelim:null,size:"auto"}},htmlBuilder:Qr,mathmlBuilder:en});var nn=function(e,t){var r,n,a=t.style;"supsub"===e.type?(r=e.sup?bt(e.sup,t.havingStyle(a.sup()),t):bt(e.sub,t.havingStyle(a.sub()),t),n=Ft(e.base,"horizBrace")):n=Ft(e,"horizBrace");var i,o=bt(n.base,t.havingBaseStyle(b.DISPLAY)),s=Pt(n,t);if(n.isOver?(i=je.makeVList({positionType:"firstBaseline",children:[{type:"elem",elem:o},{type:"kern",size:.1},{type:"elem",elem:s}]},t)).children[0].children[0].children[1].classes.push("svg-align"):(i=je.makeVList({positionType:"bottom",positionData:o.depth+.1+s.height,children:[{type:"elem",elem:s},{type:"kern",size:.1},{type:"elem",elem:o}]},t)).children[0].children[0].children[0].classes.push("svg-align"),r){var l=je.makeSpan(["mord",n.isOver?"mover":"munder"],[i],t);i=n.isOver?je.makeVList({positionType:"firstBaseline",children:[{type:"elem",elem:l},{type:"kern",size:.2},{type:"elem",elem:r}]},t):je.makeVList({positionType:"bottom",positionData:l.depth+.2+r.height+r.depth,children:[{type:"elem",elem:r},{type:"kern",size:.2},{type:"elem",elem:l}]},t)}return je.makeSpan(["mord",n.isOver?"mover":"munder"],[i],t)};nt({type:"horizBrace",names:["\\overbrace","\\underbrace"],props:{numArgs:1},handler:function(e,t){var r=e.parser,n=e.funcName;return{type:"horizBrace",mode:r.mode,label:n,isOver:/^\\over/.test(n),base:t[0]}},htmlBuilder:nn,mathmlBuilder:function(e,t){var r=Dt(e.label);return new Mt.MathNode(e.isOver?"mover":"munder",[Nt(e.base,t),r])}}),nt({type:"href",names:["\\href"],props:{numArgs:2,argTypes:["url","original"],allowedInText:!0},handler:function(e,t){var r=e.parser,n=t[1],a=Ft(t[0],"url").url;return r.settings.isTrusted({command:"\\href",url:a})?{type:"href",mode:r.mode,href:a,body:ot(n)}:r.formatUnsupportedCmd("\\href")},htmlBuilder:function(e,t){var r=ut(e.body,t,!1);return je.makeAnchor(e.href,[],r,t)},mathmlBuilder:function(e,t){var r=qt(e.body,t);return r instanceof kt||(r=new kt("mrow",[r])),r.setAttribute("href",e.href),r}}),nt({type:"href",names:["\\url"],props:{numArgs:1,argTypes:["url"],allowedInText:!0},handler:function(e,t){var r=e.parser,n=Ft(t[0],"url").url;if(!r.settings.isTrusted({command:"\\url",url:n}))return r.formatUnsupportedCmd("\\url");for(var a=[],i=0;i0&&(n=Le(e.totalheight,t)-r,n=Number(n.toFixed(2)));var a=0;e.width.number>0&&(a=Le(e.width,t));var i={height:r+n+"em"};a>0&&(i.width=a+"em"),n>0&&(i.verticalAlign=-n+"em");var o=new C(e.src,e.alt,i);return o.height=r,o.depth=n,o},mathmlBuilder:function(e,t){var r=new Mt.MathNode("mglyph",[]);r.setAttribute("alt",e.alt);var n=Le(e.height,t),a=0;if(e.totalheight.number>0&&(a=(a=Le(e.totalheight,t)-n).toFixed(2),r.setAttribute("valign","-"+a+"em")),r.setAttribute("height",n+a+"em"),e.width.number>0){var i=Le(e.width,t);r.setAttribute("width",i+"em")}return r.setAttribute("src",e.src),r}}),nt({type:"kern",names:["\\kern","\\mkern","\\hskip","\\mskip"],props:{numArgs:1,argTypes:["size"],primitive:!0,allowedInText:!0},handler:function(e,t){var r=e.parser,n=e.funcName,a=Ft(t[0],"size");if(r.settings.strict){var i="m"===n[1],o="mu"===a.value.unit;i?(o||r.settings.reportNonstrict("mathVsTextUnits","LaTeX's "+n+" supports only mu units, not "+a.value.unit+" units"),"math"!==r.mode&&r.settings.reportNonstrict("mathVsTextUnits","LaTeX's "+n+" works only in math mode")):o&&r.settings.reportNonstrict("mathVsTextUnits","LaTeX's "+n+" doesn't support mu units")}return{type:"kern",mode:r.mode,dimension:a.value}},htmlBuilder:function(e,t){return je.makeGlue(e.dimension,t)},mathmlBuilder:function(e,t){var r=Le(e.dimension,t);return new Mt.SpaceNode(r)}}),nt({type:"lap",names:["\\mathllap","\\mathrlap","\\mathclap"],props:{numArgs:1,allowedInText:!0},handler:function(e,t){var r=e.parser,n=e.funcName,a=t[0];return{type:"lap",mode:r.mode,alignment:n.slice(5),body:a}},htmlBuilder:function(e,t){var r;"clap"===e.alignment?(r=je.makeSpan([],[bt(e.body,t)]),r=je.makeSpan(["inner"],[r],t)):r=je.makeSpan(["inner"],[bt(e.body,t)]);var n=je.makeSpan(["fix"],[]),a=je.makeSpan([e.alignment],[r,n],t),i=je.makeSpan(["strut"]);return i.style.height=a.height+a.depth+"em",i.style.verticalAlign=-a.depth+"em",a.children.unshift(i),a=je.makeSpan(["thinbox"],[a],t),je.makeSpan(["mord","vbox"],[a],t)},mathmlBuilder:function(e,t){var r=new Mt.MathNode("mpadded",[Nt(e.body,t)]);if("rlap"!==e.alignment){var n="llap"===e.alignment?"-1":"-0.5";r.setAttribute("lspace",n+"width")}return r.setAttribute("width","0px"),r}}),nt({type:"styling",names:["\\(","$"],props:{numArgs:0,allowedInText:!0,allowedInMath:!1},handler:function(e,t){var r=e.funcName,n=e.parser,a=n.mode;n.switchMode("math");var i="\\("===r?"\\)":"$",o=n.parseExpression(!1,i);return n.expect(i),n.switchMode(a),{type:"styling",mode:n.mode,style:"text",body:o}}}),nt({type:"text",names:["\\)","\\]"],props:{numArgs:0,allowedInText:!0,allowedInMath:!1},handler:function(e,t){throw new n("Mismatched "+e.funcName)}});var on=function(e,t){switch(t.style.size){case b.DISPLAY.size:return e.display;case b.TEXT.size:return e.text;case b.SCRIPT.size:return e.script;case b.SCRIPTSCRIPT.size:return e.scriptscript;default:return e.text}};nt({type:"mathchoice",names:["\\mathchoice"],props:{numArgs:4,primitive:!0},handler:function(e,t){return{type:"mathchoice",mode:e.parser.mode,display:ot(t[0]),text:ot(t[1]),script:ot(t[2]),scriptscript:ot(t[3])}},htmlBuilder:function(e,t){var r=on(e,t),n=ut(r,t,!1);return je.makeFragment(n)},mathmlBuilder:function(e,t){var r=on(e,t);return qt(r,t)}});var sn=function(e,t,r,n,a,i,o){var s,l,h;if(e=je.makeSpan([],[e]),t){var m=bt(t,n.havingStyle(a.sup()),n);l={elem:m,kern:Math.max(n.fontMetrics().bigOpSpacing1,n.fontMetrics().bigOpSpacing3-m.depth)}}if(r){var c=bt(r,n.havingStyle(a.sub()),n);s={elem:c,kern:Math.max(n.fontMetrics().bigOpSpacing2,n.fontMetrics().bigOpSpacing4-c.height)}}if(l&&s){var u=n.fontMetrics().bigOpSpacing5+s.elem.height+s.elem.depth+s.kern+e.depth+o;h=je.makeVList({positionType:"bottom",positionData:u,children:[{type:"kern",size:n.fontMetrics().bigOpSpacing5},{type:"elem",elem:s.elem,marginLeft:-i+"em"},{type:"kern",size:s.kern},{type:"elem",elem:e},{type:"kern",size:l.kern},{type:"elem",elem:l.elem,marginLeft:i+"em"},{type:"kern",size:n.fontMetrics().bigOpSpacing5}]},n)}else if(s){var p=e.height-o;h=je.makeVList({positionType:"top",positionData:p,children:[{type:"kern",size:n.fontMetrics().bigOpSpacing5},{type:"elem",elem:s.elem,marginLeft:-i+"em"},{type:"kern",size:s.kern},{type:"elem",elem:e}]},n)}else{if(!l)return e;var d=e.depth+o;h=je.makeVList({positionType:"bottom",positionData:d,children:[{type:"elem",elem:e},{type:"kern",size:l.kern},{type:"elem",elem:l.elem,marginLeft:i+"em"},{type:"kern",size:n.fontMetrics().bigOpSpacing5}]},n)}return je.makeSpan(["mop","op-limits"],[h],n)},ln=["\\smallint"],hn=function(e,t){var r,n,a,i=!1;"supsub"===e.type?(r=e.sup,n=e.sub,a=Ft(e.base,"op"),i=!0):a=Ft(e,"op");var o,s=t.style,h=!1;if(s.size===b.DISPLAY.size&&a.symbol&&!l.contains(ln,a.name)&&(h=!0),a.symbol){var m=h?"Size2-Regular":"Size1-Regular",c="";if("\\oiint"!==a.name&&"\\oiiint"!==a.name||(c=a.name.substr(1),a.name="oiint"===c?"\\iint":"\\iiint"),o=je.makeSymbol(a.name,m,"math",t,["mop","op-symbol",h?"large-op":"small-op"]),c.length>0){var u=o.italic,p=je.staticSvg(c+"Size"+(h?"2":"1"),t);o=je.makeVList({positionType:"individualShift",children:[{type:"elem",elem:o,shift:0},{type:"elem",elem:p,shift:h?.08:0}]},t),a.name="\\"+c,o.classes.unshift("mop"),o.italic=u}}else if(a.body){var d=ut(a.body,t,!0);1===d.length&&d[0]instanceof O?(o=d[0]).classes[0]="mop":o=je.makeSpan(["mop"],d,t)}else{for(var f=[],g=1;g0){for(var s=a.body.map((function(e){var t=e.text;return"string"==typeof t?{type:"textord",mode:e.mode,text:t}:e})),l=ut(s,t.withFont("mathrm"),!0),h=0;h=0?s.setAttribute("height","+"+a+"em"):(s.setAttribute("height",a+"em"),s.setAttribute("depth","+"+-a+"em")),s.setAttribute("voffset",a+"em"),s}});var fn=["\\tiny","\\sixptsize","\\scriptsize","\\footnotesize","\\small","\\normalsize","\\large","\\Large","\\LARGE","\\huge","\\Huge"];nt({type:"sizing",names:fn,props:{numArgs:0,allowedInText:!0},handler:function(e,t){var r=e.breakOnTokenText,n=e.funcName,a=e.parser,i=a.parseExpression(!1,r);return{type:"sizing",mode:a.mode,size:fn.indexOf(n)+1,body:i}},htmlBuilder:function(e,t){var r=t.havingSize(e.size);return dn(e.body,r,t)},mathmlBuilder:function(e,t){var r=t.havingSize(e.size),n=Bt(e.body,r),a=new Mt.MathNode("mstyle",n);return a.setAttribute("mathsize",r.sizeMultiplier+"em"),a}}),nt({type:"smash",names:["\\smash"],props:{numArgs:1,numOptionalArgs:1,allowedInText:!0},handler:function(e,t,r){var n=e.parser,a=!1,i=!1,o=r[0]&&Ft(r[0],"ordgroup");if(o)for(var s="",l=0;lr.height+r.depth+i&&(i=(i+c-r.height-r.depth)/2);var u=l.height-r.height-i-h;r.style.paddingLeft=m+"em";var p=je.makeVList({positionType:"firstBaseline",children:[{type:"elem",elem:r,wrapperClasses:["svg-align"]},{type:"kern",size:-(r.height+u)},{type:"elem",elem:l},{type:"kern",size:h}]},t);if(e.index){var d=t.havingStyle(b.SCRIPTSCRIPT),f=bt(e.index,d,t),g=.6*(p.height-p.depth),v=je.makeVList({positionType:"shift",positionData:-g,children:[{type:"elem",elem:f}]},t),y=je.makeSpan(["root"],[v]);return je.makeSpan(["mord","sqrt"],[y,p],t)}return je.makeSpan(["mord","sqrt"],[p],t)},mathmlBuilder:function(e,t){var r=e.body,n=e.index;return n?new Mt.MathNode("mroot",[Nt(r,t),Nt(n,t)]):new Mt.MathNode("msqrt",[Nt(r,t)])}});var gn={display:b.DISPLAY,text:b.TEXT,script:b.SCRIPT,scriptscript:b.SCRIPTSCRIPT};nt({type:"styling",names:["\\displaystyle","\\textstyle","\\scriptstyle","\\scriptscriptstyle"],props:{numArgs:0,allowedInText:!0,primitive:!0},handler:function(e,t){var r=e.breakOnTokenText,n=e.funcName,a=e.parser,i=a.parseExpression(!0,r),o=n.slice(1,n.length-5);return{type:"styling",mode:a.mode,style:o,body:i}},htmlBuilder:function(e,t){var r=gn[e.style],n=t.havingStyle(r).withFont("");return dn(e.body,n,t)},mathmlBuilder:function(e,t){var r=gn[e.style],n=t.havingStyle(r),a=Bt(e.body,n),i=new Mt.MathNode("mstyle",a),o={display:["0","true"],text:["0","false"],script:["1","false"],scriptscript:["2","false"]}[e.style];return i.setAttribute("scriptlevel",o[0]),i.setAttribute("displaystyle",o[1]),i}});var vn=function(e,t){var r=e.base;return r?"op"===r.type?r.limits&&(t.style.size===b.DISPLAY.size||r.alwaysHandleSupSub)?hn:null:"operatorname"===r.type?r.alwaysHandleSupSub&&(t.style.size===b.DISPLAY.size||r.limits)?pn:null:"accent"===r.type?l.isCharacterBox(r.base)?Ut:null:"horizBrace"===r.type&&!e.sub===r.isOver?nn:null:null};at({type:"supsub",htmlBuilder:function(e,t){var r=vn(e,t);if(r)return r(e,t);var n,a,i,o=e.base,s=e.sup,h=e.sub,m=bt(o,t),c=t.fontMetrics(),u=0,p=0,d=o&&l.isCharacterBox(o);if(s){var f=t.havingStyle(t.style.sup());n=bt(s,f,t),d||(u=m.height-f.fontMetrics().supDrop*f.sizeMultiplier/t.sizeMultiplier)}if(h){var g=t.havingStyle(t.style.sub());a=bt(h,g,t),d||(p=m.depth+g.fontMetrics().subDrop*g.sizeMultiplier/t.sizeMultiplier)}i=t.style===b.DISPLAY?c.sup1:t.style.cramped?c.sup3:c.sup2;var v,y=t.sizeMultiplier,x=.5/c.ptPerEm/y+"em",w=null;if(a){var k=e.base&&"op"===e.base.type&&e.base.name&&("\\oiint"===e.base.name||"\\oiiint"===e.base.name);(m instanceof O||k)&&(w=-m.italic+"em")}if(n&&a){u=Math.max(u,i,n.depth+.25*c.xHeight),p=Math.max(p,c.sub2);var S=4*c.defaultRuleThickness;if(u-n.depth-(a.height-p)0&&(u+=M,p-=M)}var z=[{type:"elem",elem:a,shift:p,marginRight:x,marginLeft:w},{type:"elem",elem:n,shift:-u,marginRight:x}];v=je.makeVList({positionType:"individualShift",children:z},t)}else if(a){p=Math.max(p,c.sub1,a.height-.8*c.xHeight);var A=[{type:"elem",elem:a,marginLeft:w,marginRight:x}];v=je.makeVList({positionType:"shift",positionData:p,children:A},t)}else{if(!n)throw new Error("supsub must have either sup or sub.");u=Math.max(u,i,n.depth+.25*c.xHeight),v=je.makeVList({positionType:"shift",positionData:-u,children:[{type:"elem",elem:n,marginRight:x}]},t)}var T=gt(m,"right")||"mord";return je.makeSpan([T],[m,je.makeSpan(["msupsub"],[v])],t)},mathmlBuilder:function(e,t){var r,n=!1;e.base&&"horizBrace"===e.base.type&&!!e.sup===e.base.isOver&&(n=!0,r=e.base.isOver),!e.base||"op"!==e.base.type&&"operatorname"!==e.base.type||(e.base.parentIsSupSub=!0);var a,i=[Nt(e.base,t)];if(e.sub&&i.push(Nt(e.sub,t)),e.sup&&i.push(Nt(e.sup,t)),n)a=r?"mover":"munder";else if(e.sub)if(e.sup){var o=e.base;a=o&&"op"===o.type&&o.limits&&t.style===b.DISPLAY||o&&"operatorname"===o.type&&o.alwaysHandleSupSub&&(t.style===b.DISPLAY||o.limits)?"munderover":"msubsup"}else{var s=e.base;a=s&&"op"===s.type&&s.limits&&(t.style===b.DISPLAY||s.alwaysHandleSupSub)||s&&"operatorname"===s.type&&s.alwaysHandleSupSub&&(s.limits||t.style===b.DISPLAY)?"munder":"msub"}else{var l=e.base;a=l&&"op"===l.type&&l.limits&&(t.style===b.DISPLAY||l.alwaysHandleSupSub)||l&&"operatorname"===l.type&&l.alwaysHandleSupSub&&(l.limits||t.style===b.DISPLAY)?"mover":"msup"}return new Mt.MathNode(a,i)}}),at({type:"atom",htmlBuilder:function(e,t){return je.mathsym(e.text,e.mode,t,["m"+e.family])},mathmlBuilder:function(e,t){var r=new Mt.MathNode("mo",[zt(e.text,e.mode)]);if("bin"===e.family){var n=Tt(e,t);"bold-italic"===n&&r.setAttribute("mathvariant",n)}else"punct"===e.family?r.setAttribute("separator","true"):"open"!==e.family&&"close"!==e.family||r.setAttribute("stretchy","false");return r}});var bn={mi:"italic",mn:"normal",mtext:"normal"};at({type:"mathord",htmlBuilder:function(e,t){return je.makeOrd(e,t,"mathord")},mathmlBuilder:function(e,t){var r=new Mt.MathNode("mi",[zt(e.text,e.mode,t)]),n=Tt(e,t)||"italic";return n!==bn[r.type]&&r.setAttribute("mathvariant",n),r}}),at({type:"textord",htmlBuilder:function(e,t){return je.makeOrd(e,t,"textord")},mathmlBuilder:function(e,t){var r,n=zt(e.text,e.mode,t),a=Tt(e,t)||"normal";return r="text"===e.mode?new Mt.MathNode("mtext",[n]):/[0-9]/.test(e.text)?new Mt.MathNode("mn",[n]):"\\prime"===e.text?new Mt.MathNode("mo",[n]):new Mt.MathNode("mi",[n]),a!==bn[r.type]&&r.setAttribute("mathvariant",a),r}});var yn={"\\nobreak":"nobreak","\\allowbreak":"allowbreak"},xn={" ":{},"\\ ":{},"~":{className:"nobreak"},"\\space":{},"\\nobreakspace":{className:"nobreak"}};at({type:"spacing",htmlBuilder:function(e,t){if(xn.hasOwnProperty(e.text)){var r=xn[e.text].className||"";if("text"===e.mode){var a=je.makeOrd(e,t,"textord");return a.classes.push(r),a}return je.makeSpan(["mspace",r],[je.mathsym(e.text,e.mode,t)],t)}if(yn.hasOwnProperty(e.text))return je.makeSpan(["mspace",yn[e.text]],[],t);throw new n('Unknown type of space "'+e.text+'"')},mathmlBuilder:function(e,t){if(!xn.hasOwnProperty(e.text)){if(yn.hasOwnProperty(e.text))return new Mt.MathNode("mspace");throw new n('Unknown type of space "'+e.text+'"')}return new Mt.MathNode("mtext",[new Mt.TextNode("\xa0")])}});var wn=function(){var e=new Mt.MathNode("mtd",[]);return e.setAttribute("width","50%"),e};at({type:"tag",mathmlBuilder:function(e,t){var r=new Mt.MathNode("mtable",[new Mt.MathNode("mtr",[wn(),new Mt.MathNode("mtd",[qt(e.body,t)]),wn(),new Mt.MathNode("mtd",[qt(e.tag,t)])])]);return r.setAttribute("width","100%"),r}});var kn={"\\text":void 0,"\\textrm":"textrm","\\textsf":"textsf","\\texttt":"texttt","\\textnormal":"textrm"},Sn={"\\textbf":"textbf","\\textmd":"textmd"},Mn={"\\textit":"textit","\\textup":"textup"},zn=function(e,t){var r=e.font;return r?kn[r]?t.withTextFontFamily(kn[r]):Sn[r]?t.withTextFontWeight(Sn[r]):t.withTextFontShape(Mn[r]):t};nt({type:"text",names:["\\text","\\textrm","\\textsf","\\texttt","\\textnormal","\\textbf","\\textmd","\\textit","\\textup"],props:{numArgs:1,argTypes:["text"],allowedInArgument:!0,allowedInText:!0},handler:function(e,t){var r=e.parser,n=e.funcName,a=t[0];return{type:"text",mode:r.mode,body:ot(a),font:n}},htmlBuilder:function(e,t){var r=zn(e,t),n=ut(e.body,r,!0);return je.makeSpan(["mord","text"],n,r)},mathmlBuilder:function(e,t){var r=zn(e,t);return qt(e.body,r)}}),nt({type:"underline",names:["\\underline"],props:{numArgs:1,allowedInText:!0},handler:function(e,t){return{type:"underline",mode:e.parser.mode,body:t[0]}},htmlBuilder:function(e,t){var r=bt(e.body,t),n=je.makeLineSpan("underline-line",t),a=t.fontMetrics().defaultRuleThickness,i=je.makeVList({positionType:"top",positionData:r.height,children:[{type:"kern",size:a},{type:"elem",elem:n},{type:"kern",size:3*a},{type:"elem",elem:r}]},t);return je.makeSpan(["mord","underline"],[i],t)},mathmlBuilder:function(e,t){var r=new Mt.MathNode("mo",[new Mt.TextNode("\u203e")]);r.setAttribute("stretchy","true");var n=new Mt.MathNode("munder",[Nt(e.body,t),r]);return n.setAttribute("accentunder","true"),n}}),nt({type:"vcenter",names:["\\vcenter"],props:{numArgs:1,argTypes:["original"],allowedInText:!1},handler:function(e,t){return{type:"vcenter",mode:e.parser.mode,body:t[0]}},htmlBuilder:function(e,t){var r=bt(e.body,t),n=t.fontMetrics().axisHeight,a=.5*(r.height-n-(r.depth+n));return je.makeVList({positionType:"shift",positionData:a,children:[{type:"elem",elem:r}]},t)},mathmlBuilder:function(e,t){return new Mt.MathNode("mpadded",[Nt(e.body,t)],["vcenter"])}}),nt({type:"verb",names:["\\verb"],props:{numArgs:0,allowedInText:!0},handler:function(e,t,r){throw new n("\\verb ended by end of line instead of matching delimiter")},htmlBuilder:function(e,t){for(var r=An(e),n=[],a=t.havingStyle(t.style.text()),i=0;i0&&(this.undefStack[this.undefStack.length-1][e]=t)}else{var a=this.undefStack[this.undefStack.length-1];a&&!a.hasOwnProperty(e)&&(a[e]=this.current[e])}this.current[e]=t},e}(),Rn={},En=Rn;function Hn(e,t){Rn[e]=t}Hn("\\noexpand",(function(e){var t=e.popToken();return e.isExpandable(t.text)&&(t.noexpand=!0,t.treatAsRelax=!0),{tokens:[t],numArgs:0}})),Hn("\\expandafter",(function(e){var t=e.popToken();return e.expandOnce(!0),{tokens:[t],numArgs:0}})),Hn("\\@firstoftwo",(function(e){return{tokens:e.consumeArgs(2)[0],numArgs:0}})),Hn("\\@secondoftwo",(function(e){return{tokens:e.consumeArgs(2)[1],numArgs:0}})),Hn("\\@ifnextchar",(function(e){var t=e.consumeArgs(3);e.consumeSpaces();var r=e.future();return 1===t[0].length&&t[0][0].text===r.text?{tokens:t[1],numArgs:0}:{tokens:t[2],numArgs:0}})),Hn("\\@ifstar","\\@ifnextchar *{\\@firstoftwo{#1}}"),Hn("\\TextOrMath",(function(e){var t=e.consumeArgs(2);return"text"===e.mode?{tokens:t[0],numArgs:0}:{tokens:t[1],numArgs:0}}));var Ln={0:0,1:1,2:2,3:3,4:4,5:5,6:6,7:7,8:8,9:9,a:10,A:10,b:11,B:11,c:12,C:12,d:13,D:13,e:14,E:14,f:15,F:15};Hn("\\char",(function(e){var t,r=e.popToken(),a="";if("'"===r.text)t=8,r=e.popToken();else if('"'===r.text)t=16,r=e.popToken();else if("`"===r.text)if("\\"===(r=e.popToken()).text[0])a=r.text.charCodeAt(1);else{if("EOF"===r.text)throw new n("\\char` missing argument");a=r.text.charCodeAt(0)}else t=10;if(t){if(null==(a=Ln[r.text])||a>=t)throw new n("Invalid base-"+t+" digit "+r.text);for(var i;null!=(i=Ln[e.future().text])&&i":"\\dotsb","-":"\\dotsb","*":"\\dotsb",":":"\\dotsb","\\DOTSB":"\\dotsb","\\coprod":"\\dotsb","\\bigvee":"\\dotsb","\\bigwedge":"\\dotsb","\\biguplus":"\\dotsb","\\bigcap":"\\dotsb","\\bigcup":"\\dotsb","\\prod":"\\dotsb","\\sum":"\\dotsb","\\bigotimes":"\\dotsb","\\bigoplus":"\\dotsb","\\bigodot":"\\dotsb","\\bigsqcup":"\\dotsb","\\And":"\\dotsb","\\longrightarrow":"\\dotsb","\\Longrightarrow":"\\dotsb","\\longleftarrow":"\\dotsb","\\Longleftarrow":"\\dotsb","\\longleftrightarrow":"\\dotsb","\\Longleftrightarrow":"\\dotsb","\\mapsto":"\\dotsb","\\longmapsto":"\\dotsb","\\hookrightarrow":"\\dotsb","\\doteq":"\\dotsb","\\mathbin":"\\dotsb","\\mathrel":"\\dotsb","\\relbar":"\\dotsb","\\Relbar":"\\dotsb","\\xrightarrow":"\\dotsb","\\xleftarrow":"\\dotsb","\\DOTSI":"\\dotsi","\\int":"\\dotsi","\\oint":"\\dotsi","\\iint":"\\dotsi","\\iiint":"\\dotsi","\\iiiint":"\\dotsi","\\idotsint":"\\dotsi","\\DOTSX":"\\dotsx"};Hn("\\dots",(function(e){var t="\\dotso",r=e.expandAfterFuture().text;return r in Pn?t=Pn[r]:("\\not"===r.substr(0,4)||r in X.math&&l.contains(["bin","rel"],X.math[r].group))&&(t="\\dotsb"),t}));var Fn={")":!0,"]":!0,"\\rbrack":!0,"\\}":!0,"\\rbrace":!0,"\\rangle":!0,"\\rceil":!0,"\\rfloor":!0,"\\rgroup":!0,"\\rmoustache":!0,"\\right":!0,"\\bigr":!0,"\\biggr":!0,"\\Bigr":!0,"\\Biggr":!0,$:!0,";":!0,".":!0,",":!0};Hn("\\dotso",(function(e){return e.future().text in Fn?"\\ldots\\,":"\\ldots"})),Hn("\\dotsc",(function(e){var t=e.future().text;return t in Fn&&","!==t?"\\ldots\\,":"\\ldots"})),Hn("\\cdots",(function(e){return e.future().text in Fn?"\\@cdots\\,":"\\@cdots"})),Hn("\\dotsb","\\cdots"),Hn("\\dotsm","\\cdots"),Hn("\\dotsi","\\!\\cdots"),Hn("\\dotsx","\\ldots\\,"),Hn("\\DOTSI","\\relax"),Hn("\\DOTSB","\\relax"),Hn("\\DOTSX","\\relax"),Hn("\\tmspace","\\TextOrMath{\\kern#1#3}{\\mskip#1#2}\\relax"),Hn("\\,","\\tmspace+{3mu}{.1667em}"),Hn("\\thinspace","\\,"),Hn("\\>","\\mskip{4mu}"),Hn("\\:","\\tmspace+{4mu}{.2222em}"),Hn("\\medspace","\\:"),Hn("\\;","\\tmspace+{5mu}{.2777em}"),Hn("\\thickspace","\\;"),Hn("\\!","\\tmspace-{3mu}{.1667em}"),Hn("\\negthinspace","\\!"),Hn("\\negmedspace","\\tmspace-{4mu}{.2222em}"),Hn("\\negthickspace","\\tmspace-{5mu}{.277em}"),Hn("\\enspace","\\kern.5em "),Hn("\\enskip","\\hskip.5em\\relax"),Hn("\\quad","\\hskip1em\\relax"),Hn("\\qquad","\\hskip2em\\relax"),Hn("\\tag","\\@ifstar\\tag@literal\\tag@paren"),Hn("\\tag@paren","\\tag@literal{({#1})}"),Hn("\\tag@literal",(function(e){if(e.macros.get("\\df@tag"))throw new n("Multiple \\tag");return"\\gdef\\df@tag{\\text{#1}}"})),Hn("\\bmod","\\mathchoice{\\mskip1mu}{\\mskip1mu}{\\mskip5mu}{\\mskip5mu}\\mathbin{\\rm mod}\\mathchoice{\\mskip1mu}{\\mskip1mu}{\\mskip5mu}{\\mskip5mu}"),Hn("\\pod","\\allowbreak\\mathchoice{\\mkern18mu}{\\mkern8mu}{\\mkern8mu}{\\mkern8mu}(#1)"),Hn("\\pmod","\\pod{{\\rm mod}\\mkern6mu#1}"),Hn("\\mod","\\allowbreak\\mathchoice{\\mkern18mu}{\\mkern12mu}{\\mkern12mu}{\\mkern12mu}{\\rm mod}\\,\\,#1"),Hn("\\pmb","\\html@mathml{\\@binrel{#1}{\\mathrlap{#1}\\kern0.5px#1}}{\\mathbf{#1}}"),Hn("\\newline","\\\\\\relax"),Hn("\\TeX","\\textrm{\\html@mathml{T\\kern-.1667em\\raisebox{-.5ex}{E}\\kern-.125emX}{TeX}}");var Vn=D["Main-Regular"]["T".charCodeAt(0)][1]-.7*D["Main-Regular"]["A".charCodeAt(0)][1]+"em";Hn("\\LaTeX","\\textrm{\\html@mathml{L\\kern-.36em\\raisebox{"+Vn+"}{\\scriptstyle A}\\kern-.15em\\TeX}{LaTeX}}"),Hn("\\KaTeX","\\textrm{\\html@mathml{K\\kern-.17em\\raisebox{"+Vn+"}{\\scriptstyle A}\\kern-.15em\\TeX}{KaTeX}}"),Hn("\\hspace","\\@ifstar\\@hspacer\\@hspace"),Hn("\\@hspace","\\hskip #1\\relax"),Hn("\\@hspacer","\\rule{0pt}{0pt}\\hskip #1\\relax"),Hn("\\ordinarycolon",":"),Hn("\\vcentcolon","\\mathrel{\\mathop\\ordinarycolon}"),Hn("\\dblcolon",'\\html@mathml{\\mathrel{\\vcentcolon\\mathrel{\\mkern-.9mu}\\vcentcolon}}{\\mathop{\\char"2237}}'),Hn("\\coloneqq",'\\html@mathml{\\mathrel{\\vcentcolon\\mathrel{\\mkern-1.2mu}=}}{\\mathop{\\char"2254}}'),Hn("\\Coloneqq",'\\html@mathml{\\mathrel{\\dblcolon\\mathrel{\\mkern-1.2mu}=}}{\\mathop{\\char"2237\\char"3d}}'),Hn("\\coloneq",'\\html@mathml{\\mathrel{\\vcentcolon\\mathrel{\\mkern-1.2mu}\\mathrel{-}}}{\\mathop{\\char"3a\\char"2212}}'),Hn("\\Coloneq",'\\html@mathml{\\mathrel{\\dblcolon\\mathrel{\\mkern-1.2mu}\\mathrel{-}}}{\\mathop{\\char"2237\\char"2212}}'),Hn("\\eqqcolon",'\\html@mathml{\\mathrel{=\\mathrel{\\mkern-1.2mu}\\vcentcolon}}{\\mathop{\\char"2255}}'),Hn("\\Eqqcolon",'\\html@mathml{\\mathrel{=\\mathrel{\\mkern-1.2mu}\\dblcolon}}{\\mathop{\\char"3d\\char"2237}}'),Hn("\\eqcolon",'\\html@mathml{\\mathrel{\\mathrel{-}\\mathrel{\\mkern-1.2mu}\\vcentcolon}}{\\mathop{\\char"2239}}'),Hn("\\Eqcolon",'\\html@mathml{\\mathrel{\\mathrel{-}\\mathrel{\\mkern-1.2mu}\\dblcolon}}{\\mathop{\\char"2212\\char"2237}}'),Hn("\\colonapprox",'\\html@mathml{\\mathrel{\\vcentcolon\\mathrel{\\mkern-1.2mu}\\approx}}{\\mathop{\\char"3a\\char"2248}}'),Hn("\\Colonapprox",'\\html@mathml{\\mathrel{\\dblcolon\\mathrel{\\mkern-1.2mu}\\approx}}{\\mathop{\\char"2237\\char"2248}}'),Hn("\\colonsim",'\\html@mathml{\\mathrel{\\vcentcolon\\mathrel{\\mkern-1.2mu}\\sim}}{\\mathop{\\char"3a\\char"223c}}'),Hn("\\Colonsim",'\\html@mathml{\\mathrel{\\dblcolon\\mathrel{\\mkern-1.2mu}\\sim}}{\\mathop{\\char"2237\\char"223c}}'),Hn("\u2237","\\dblcolon"),Hn("\u2239","\\eqcolon"),Hn("\u2254","\\coloneqq"),Hn("\u2255","\\eqqcolon"),Hn("\u2a74","\\Coloneqq"),Hn("\\ratio","\\vcentcolon"),Hn("\\coloncolon","\\dblcolon"),Hn("\\colonequals","\\coloneqq"),Hn("\\coloncolonequals","\\Coloneqq"),Hn("\\equalscolon","\\eqqcolon"),Hn("\\equalscoloncolon","\\Eqqcolon"),Hn("\\colonminus","\\coloneq"),Hn("\\coloncolonminus","\\Coloneq"),Hn("\\minuscolon","\\eqcolon"),Hn("\\minuscoloncolon","\\Eqcolon"),Hn("\\coloncolonapprox","\\Colonapprox"),Hn("\\coloncolonsim","\\Colonsim"),Hn("\\simcolon","\\mathrel{\\sim\\mathrel{\\mkern-1.2mu}\\vcentcolon}"),Hn("\\simcoloncolon","\\mathrel{\\sim\\mathrel{\\mkern-1.2mu}\\dblcolon}"),Hn("\\approxcolon","\\mathrel{\\approx\\mathrel{\\mkern-1.2mu}\\vcentcolon}"),Hn("\\approxcoloncolon","\\mathrel{\\approx\\mathrel{\\mkern-1.2mu}\\dblcolon}"),Hn("\\notni","\\html@mathml{\\not\\ni}{\\mathrel{\\char`\u220c}}"),Hn("\\limsup","\\DOTSB\\operatorname*{lim\\,sup}"),Hn("\\liminf","\\DOTSB\\operatorname*{lim\\,inf}"),Hn("\\injlim","\\DOTSB\\operatorname*{inj\\,lim}"),Hn("\\projlim","\\DOTSB\\operatorname*{proj\\,lim}"),Hn("\\varlimsup","\\DOTSB\\operatorname*{\\overline{lim}}"),Hn("\\varliminf","\\DOTSB\\operatorname*{\\underline{lim}}"),Hn("\\varinjlim","\\DOTSB\\operatorname*{\\underrightarrow{lim}}"),Hn("\\varprojlim","\\DOTSB\\operatorname*{\\underleftarrow{lim}}"),Hn("\\gvertneqq","\\html@mathml{\\@gvertneqq}{\u2269}"),Hn("\\lvertneqq","\\html@mathml{\\@lvertneqq}{\u2268}"),Hn("\\ngeqq","\\html@mathml{\\@ngeqq}{\u2271}"),Hn("\\ngeqslant","\\html@mathml{\\@ngeqslant}{\u2271}"),Hn("\\nleqq","\\html@mathml{\\@nleqq}{\u2270}"),Hn("\\nleqslant","\\html@mathml{\\@nleqslant}{\u2270}"),Hn("\\nshortmid","\\html@mathml{\\@nshortmid}{\u2224}"),Hn("\\nshortparallel","\\html@mathml{\\@nshortparallel}{\u2226}"),Hn("\\nsubseteqq","\\html@mathml{\\@nsubseteqq}{\u2288}"),Hn("\\nsupseteqq","\\html@mathml{\\@nsupseteqq}{\u2289}"),Hn("\\varsubsetneq","\\html@mathml{\\@varsubsetneq}{\u228a}"),Hn("\\varsubsetneqq","\\html@mathml{\\@varsubsetneqq}{\u2acb}"),Hn("\\varsupsetneq","\\html@mathml{\\@varsupsetneq}{\u228b}"),Hn("\\varsupsetneqq","\\html@mathml{\\@varsupsetneqq}{\u2acc}"),Hn("\\imath","\\html@mathml{\\@imath}{\u0131}"),Hn("\\jmath","\\html@mathml{\\@jmath}{\u0237}"),Hn("\\llbracket","\\html@mathml{\\mathopen{[\\mkern-3.2mu[}}{\\mathopen{\\char`\u27e6}}"),Hn("\\rrbracket","\\html@mathml{\\mathclose{]\\mkern-3.2mu]}}{\\mathclose{\\char`\u27e7}}"),Hn("\u27e6","\\llbracket"),Hn("\u27e7","\\rrbracket"),Hn("\\lBrace","\\html@mathml{\\mathopen{\\{\\mkern-3.2mu[}}{\\mathopen{\\char`\u2983}}"),Hn("\\rBrace","\\html@mathml{\\mathclose{]\\mkern-3.2mu\\}}}{\\mathclose{\\char`\u2984}}"),Hn("\u2983","\\lBrace"),Hn("\u2984","\\rBrace"),Hn("\\minuso","\\mathbin{\\html@mathml{{\\mathrlap{\\mathchoice{\\kern{0.145em}}{\\kern{0.145em}}{\\kern{0.1015em}}{\\kern{0.0725em}}\\circ}{-}}}{\\char`\u29b5}}"),Hn("\u29b5","\\minuso"),Hn("\\darr","\\downarrow"),Hn("\\dArr","\\Downarrow"),Hn("\\Darr","\\Downarrow"),Hn("\\lang","\\langle"),Hn("\\rang","\\rangle"),Hn("\\uarr","\\uparrow"),Hn("\\uArr","\\Uparrow"),Hn("\\Uarr","\\Uparrow"),Hn("\\N","\\mathbb{N}"),Hn("\\R","\\mathbb{R}"),Hn("\\Z","\\mathbb{Z}"),Hn("\\alef","\\aleph"),Hn("\\alefsym","\\aleph"),Hn("\\Alpha","\\mathrm{A}"),Hn("\\Beta","\\mathrm{B}"),Hn("\\bull","\\bullet"),Hn("\\Chi","\\mathrm{X}"),Hn("\\clubs","\\clubsuit"),Hn("\\cnums","\\mathbb{C}"),Hn("\\Complex","\\mathbb{C}"),Hn("\\Dagger","\\ddagger"),Hn("\\diamonds","\\diamondsuit"),Hn("\\empty","\\emptyset"),Hn("\\Epsilon","\\mathrm{E}"),Hn("\\Eta","\\mathrm{H}"),Hn("\\exist","\\exists"),Hn("\\harr","\\leftrightarrow"),Hn("\\hArr","\\Leftrightarrow"),Hn("\\Harr","\\Leftrightarrow"),Hn("\\hearts","\\heartsuit"),Hn("\\image","\\Im"),Hn("\\infin","\\infty"),Hn("\\Iota","\\mathrm{I}"),Hn("\\isin","\\in"),Hn("\\Kappa","\\mathrm{K}"),Hn("\\larr","\\leftarrow"),Hn("\\lArr","\\Leftarrow"),Hn("\\Larr","\\Leftarrow"),Hn("\\lrarr","\\leftrightarrow"),Hn("\\lrArr","\\Leftrightarrow"),Hn("\\Lrarr","\\Leftrightarrow"),Hn("\\Mu","\\mathrm{M}"),Hn("\\natnums","\\mathbb{N}"),Hn("\\Nu","\\mathrm{N}"),Hn("\\Omicron","\\mathrm{O}"),Hn("\\plusmn","\\pm"),Hn("\\rarr","\\rightarrow"),Hn("\\rArr","\\Rightarrow"),Hn("\\Rarr","\\Rightarrow"),Hn("\\real","\\Re"),Hn("\\reals","\\mathbb{R}"),Hn("\\Reals","\\mathbb{R}"),Hn("\\Rho","\\mathrm{P}"),Hn("\\sdot","\\cdot"),Hn("\\sect","\\S"),Hn("\\spades","\\spadesuit"),Hn("\\sub","\\subset"),Hn("\\sube","\\subseteq"),Hn("\\supe","\\supseteq"),Hn("\\Tau","\\mathrm{T}"),Hn("\\thetasym","\\vartheta"),Hn("\\weierp","\\wp"),Hn("\\Zeta","\\mathrm{Z}"),Hn("\\argmin","\\DOTSB\\operatorname*{arg\\,min}"),Hn("\\argmax","\\DOTSB\\operatorname*{arg\\,max}"),Hn("\\plim","\\DOTSB\\mathop{\\operatorname{plim}}\\limits"),Hn("\\bra","\\mathinner{\\langle{#1}|}"),Hn("\\ket","\\mathinner{|{#1}\\rangle}"),Hn("\\braket","\\mathinner{\\langle{#1}\\rangle}"),Hn("\\Bra","\\left\\langle#1\\right|"),Hn("\\Ket","\\left|#1\\right\\rangle"),Hn("\\angln","{\\angl n}"),Hn("\\blue","\\textcolor{##6495ed}{#1}"),Hn("\\orange","\\textcolor{##ffa500}{#1}"),Hn("\\pink","\\textcolor{##ff00af}{#1}"),Hn("\\red","\\textcolor{##df0030}{#1}"),Hn("\\green","\\textcolor{##28ae7b}{#1}"),Hn("\\gray","\\textcolor{gray}{#1}"),Hn("\\purple","\\textcolor{##9d38bd}{#1}"),Hn("\\blueA","\\textcolor{##ccfaff}{#1}"),Hn("\\blueB","\\textcolor{##80f6ff}{#1}"),Hn("\\blueC","\\textcolor{##63d9ea}{#1}"),Hn("\\blueD","\\textcolor{##11accd}{#1}"),Hn("\\blueE","\\textcolor{##0c7f99}{#1}"),Hn("\\tealA","\\textcolor{##94fff5}{#1}"),Hn("\\tealB","\\textcolor{##26edd5}{#1}"),Hn("\\tealC","\\textcolor{##01d1c1}{#1}"),Hn("\\tealD","\\textcolor{##01a995}{#1}"),Hn("\\tealE","\\textcolor{##208170}{#1}"),Hn("\\greenA","\\textcolor{##b6ffb0}{#1}"),Hn("\\greenB","\\textcolor{##8af281}{#1}"),Hn("\\greenC","\\textcolor{##74cf70}{#1}"),Hn("\\greenD","\\textcolor{##1fab54}{#1}"),Hn("\\greenE","\\textcolor{##0d923f}{#1}"),Hn("\\goldA","\\textcolor{##ffd0a9}{#1}"),Hn("\\goldB","\\textcolor{##ffbb71}{#1}"),Hn("\\goldC","\\textcolor{##ff9c39}{#1}"),Hn("\\goldD","\\textcolor{##e07d10}{#1}"),Hn("\\goldE","\\textcolor{##a75a05}{#1}"),Hn("\\redA","\\textcolor{##fca9a9}{#1}"),Hn("\\redB","\\textcolor{##ff8482}{#1}"),Hn("\\redC","\\textcolor{##f9685d}{#1}"),Hn("\\redD","\\textcolor{##e84d39}{#1}"),Hn("\\redE","\\textcolor{##bc2612}{#1}"),Hn("\\maroonA","\\textcolor{##ffbde0}{#1}"),Hn("\\maroonB","\\textcolor{##ff92c6}{#1}"),Hn("\\maroonC","\\textcolor{##ed5fa6}{#1}"),Hn("\\maroonD","\\textcolor{##ca337c}{#1}"),Hn("\\maroonE","\\textcolor{##9e034e}{#1}"),Hn("\\purpleA","\\textcolor{##ddd7ff}{#1}"),Hn("\\purpleB","\\textcolor{##c6b9fc}{#1}"),Hn("\\purpleC","\\textcolor{##aa87ff}{#1}"),Hn("\\purpleD","\\textcolor{##7854ab}{#1}"),Hn("\\purpleE","\\textcolor{##543b78}{#1}"),Hn("\\mintA","\\textcolor{##f5f9e8}{#1}"),Hn("\\mintB","\\textcolor{##edf2df}{#1}"),Hn("\\mintC","\\textcolor{##e0e5cc}{#1}"),Hn("\\grayA","\\textcolor{##f6f7f7}{#1}"),Hn("\\grayB","\\textcolor{##f0f1f2}{#1}"),Hn("\\grayC","\\textcolor{##e3e5e6}{#1}"),Hn("\\grayD","\\textcolor{##d6d8da}{#1}"),Hn("\\grayE","\\textcolor{##babec2}{#1}"),Hn("\\grayF","\\textcolor{##888d93}{#1}"),Hn("\\grayG","\\textcolor{##626569}{#1}"),Hn("\\grayH","\\textcolor{##3b3e40}{#1}"),Hn("\\grayI","\\textcolor{##21242c}{#1}"),Hn("\\kaBlue","\\textcolor{##314453}{#1}"),Hn("\\kaGreen","\\textcolor{##71B307}{#1}");var Gn={"\\relax":!0,"^":!0,_:!0,"\\limits":!0,"\\nolimits":!0},Un=function(){function e(e,t,r){this.settings=void 0,this.expansionCount=void 0,this.lexer=void 0,this.macros=void 0,this.stack=void 0,this.mode=void 0,this.settings=t,this.expansionCount=0,this.feed(e),this.macros=new On(En,t.macros),this.mode=r,this.stack=[]}var t=e.prototype;return t.feed=function(e){this.lexer=new In(e,this.settings)},t.switchMode=function(e){this.mode=e},t.beginGroup=function(){this.macros.beginGroup()},t.endGroup=function(){this.macros.endGroup()},t.future=function(){return 0===this.stack.length&&this.pushToken(this.lexer.lex()),this.stack[this.stack.length-1]},t.popToken=function(){return this.future(),this.stack.pop()},t.pushToken=function(e){this.stack.push(e)},t.pushTokens=function(e){var t;(t=this.stack).push.apply(t,e)},t.scanArgument=function(e){var t,r,n;if(e){if(this.consumeSpaces(),"["!==this.future().text)return null;t=this.popToken();var a=this.consumeArg(["]"]);n=a.tokens,r=a.end}else{var i=this.consumeArg();n=i.tokens,t=i.start,r=i.end}return this.pushToken(new qn("EOF",r.loc)),this.pushTokens(n),t.range(r,"")},t.consumeSpaces=function(){for(;;){if(" "!==this.future().text)break;this.stack.pop()}},t.consumeArg=function(e){var t=[],r=e&&e.length>0;r||this.consumeSpaces();var a,i=this.future(),o=0,s=0;do{if(a=this.popToken(),t.push(a),"{"===a.text)++o;else if("}"===a.text){if(-1===--o)throw new n("Extra }",a)}else if("EOF"===a.text)throw new n("Unexpected end of input in a macro argument, expected '"+(e&&r?e[s]:"}")+"'",a);if(e&&r)if((0===o||1===o&&"{"===e[s])&&a.text===e[s]){if(++s===e.length){t.splice(-s,s);break}}else s=0}while(0!==o||r);return"{"===i.text&&"}"===t[t.length-1].text&&(t.pop(),t.shift()),t.reverse(),{tokens:t,start:i,end:a}},t.consumeArgs=function(e,t){if(t){if(t.length!==e+1)throw new n("The length of delimiters doesn't match the number of args!");for(var r=t[0],a=0;athis.settings.maxExpand)throw new n("Too many expansions: infinite loop or need to increase maxExpand setting");var i=a.tokens,o=this.consumeArgs(a.numArgs,a.delimiters);if(a.numArgs)for(var s=(i=i.slice()).length-1;s>=0;--s){var l=i[s];if("#"===l.text){if(0===s)throw new n("Incomplete placeholder at end of macro body",l);if("#"===(l=i[--s]).text)i.splice(s+1,1);else{if(!/^[1-9]$/.test(l.text))throw new n("Not a valid argument number",l);var h;(h=i).splice.apply(h,[s,2].concat(o[+l.text-1]))}}}return this.pushTokens(i),i},t.expandAfterFuture=function(){return this.expandOnce(),this.future()},t.expandNextToken=function(){for(;;){var e=this.expandOnce();if(e instanceof qn){if("\\relax"!==e.text&&!e.treatAsRelax)return this.stack.pop();this.stack.pop()}}throw new Error},t.expandMacro=function(e){return this.macros.has(e)?this.expandTokens([new qn(e)]):void 0},t.expandTokens=function(e){var t=[],r=this.stack.length;for(this.pushTokens(e);this.stack.length>r;){var n=this.expandOnce(!0);n instanceof qn&&(n.treatAsRelax&&(n.noexpand=!1,n.treatAsRelax=!1),t.push(this.stack.pop()))}return t},t.expandMacroAsText=function(e){var t=this.expandMacro(e);return t?t.map((function(e){return e.text})).join(""):t},t._getExpansion=function(e){var t=this.macros.get(e);if(null==t)return t;var r="function"==typeof t?t(this):t;if("string"==typeof r){var n=0;if(-1!==r.indexOf("#"))for(var a=r.replace(/##/g,"");-1!==a.indexOf("#"+(n+1));)++n;for(var i=new In(r,this.settings),o=[],s=i.lex();"EOF"!==s.text;)o.push(s),s=i.lex();return o.reverse(),{tokens:o,numArgs:n}}return r},t.isDefined=function(e){return this.macros.has(e)||Tn.hasOwnProperty(e)||X.math.hasOwnProperty(e)||X.text.hasOwnProperty(e)||Gn.hasOwnProperty(e)},t.isExpandable=function(e){var t=this.macros.get(e);return null!=t?"string"==typeof t||"function"==typeof t||!t.unexpandable:Tn.hasOwnProperty(e)&&!Tn[e].primitive},e}(),Yn={"\u0301":{text:"\\'",math:"\\acute"},"\u0300":{text:"\\`",math:"\\grave"},"\u0308":{text:'\\"',math:"\\ddot"},"\u0303":{text:"\\~",math:"\\tilde"},"\u0304":{text:"\\=",math:"\\bar"},"\u0306":{text:"\\u",math:"\\breve"},"\u030c":{text:"\\v",math:"\\check"},"\u0302":{text:"\\^",math:"\\hat"},"\u0307":{text:"\\.",math:"\\dot"},"\u030a":{text:"\\r",math:"\\mathring"},"\u030b":{text:"\\H"}},Wn={"\xe1":"a\u0301","\xe0":"a\u0300","\xe4":"a\u0308","\u01df":"a\u0308\u0304","\xe3":"a\u0303","\u0101":"a\u0304","\u0103":"a\u0306","\u1eaf":"a\u0306\u0301","\u1eb1":"a\u0306\u0300","\u1eb5":"a\u0306\u0303","\u01ce":"a\u030c","\xe2":"a\u0302","\u1ea5":"a\u0302\u0301","\u1ea7":"a\u0302\u0300","\u1eab":"a\u0302\u0303","\u0227":"a\u0307","\u01e1":"a\u0307\u0304","\xe5":"a\u030a","\u01fb":"a\u030a\u0301","\u1e03":"b\u0307","\u0107":"c\u0301","\u010d":"c\u030c","\u0109":"c\u0302","\u010b":"c\u0307","\u010f":"d\u030c","\u1e0b":"d\u0307","\xe9":"e\u0301","\xe8":"e\u0300","\xeb":"e\u0308","\u1ebd":"e\u0303","\u0113":"e\u0304","\u1e17":"e\u0304\u0301","\u1e15":"e\u0304\u0300","\u0115":"e\u0306","\u011b":"e\u030c","\xea":"e\u0302","\u1ebf":"e\u0302\u0301","\u1ec1":"e\u0302\u0300","\u1ec5":"e\u0302\u0303","\u0117":"e\u0307","\u1e1f":"f\u0307","\u01f5":"g\u0301","\u1e21":"g\u0304","\u011f":"g\u0306","\u01e7":"g\u030c","\u011d":"g\u0302","\u0121":"g\u0307","\u1e27":"h\u0308","\u021f":"h\u030c","\u0125":"h\u0302","\u1e23":"h\u0307","\xed":"i\u0301","\xec":"i\u0300","\xef":"i\u0308","\u1e2f":"i\u0308\u0301","\u0129":"i\u0303","\u012b":"i\u0304","\u012d":"i\u0306","\u01d0":"i\u030c","\xee":"i\u0302","\u01f0":"j\u030c","\u0135":"j\u0302","\u1e31":"k\u0301","\u01e9":"k\u030c","\u013a":"l\u0301","\u013e":"l\u030c","\u1e3f":"m\u0301","\u1e41":"m\u0307","\u0144":"n\u0301","\u01f9":"n\u0300","\xf1":"n\u0303","\u0148":"n\u030c","\u1e45":"n\u0307","\xf3":"o\u0301","\xf2":"o\u0300","\xf6":"o\u0308","\u022b":"o\u0308\u0304","\xf5":"o\u0303","\u1e4d":"o\u0303\u0301","\u1e4f":"o\u0303\u0308","\u022d":"o\u0303\u0304","\u014d":"o\u0304","\u1e53":"o\u0304\u0301","\u1e51":"o\u0304\u0300","\u014f":"o\u0306","\u01d2":"o\u030c","\xf4":"o\u0302","\u1ed1":"o\u0302\u0301","\u1ed3":"o\u0302\u0300","\u1ed7":"o\u0302\u0303","\u022f":"o\u0307","\u0231":"o\u0307\u0304","\u0151":"o\u030b","\u1e55":"p\u0301","\u1e57":"p\u0307","\u0155":"r\u0301","\u0159":"r\u030c","\u1e59":"r\u0307","\u015b":"s\u0301","\u1e65":"s\u0301\u0307","\u0161":"s\u030c","\u1e67":"s\u030c\u0307","\u015d":"s\u0302","\u1e61":"s\u0307","\u1e97":"t\u0308","\u0165":"t\u030c","\u1e6b":"t\u0307","\xfa":"u\u0301","\xf9":"u\u0300","\xfc":"u\u0308","\u01d8":"u\u0308\u0301","\u01dc":"u\u0308\u0300","\u01d6":"u\u0308\u0304","\u01da":"u\u0308\u030c","\u0169":"u\u0303","\u1e79":"u\u0303\u0301","\u016b":"u\u0304","\u1e7b":"u\u0304\u0308","\u016d":"u\u0306","\u01d4":"u\u030c","\xfb":"u\u0302","\u016f":"u\u030a","\u0171":"u\u030b","\u1e7d":"v\u0303","\u1e83":"w\u0301","\u1e81":"w\u0300","\u1e85":"w\u0308","\u0175":"w\u0302","\u1e87":"w\u0307","\u1e98":"w\u030a","\u1e8d":"x\u0308","\u1e8b":"x\u0307","\xfd":"y\u0301","\u1ef3":"y\u0300","\xff":"y\u0308","\u1ef9":"y\u0303","\u0233":"y\u0304","\u0177":"y\u0302","\u1e8f":"y\u0307","\u1e99":"y\u030a","\u017a":"z\u0301","\u017e":"z\u030c","\u1e91":"z\u0302","\u017c":"z\u0307","\xc1":"A\u0301","\xc0":"A\u0300","\xc4":"A\u0308","\u01de":"A\u0308\u0304","\xc3":"A\u0303","\u0100":"A\u0304","\u0102":"A\u0306","\u1eae":"A\u0306\u0301","\u1eb0":"A\u0306\u0300","\u1eb4":"A\u0306\u0303","\u01cd":"A\u030c","\xc2":"A\u0302","\u1ea4":"A\u0302\u0301","\u1ea6":"A\u0302\u0300","\u1eaa":"A\u0302\u0303","\u0226":"A\u0307","\u01e0":"A\u0307\u0304","\xc5":"A\u030a","\u01fa":"A\u030a\u0301","\u1e02":"B\u0307","\u0106":"C\u0301","\u010c":"C\u030c","\u0108":"C\u0302","\u010a":"C\u0307","\u010e":"D\u030c","\u1e0a":"D\u0307","\xc9":"E\u0301","\xc8":"E\u0300","\xcb":"E\u0308","\u1ebc":"E\u0303","\u0112":"E\u0304","\u1e16":"E\u0304\u0301","\u1e14":"E\u0304\u0300","\u0114":"E\u0306","\u011a":"E\u030c","\xca":"E\u0302","\u1ebe":"E\u0302\u0301","\u1ec0":"E\u0302\u0300","\u1ec4":"E\u0302\u0303","\u0116":"E\u0307","\u1e1e":"F\u0307","\u01f4":"G\u0301","\u1e20":"G\u0304","\u011e":"G\u0306","\u01e6":"G\u030c","\u011c":"G\u0302","\u0120":"G\u0307","\u1e26":"H\u0308","\u021e":"H\u030c","\u0124":"H\u0302","\u1e22":"H\u0307","\xcd":"I\u0301","\xcc":"I\u0300","\xcf":"I\u0308","\u1e2e":"I\u0308\u0301","\u0128":"I\u0303","\u012a":"I\u0304","\u012c":"I\u0306","\u01cf":"I\u030c","\xce":"I\u0302","\u0130":"I\u0307","\u0134":"J\u0302","\u1e30":"K\u0301","\u01e8":"K\u030c","\u0139":"L\u0301","\u013d":"L\u030c","\u1e3e":"M\u0301","\u1e40":"M\u0307","\u0143":"N\u0301","\u01f8":"N\u0300","\xd1":"N\u0303","\u0147":"N\u030c","\u1e44":"N\u0307","\xd3":"O\u0301","\xd2":"O\u0300","\xd6":"O\u0308","\u022a":"O\u0308\u0304","\xd5":"O\u0303","\u1e4c":"O\u0303\u0301","\u1e4e":"O\u0303\u0308","\u022c":"O\u0303\u0304","\u014c":"O\u0304","\u1e52":"O\u0304\u0301","\u1e50":"O\u0304\u0300","\u014e":"O\u0306","\u01d1":"O\u030c","\xd4":"O\u0302","\u1ed0":"O\u0302\u0301","\u1ed2":"O\u0302\u0300","\u1ed6":"O\u0302\u0303","\u022e":"O\u0307","\u0230":"O\u0307\u0304","\u0150":"O\u030b","\u1e54":"P\u0301","\u1e56":"P\u0307","\u0154":"R\u0301","\u0158":"R\u030c","\u1e58":"R\u0307","\u015a":"S\u0301","\u1e64":"S\u0301\u0307","\u0160":"S\u030c","\u1e66":"S\u030c\u0307","\u015c":"S\u0302","\u1e60":"S\u0307","\u0164":"T\u030c","\u1e6a":"T\u0307","\xda":"U\u0301","\xd9":"U\u0300","\xdc":"U\u0308","\u01d7":"U\u0308\u0301","\u01db":"U\u0308\u0300","\u01d5":"U\u0308\u0304","\u01d9":"U\u0308\u030c","\u0168":"U\u0303","\u1e78":"U\u0303\u0301","\u016a":"U\u0304","\u1e7a":"U\u0304\u0308","\u016c":"U\u0306","\u01d3":"U\u030c","\xdb":"U\u0302","\u016e":"U\u030a","\u0170":"U\u030b","\u1e7c":"V\u0303","\u1e82":"W\u0301","\u1e80":"W\u0300","\u1e84":"W\u0308","\u0174":"W\u0302","\u1e86":"W\u0307","\u1e8c":"X\u0308","\u1e8a":"X\u0307","\xdd":"Y\u0301","\u1ef2":"Y\u0300","\u0178":"Y\u0308","\u1ef8":"Y\u0303","\u0232":"Y\u0304","\u0176":"Y\u0302","\u1e8e":"Y\u0307","\u0179":"Z\u0301","\u017d":"Z\u030c","\u1e90":"Z\u0302","\u017b":"Z\u0307","\u03ac":"\u03b1\u0301","\u1f70":"\u03b1\u0300","\u1fb1":"\u03b1\u0304","\u1fb0":"\u03b1\u0306","\u03ad":"\u03b5\u0301","\u1f72":"\u03b5\u0300","\u03ae":"\u03b7\u0301","\u1f74":"\u03b7\u0300","\u03af":"\u03b9\u0301","\u1f76":"\u03b9\u0300","\u03ca":"\u03b9\u0308","\u0390":"\u03b9\u0308\u0301","\u1fd2":"\u03b9\u0308\u0300","\u1fd1":"\u03b9\u0304","\u1fd0":"\u03b9\u0306","\u03cc":"\u03bf\u0301","\u1f78":"\u03bf\u0300","\u03cd":"\u03c5\u0301","\u1f7a":"\u03c5\u0300","\u03cb":"\u03c5\u0308","\u03b0":"\u03c5\u0308\u0301","\u1fe2":"\u03c5\u0308\u0300","\u1fe1":"\u03c5\u0304","\u1fe0":"\u03c5\u0306","\u03ce":"\u03c9\u0301","\u1f7c":"\u03c9\u0300","\u038e":"\u03a5\u0301","\u1fea":"\u03a5\u0300","\u03ab":"\u03a5\u0308","\u1fe9":"\u03a5\u0304","\u1fe8":"\u03a5\u0306","\u038f":"\u03a9\u0301","\u1ffa":"\u03a9\u0300"},Xn=function(){function e(e,t){this.mode=void 0,this.gullet=void 0,this.settings=void 0,this.leftrightDepth=void 0,this.nextToken=void 0,this.mode="math",this.gullet=new Un(e,t,this.mode),this.settings=t,this.leftrightDepth=0}var t=e.prototype;return t.expect=function(e,t){if(void 0===t&&(t=!0),this.fetch().text!==e)throw new n("Expected '"+e+"', got '"+this.fetch().text+"'",this.fetch());t&&this.consume()},t.consume=function(){this.nextToken=null},t.fetch=function(){return null==this.nextToken&&(this.nextToken=this.gullet.expandNextToken()),this.nextToken},t.switchMode=function(e){this.mode=e,this.gullet.switchMode(e)},t.parse=function(){this.settings.globalGroup||this.gullet.beginGroup(),this.settings.colorIsTextColor&&this.gullet.macros.set("\\color","\\textcolor");var e=this.parseExpression(!1);return this.expect("EOF"),this.settings.globalGroup||this.gullet.endGroup(),e},t.parseExpression=function(t,r){for(var n=[];;){"math"===this.mode&&this.consumeSpaces();var a=this.fetch();if(-1!==e.endOfExpression.indexOf(a.text))break;if(r&&a.text===r)break;if(t&&Tn[a.text]&&Tn[a.text].infix)break;var i=this.parseAtom(r);if(!i)break;"internal"!==i.type&&n.push(i)}return"text"===this.mode&&this.formLigatures(n),this.handleInfixNodes(n)},t.handleInfixNodes=function(e){for(var t,r=-1,a=0;a=0&&this.settings.reportNonstrict("unicodeTextInMathMode",'Latin-1/Unicode text character "'+t[0]+'" used in math mode',e);var s,l=X[this.mode][t].group,h=Bn.range(e);if(U.hasOwnProperty(l)){var m=l;s={type:"atom",mode:this.mode,family:m,loc:h,text:t}}else s={type:l,mode:this.mode,loc:h,text:t};i=s}else{if(!(t.charCodeAt(0)>=128))return null;this.settings.strict&&(w(t.charCodeAt(0))?"math"===this.mode&&this.settings.reportNonstrict("unicodeTextInMathMode",'Unicode text character "'+t[0]+'" used in math mode',e):this.settings.reportNonstrict("unknownSymbol",'Unrecognized Unicode character "'+t[0]+'" ('+t.charCodeAt(0)+")",e)),i={type:"textord",mode:"text",loc:Bn.range(e),text:t}}if(this.consume(),o)for(var c=0;c=12.0.0" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 00000000..c849f52b --- /dev/null +++ b/package.json @@ -0,0 +1,5 @@ +{ + "dependencies": { + "highlight.js": "^11.9.0" + } +} diff --git a/pages/01_why_Julia/index.html b/pages/01_why_Julia/index.html new file mode 100644 index 00000000..635281db --- /dev/null +++ b/pages/01_why_Julia/index.html @@ -0,0 +1,409 @@ + Why Julia?
Bayesian Stats
Julia Logo
Code on GitHub
  1. Speed
  2. Ease of Use
  3. Multiple Dispatch
    1. Example: Dogs and Cats
    2. Example: One-hot Vector
  4. Julia: the right approach
  5. Footnotes
  6. References

Why Julia?

Julia (Bezanson, Edelman, Karpinski & Shah, 2017) is a relatively new language, first released in 2012, aims to be both high-level and fast. Julia is a fast dynamic-typed language that just-in-time (JIT) compiles into native code using LLVM. It "runs like C but reads like Python", meaning that is blazing fast, easy to prototype and read/write code. It is multi-paradigm, combining features of imperative, functional, and object-oriented programming.

Why was Julia created? Definitely read this now impressively old post by Julia founders. Here is a clarifying quote:

We want the speed of C with the dynamism of Ruby. We want a language that's homoiconic, with true macros like Lisp, but with obvious, familiar mathematical notation like Matlab. We want something as usable for general programming as Python, as easy for statistics as R, as natural for string processing as Perl, as powerful for linear algebra as Matlab, as good at gluing programs together as the shell. Something that is dirt simple to learn, yet keeps the most serious hackers happy. We want it interactive and we want it compiled.

Why this needs to be an extra language? Why cannot Python (or R) be made that fast for instance? See the official compact answer to this in the Julia manual FAQ:

The basic issue is that there is nothing special about Julia's compiler: we use a commonplace compiler (LLVM) with no "secret sauce" that other language developers don't know about. Julia's performance advantage derives almost entirely from its front-end: its language semantics allow a well-written Julia program to give more opportunities to the compiler to generate efficient code and memory layouts. If you tried to compile Matlab or Python code to Julia, our compiler would be limited by the semantics of Matlab or Python to producing code no better than that of existing compilers for those languages (and probably worse).

Julia's advantage is that good performance is not limited to a small subset of "built-in" types and operations, and one can write high-level type-generic code that works on arbitrary user-defined types while remaining fast and memory-efficient. Types in languages like Python simply don't provide enough information to the compiler for similar capabilities, so as soon as you used those languages as a Julia front-end you would be stuck.

These are the "official" answers from the Julia community. Now let me share with you my opinion. From my point-of-view Julia has three main features that makes it a unique language to work with, specially in scientific computing:

  • Speed

  • Ease of Use

  • Multiple Dispatch

Now let's dive into each one of those three features.

Speed

Yes, Julia is fast. Very fast! It was made for speed from the drawing board. It bypass any sort of intermediate representation and translate code into machine native code using LLVM compiler. Comparing this with R, that uses either FORTRAN or C, or Python, that uses CPython; and you'll clearly see that Julia has a major speed advantage over other languages that are common in data science and statistics. Julia exposes the machine code to LLVM's compiler which in turn can optimize code as it wishes, like a good compiler such as LLVM excels in.

One notable example: NASA uses Julia to analyze the "Largest Batch of Earth-Sized Planets Ever Found". Also, you can find benchmarks for a range of common code patterns, such as function calls, string parsing, sorting, numerical loops, random number generation, recursion, and array operations using Julia and also several other languages such as C, Rust, Go, JavaScript, R, Python, Fortran and Java. The figure below was taken from Julia's website. As you can see Julia is indeed fast:

Common Benchmarks

Common Benchmarks

Let me demonstrate how fast Julia is. Here is a simple "groupby" operation using random stuff to emulate common data analysis "split-apply-combine" operations in three languages[1] :

  • Julia: using DataFrames.jl - 0.4ms

  • Python: using Pandas and NumPy - 1.76ms

  • R: using {dplyr} - 3.22ms

Here is Julia:

using Random
+using StatsBase
+using DataFrames
+using BenchmarkTools
+using Chain
+Random.seed!(123)
+
+n = 10_000
+
+df = DataFrame(
+    x=sample(["A", "B", "C", "D"], n, replace=true),
+    y=rand(n),
+    z=randn(n),
+)
+
+@btime @chain $df begin  # passing `df` as reference so the compiler cannot optimize
+    groupby(:x)
+    combine(:y => median, :z => mean)
+end

Here is Python:

import pandas as pd
+import numpy as np
+
+n = 10000
+
+df = pd.DataFrame({'x': np.random.choice(['A', 'B', 'C', 'D'], n, replace=True),
+                   'y': np.random.randn(n),
+                   'z': np.random.rand(n)})
+
+%timeit df.groupby('x').agg({'y': 'median', 'z': 'mean'})

Here is R:

library(dplyr)
+
+n <- 10e3
+df <- tibble(
+    x = sample(c("A", "B", "C", "D"), n, replace = TRUE),
+    y = runif(n),
+    z = rnorm(n)
+)
+
+bench::mark(
+    df %>%
+        group_by(x) %>%
+        summarize(
+            median(y),
+            mean(z)
+        )
+)

So clearly Julia is the winner here, being 4x faster than Python and almost 10x faster than R. Also note that Pandas (along with NumPy) and {dplyr} are all written in C or C++. Additionally, I didn't let Julia cheat by allowing the compiler optimize for df by passing a reference $df. So, I guess this is a fair comparison.

Ease of Use

What is most striking is that Julia can be as fast as C (and faster than Java in some applications) while having a very simple and intelligible syntax. This feature along with its speed is what Julia creators denote as "the two language problem" that Julia addresses. The "two language problem" is a very typical situation in scientific computing where a researcher or computer scientist devises an algorithm or a solution that he or she prototypes in an easy to code language (like Python) and, if it works, he or she would code in a fast language that is not easy to code (C or FORTRAN). Thus, we have two languages involved in the process of developing a new solution. One which is easy to prototype but is not suited for implementation (mostly due to being slow). And another one which is not so easy to code (and, consequently, not easy to prototype) but suited for implementation (mostly because it is fast). Julia comes to eliminate such situations by being the same language that you prototype (ease of use) and implement the solution (speed).

Also, Julia lets you use unicode characters as variables or parameters. This means no more using sigma or sigma_i, and instead just use σ or σᵢ as you would in mathematical notation. When you see code for an algorithm or for a mathematical equation you see a one-to-one relation to code and math. This is a powerful feature.

I think that the "two language problem" and the one-to-one code and math relation are best described by one of the creators of Julia, Alan Edelman, in a TED Talk (see the video below):

I will try to exemplify what would be the "two language problem" by showing you how I would code a simple Metropolis algorithm for a bivariate normal distribution. I would mostly prototype it in a dynamically-typed language such as R or Python. Then, deploy the algorithm using a fast but hard to code language such as C++. This is exactly what I'll do now. The algorithm will be coded in Julia, R, C++ and Stan. There are two caveats. First, I am coding the original 1950s Metropolis version, not the 1970s Metropolis-Hastings, which implies symmetrical proposal distributions just for the sake of the example. Second, the proposals are based on a uniform distribution on the current proposal values of the proposal values ± a certain width.

Let's start with Julia which uses the Distributions.jl package for its probabilistic distributions along with logpdf() defined methods for all of the distributions.

using Distributions
+function metropolis(S::Int64, width::Float64, ρ::Float64;
+                    μ_x::Float64=0.0, μ_y::Float64=0.0,
+                    σ_x::Float64=1.0, σ_y::Float64=1.0)
+    binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y]);
+    draws = Matrix{Float64}(undef, S, 2);
+    x = randn(); y = randn();
+    accepted = 0::Int64;
+    for s in 1:S
+        x_ = rand(Uniform(x - width, x + width));
+        y_ = rand(Uniform(y - width, y + width));
+        r = exp(logpdf(binormal, [x_, y_]) - logpdf(binormal, [x, y]));
+
+        if r > rand(Uniform())
+            x = x_;
+            y = y_;
+            accepted += 1;
+        end
+        @inbounds draws[s, :] = [x y];
+    end
+    println("Acceptance rate is $(accepted / S)")
+    return draws
+end

Now let's go to the R version (from now on no more fancy names like μ or σ 😭). Since this is a bivariate normal I am using the package {mnormt} which allows for very fast (FORTRAN code) computation of multivariate normal distributions' pdf and logpdf.

metropolis <- function(S, width,
+                       mu_X = 0, mu_Y = 0,
+                       sigma_X = 1, sigma_Y = 1,
+                       rho) {
+   Sigma <- diag(2)
+   Sigma[1, 2] <- rho
+   Sigma[2, 1] <- rho
+   draws <- matrix(nrow = S, ncol = 2)
+   x <- rnorm(1)
+   y <- rnorm(1)
+   accepted <- 0
+   for (s in 1:S) {
+      x_ <- runif(1, x - width, x + width)
+      y_ <- runif(1, y - width, y + width)
+      r <- exp(mnormt::dmnorm(c(x_, y_), mean = c(mu_X, mu_Y), varcov = Sigma, log = TRUE) -
+                        mnormt::dmnorm(c(x, y), mean = c(mu_X, mu_Y), varcov = Sigma, log = TRUE))
+      if (r > runif(1, 0, 1)) {
+        x <- x_
+        y <- y_
+        accepted <- accepted + 1
+      }
+      draws[s, 1] <- x
+      draws[s, 2] <- y
+   }
+   print(paste0("Acceptance rate is ", accepted / S))
+   return(draws)
+}

Now C++. Here I am using the Eigen library. Note that, since C++ is a very powerful language to be used as "close to the metal" as possible, I don't have any convenient predefined multivariate normal to use. So I will have to create this from zero[2].
Ok, be ready! This is a mouthful:

#include <Eigen/Eigen>
+#include <cmath>
+#include <iostream>
+#include <random>
+
+using std::cout;
+std::random_device rd{};
+std::mt19937 gen{rd()};
+
+// Random Number Generator Stuff
+double random_normal(double mean = 0, double std = 1) {
+  std::normal_distribution<double> d{mean, std};
+  return d(gen);
+};
+
+double random_unif(double min = 0, double max = 1) {
+  std::uniform_real_distribution<double> d{min, max};
+  return d(gen);
+};
+
+// Multivariate Normal
+struct Mvn {
+  Mvn(const Eigen::VectorXd &mu, const Eigen::MatrixXd &s)
+      : mean(mu), sigma(s) {}
+  double pdf(const Eigen::VectorXd &x) const;
+  double lpdf(const Eigen::VectorXd &x) const;
+  Eigen::VectorXd mean;
+  Eigen::MatrixXd sigma;
+};
+
+double Mvn::pdf(const Eigen::VectorXd &x) const {
+  double n = x.rows();
+  double sqrt2pi = std::sqrt(2 * M_PI);
+  double quadform = (x - mean).transpose() * sigma.inverse() * (x - mean);
+  double norm = std::pow(sqrt2pi, -n) * std::pow(sigma.determinant(), -0.5);
+
+  return norm * exp(-0.5 * quadform);
+}
+
+double Mvn::lpdf(const Eigen::VectorXd &x) const {
+  double n = x.rows();
+  double sqrt2pi = std::sqrt(2 * M_PI);
+  double quadform = (x - mean).transpose() * sigma.inverse() * (x - mean);
+  double norm = std::pow(sqrt2pi, -n) * std::pow(sigma.determinant(), -0.5);
+
+  return log(norm) + (-0.5 * quadform);
+}
+
+Eigen::MatrixXd metropolis(int S, double width, double mu_X = 0,
+                                 double mu_Y = 0, double sigma_X = 1,
+                                 double sigma_Y = 1, double rho = 0.8) {
+  Eigen::MatrixXd sigma(2, 2);
+  sigma << sigma_X, rho, rho, sigma_Y;
+  Eigen::VectorXd mean(2);
+  mean << mu_X, mu_Y;
+  Mvn binormal(mean, sigma);
+
+  Eigen::MatrixXd out(S, 2);
+  double x = random_normal();
+  double y = random_normal();
+  double accepted = 0;
+  for (size_t i = 0; i < S - 1; i++) {
+    double xmw = x - width;
+    double xpw = x + width;
+    double ymw = y - width;
+    double ypw = y + width;
+
+    double x_ = random_unif(xmw, xpw);
+    double y_ = random_unif(ymw, ypw);
+
+    double r = std::exp(binormal.lpdf(Eigen::Vector2d(x_, y_)) -
+                        binormal.lpdf(Eigen::Vector2d(x, y)));
+    if (r > random_unif()) {
+      x = x_;
+      y = y_;
+      accepted++;
+    }
+    out(i, 0) = x;
+    out(i, 1) = y;
+  }
+  cout << "Acceptance rate is " << accepted / S << '\n';
+
+  return out;
+}

note that the PDF for a multivariate normal is:

PDF(μ,Σ)=(2π)−k2det⁡(Σ)−12e−12(x−μ)TΣ−1(x−μ), \text{PDF}(\boldsymbol{\mu}, \boldsymbol{\Sigma}) = (2\pi)^{-{\frac{k}{2}}}\det({\boldsymbol{\Sigma}})^{-{\frac {1}{2}}}e^{-{\frac{1}{2}}(\mathbf{x}-{\boldsymbol{\mu}})^{T}{\boldsymbol{\Sigma }}^{-1}(\mathbf{x} -{\boldsymbol{\mu}})} ,

where μ\boldsymbol{\mu} is a vector of means, kk is the number of dimensions, Σ\boldsymbol{\Sigma} is a covariance matrix, det⁡\det is the determinant and x\mathbf{x} is a vector of values that the PDF is evaluated for.

SPOILER ALERT: Julia will beat this C++ Eigen implementation by being almost 100x faster. So I will try to help C++ beat Julia (😂) by making a bivariate normal class BiNormal in order to avoid the expensive operation of inverting a covariance matrix and computing determinants in every logpdf proposal evaluation. Also since we are not doing linear algebra computations I've removed Eigen and used C++ STL's <vector>:

#define M_PI 3.14159265358979323846 /* pi */
+
+// Bivariate Normal
+struct BiNormal {
+  BiNormal(const std::vector<double> &mu, const double &rho)
+      : mean(mu), rho(rho) {}
+  double pdf(const std::vector<double> &x) const;
+  double lpdf(const std::vector<double> &x) const;
+  std::vector<double> mean;
+  double rho;
+};
+
+double BiNormal::pdf(const std::vector<double> &x) const {
+  double x_ = x[0];
+  double y_ = x[1];
+  return std::exp(-((std::pow(x_, 2) - (2 * rho * x_ * y_) + std::pow(y_, 2)) /
+                    (2 * (1 - std::pow(rho, 2))))) /
+         (2 * M_PI * std::sqrt(1 - std::pow(rho, 2)));
+}
+
+double BiNormal::lpdf(const std::vector<double> &x) const {
+  double x_ = x[0];
+  double y_ = x[1];
+  return (-((std::pow(x_, 2) - (2 * rho * x_ * y_) + std::pow(y_, 2))) /
+          (2 * (1 - std::pow(rho, 2)))) -
+         std::log(2) - std::log(M_PI) - log(std::sqrt(1 - std::pow(rho, 2)));
+}

This means that I've simplified the PDF [3] from equation (1) into:

PDF(x,y)=12π1−ρ2σXσYe−x2σX2−2ρ−xyσXσY+y2σY22(1−ρ2). \text{PDF}(x, y)= \frac{1}{2 \pi \sqrt{1 - \rho^2 } \sigma_X \sigma_Y} e^{-\frac{\frac{x^{2}}{\sigma_{X}^{2}}-2 \rho-\frac{x y}{\sigma_{X} \sigma_{Y}}+\frac{y^{2}}{\sigma_{Y}^{2}}}{2\left(1-\rho^{2}\right)}} .

Since σX=σY=1\sigma_{X} = \sigma_{Y} = 1, equation (2) boils down to:

PDF(x,y)=12π1−ρ2e−x2−2ρ−xy+y22(1−ρ2). \text{PDF}(x, y)=\frac{1}{2 \pi \sqrt{1 - \rho^2 }} e^{-\frac{x^{2} -2 \rho-x y+y^{2}}{2\left(1-\rho^{2}\right)}} .

no more determinants or matrix inversions. Easy-peasy for C++.

Now let's go to the last, but not least: Stan is a probabilistic language for specifying probabilistic models (does the same as Turing.jl does) and comes also with a very fast C++-based MCMC sampler. Stan is a personal favorite of mine and I have a whole graduate course of Bayesian statistics using Stan. Here's the Stan implementation:

functions {
+    real binormal_lpdf(real [] xy, real mu_X, real mu_Y, real sigma_X, real sigma_Y, real rho) {
+    real beta = rho * sigma_Y / sigma_X; real sigma = sigma_Y * sqrt(1 - square(rho));
+    return normal_lpdf(xy[1] | mu_X, sigma_X) +
+           normal_lpdf(xy[2] | mu_Y + beta * (xy[1] - mu_X), sigma);
+  }
+
+  matrix metropolis_rng(int S, real width,
+                        real mu_X, real mu_Y,
+                        real sigma_X, real sigma_Y,
+                        real rho) {
+    matrix[S, 2] out; real x = normal_rng(0, 1); real y = normal_rng(0, 1); real accepted = 0;
+    for (s in 1:S) {
+      real xmw = x - width; real xpw = x + width; real ymw = y - width; real ypw = y + width;
+      real x_ = uniform_rng(xmw, xpw); real y_ = uniform_rng(ymw, ypw);
+      real r = exp(binormal_lpdf({x_, y_} | mu_X, mu_Y, sigma_X, sigma_Y, rho) -
+                            binormal_lpdf({x , y } | mu_X, mu_Y, sigma_X, sigma_Y, rho));
+      if (r > uniform_rng(0, 1)) {
+        x = x_; y = y_; accepted += 1;
+      }
+      out[s, 1] = x;  out[s, 2] = y;
+    }
+    print("Acceptance rate is ", accepted / S);
+    return out;
+  }
+}

Wow, that was lot... Not let's go to the results. I've benchmarked R and Stan code using {bench} and {rstan} packages, C++ using catch2, Julia using BenchmarkTools.jl. For all benchmarks the parameters were: S = 10_000 simulations, width = 2.75 and ρ = 0.8. From fastest to slowest:

  • Stan - 3.6ms

  • Julia - 6.3ms

  • C++ BiNormal - 17ms

  • C++ MvNormal - 592ms

  • R - 1.35s which means 1350ms

Conclusion: a naïve Julia implementation beats C++ (while also beating a C++ math-helped faster implementation using bivariate normal PDFs) and gets very close to Stan, a highly specialized probabilistic language that compiles and runs on C++ with lots of contributors, funding and development time invested.

Despite being blazing fast, Julia also codes very easily.
You can write and read code without much effort.

Multiple Dispatch

I think that this is the real game changer of Julia language: The ability to define function behavior across many combinations of argument types via multiple dispatch. Multiple dispatch is a feature that allows a function or method to be dynamically dispatched based on the run-time (dynamic) type or, in the more general case, some other attribute of more than one of its arguments. This is a generalization of single-dispatch polymorphism where a function or method call is dynamically dispatched based on the derived type of the object on which the method has been called. Multiple dispatch routes the dynamic dispatch to the implementing function or method using the combined characteristics of one or more arguments.

Most languages have single-dispatch polymorphism that rely on the first parameter of a method in order to determine which method should be called. But what Julia differs is that multiple parameters are taken into account. This enables multiple definitions of similar functions that have the same initial parameter. I think that this is best explained by one of the creators of Julia, Stefan Karpinski, at JuliaCon 2019 (see the video below):

Example: Dogs and Cats

I will reproduce Karpinski's example. In the talk, Karpinski designs a structure of classes which are very common in object-oriented programming (OOP). In Julia, we don't have classes but we have structures (struct) that are meant to be "structured data": they define the kind of information that is embedded in the structure, that is a set of fields (aka "properties" or "attributes" in other languages), and then individual instances (or "objects") can be produced each with its own specific values for the fields defined by the structure.

We create an abstract type called Pet. Then, we proceed by creating two derived struct from Pet that has one field name (a String). These derived struct are Dog and Cat. We also define some methods for what happens in an "encounter" by defining a generic function meets() and several specific methods of meets() that will be multiple dispatched by Julia in runtime to define the action that one type Pet takes when it meets another Pet:

abstract type Pet end
+struct Dog <: Pet
+    name::String
+end
+struct Cat <: Pet
+    name::String
+end
+
+function encounter(a::Pet, b::Pet)
+    verb = meets(a, b)
+    return println("$(a.name) meets $(b.name) and $verb")
+end
+
+meets(a::Dog, b::Dog) = "sniffs";
+meets(a::Dog, b::Cat) = "chases";
+meets(a::Cat, b::Dog) = "hisses";
+meets(a::Cat, b::Cat) = "slinks";

Let's see what happens when we instantiate objects from Dog and Cat and call encounter on them in Julia:

fido = Dog("Fido");
+rex = Dog("Rex");
+whiskers = Cat("Whiskers");
+spots = Cat("Spots");
+
+encounter(fido, rex)
+encounter(rex, whiskers)
+encounter(spots, fido)
+encounter(whiskers, spots)
Fido meets Rex and sniffs
+Rex meets Whiskers and chases
+Spots meets Fido and hisses
+Whiskers meets Spots and slinks
+

It works as expected. Now let's translate this to modern C++ as literally as possible. Let's define a class Pet with a member variable name – in C ++ we can do this. Then we define a base function meets(), a function encounter()for two objects of the type Pet, and finally, define derived classes Dogand Cat overload meets() for them:

#include <iostream>
+#include <string>
+
+using std::string;
+using std::cout;
+
+class Pet {
+    public:
+        string name;
+};
+
+string meets(Pet a, Pet b) { return "FALLBACK"; } // If we use `return meets(a, b)` doesn't work
+
+void encounter(Pet a, Pet b) {
+    string verb = meets(a, b);
+    cout << a.name << " meets "
+         << b. name << " and " << verb << '\n';
+}
+
+class Cat : public Pet {};
+class Dog : public Pet {};
+
+string meets(Dog a, Dog b) { return "sniffs"; }
+string meets(Dog a, Cat b) { return "chases"; }
+string meets(Cat a, Dog b) { return "hisses"; }
+string meets(Cat a, Cat b) { return "slinks"; }

Now we add a main() function to the C++ script:

int main() {
+    Dog fido;      fido.name     = "Fido";
+    Dog rex;       rex.name      = "Rex";
+    Cat whiskers;  whiskers.name = "Whiskers";
+    Cat spots;     spots.name    = "Spots";
+
+    encounter(fido, rex);
+    encounter(rex, whiskers);
+    encounter(spots, fido);
+    encounter(whiskers, spots);
+
+    return 0;
+}

And this is what we get:

g++ main.cpp && ./a.out
+
+Fido meets Rex and FALLBACK
+Rex meets Whiskers and FALLBACK
+Spots meets Fido and FALLBACK
+Whiskers meets Spots and FALLBACK

Doesn't work... 🤷🏼

Example: One-hot Vector

Now let's change to another nice example of creating a one-hot vector. One-hot vector is a vector of integers in which all indices are zero (0) except for one single index that is one (1). In machine learning, one-hot encoding is a frequently used method to deal with categorical data. Because many machine learning models need their input variables to be numeric, categorical variables need to be transformed in the pre-processing part. The example below is heavily inspired by a post from Vasily Pisarev[4].

How we would represent one-hot vectors in Julia? Simple: we create a new type OneHotVector in Julia using the struct keyword and define two fields len and ind, which represents the OneHotVector length and which index is the entry 1 (i.e. which index is "hot"). Then, we define new methods for the Base functions size() and getindex() for our newly defined OneHotVector.

import Base: size, getindex
+
+struct OneHotVector <: AbstractVector{Int}
+    len::Int
+    ind::Int
+end
+
+size(v::OneHotVector) = (v.len,)
+
+getindex(v::OneHotVector, i::Integer) = Int(i == v.ind)
getindex (generic function with 928 methods)
+

Since OneHotVector is a struct derived from AbstractVector we can use all of the methods previously defined for AbstractVector and it simply works right off the bat. Here we are constructing an Array with a list comprehension:

+
onehot = [OneHotVector(3, rand(1:3)) for _ in 1:4]
4-element Vector{OneHotVector}:
+ [1, 0, 0]
+ [1, 0, 0]
+ [1, 0, 0]
+ [0, 0, 1]
+

Now I define a new function inner_sum() that is basically a recursive dot product with a summation. Here A – this is something matrix-like (although I did not indicate the types, and you can guess something only by the name), and vs is a vector of some vector-like elements. The function proceeds by taking the dot product of the "matrix" with all vector-like elements of vs and returning the accumulated values. This is all given generic definition without specifying any types. Generic programming here consists in this very function call inner() in a loop.

+
using LinearAlgebra
+
+function inner_sum(A, vs)
+    t = zero(eltype(A))
+    for v in vs
+        t += inner(v, A, v) # multiple dispatch!
+    end
+    return t
+end
+
+inner(v, A, w) = dot(v, A * w) # very general definition
inner (generic function with 1 method)
+

So, "look mom, it works":

+
A = rand(3, 3)
+vs = [rand(3) for _ in 1:4]
+inner_sum(A, vs)
2.4426500326493357
+

Since OneHotVector is a subtype of AbstractVector:

+
supertype(OneHotVector)
AbstractVector{Int64} (alias for AbstractArray{Int64, 1})
+

We can use inner_sum and it will do what it is supposed to do:

+
inner_sum(A, onehot)
1.604358084381523
+

But this default implementation is slow:

+
using BenchmarkTools
+
+@btime inner_sum($A, $onehot);
  189.279 ns (4 allocations: 320 bytes)
+
+

We can greatly optimize this procedure. Now let's redefine matrix multiplication by OneHotVector with a simple column selection. We do this by defining a new method of the * function (multiplier function) of Base Julia. Additionally we also create a new optimized method of inner() for dealing with OneHotVector:

+
import Base: *
+
+*(A::AbstractMatrix, v::OneHotVector) = A[:, v.ind]
+inner(v::OneHotVector, A, w::OneHotVector) = A[v.ind, w.ind]
inner (generic function with 2 methods)
+

That's it! Simple, huh? Now let's benchmark:

+
@btime inner_sum($A, $onehot);
  4.648 ns (0 allocations: 0 bytes)
+
+

Huge gains of speed! 🚀

+

Julia: the right approach

+

Here are some more thoughts on why I believe Julia is the right approach to scientific computation.

+

Below is a very opinionated image that divides the scientific computing languages that we've spoken so far in a 2x2 diagram with two axes: Slow-Fast and Easy-Hard. I've put C++ and FORTRAN in the hard and fast quadrant. R and Python goes into the easy and slow quadrant. Julia is the only language in the easy and fast quadrant. I don't know any language that would want to be hard and slow, so this quadrant is empty.

+

Scientific Computing Language Comparisons

+

Scientific Computing Language Comparisons

+

What I want to say with this image is that if you want to code fast and easy use Julia.

+

One other thing to note that I find quite astonishing is that Julia packages are all written in Julia. This does not happen in other scientific computing languages. For example, the whole {tidyverse} ecosystem of R packages are based on C++. NumPy and SciPy are a mix of FORTRAN and C. Scikit-Learn is also coded in C.

+

See the figure below where I compare the GitHub's "Languages" stack bar of PyTorch, TensorFlow and Flux.jl(Julia's Deep Learning package). This figure I would call "Python my a**!" 😂:

+

Python my ass

+

Python my a**!

+
⚠ Note
+
On the other hand, language interoperability is extremely useful: we want to exploit existing high-quality code in other languages from Julia (and vice versa)! Julia community have worked hard on this, from the built-in intrinsic Julia ccall function (to call C and Fortran libraries) to JuliaInterop[5] packages that connect Julia to Python, R, Matlab, C++, and more.
+

Another example comes from a Julia podcast that unfortunately I cannot recollect either what podcast was nor who was being interviewed. While being asked about how he joined the Julia bandwagon, he replied something in the likes:

+
+

"Well, I was doing some crazy calculation using a library that was a wrapper to an algorithm coded in FORTRAN and I was trying to get help with a bug. I opened an issue and after 2 weeks of no reply I've dived into the FORTRAN code (despite having no knowledge of FORTRAN). There I saw a comment from the original author describing exactly the same bug that I was experiencing and saying that he would fix this in the future. The comment was dated from 1992. At the same time a colleague of mine suggested that I could try to code the algorithm in some new language called Julia. I thought 'me?! code an algorithm?!'. So, I coded the algorithm in Julia and it was faster than the FORTRAN implementation and also without the evil bug. One thing to note that it was really easy to code the algorithm in Julia."

+
+

Having stuff in different language and wrappers can hinder further research as you can see from this example.

+

As you saw from the Karpinski's talk above, multiple dispatch empower users to define their own types (if necessary) and also allows them to extend functions and types from other users to their own special use. This results in an ecosystem that stimulates code sharing and code reuse in scientific computing that is unmatched. For instance, if I plug a differential equation from DifferentialEquations.jl into a Turing.jl model I get a Bayesian stochastic differential equation model, e.g. Bayesian SIR model for infectious disease. If I plug a Flux.jl neural network into a Turing.jl model I get a Bayesian neural network! When I saw this type of code sharing I was blown away (and I still am).

+
⚠ Note
+
This is the true power of a scientific computing language like Julia. It brings so much power and flexibility to the user and allows different ways of sharing, contributing, extending, mixing and implementing code and science. I hope this short dive into Julia has somehow sent you towards Julia.
+

Footnotes

+

+ +
[1] + please note that I've used updated versions for all languages and packages as of April, 2021. DataFrames.jl version 1.0.1, Pandas version 1.2.4, NumPy version 1.20.2, {dplyr} version 1.0.5. We did not cover R's {data.table} here. Further benchmarking information is available for example here: Tabular data benchmarking + +
+ + +
[2] + which of course I did not. The Mvn class is inspired by Iason Sarantopoulos' implementation. + +
+ + +
[3] + you can find all the math here. + +
+ + +
[4] + the post in Russian, I've "Google Translated" it to English. + +
+ + +
[5] + Julia has a lot of interoperability between languages. Check out: PyCall.jl and JuliaPy for Python; RCall.jl for Java; Cxx.jl and CxxWrap.jl for C++; Clang.jl for libclang and C; ObjectiveC.jl for Objective-C; JavaCall.jl for Java; MATLAB.jl for MATLAB; MathLink.jl for Mathematica/Wolfram Engine; OctCall.jl for GNU Octave; and ZMQ.jl for ZeroMQ. + +
+

+

References

+

Bezanson, J., Edelman, A., Karpinski, S., & Shah, V. B. (2017). Julia: A fresh approach to numerical computing. SIAM Review, 59(1), 65–98.

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/02_bayes_stats/index.html b/pages/02_bayes_stats/index.html new file mode 100644 index 00000000..13d04cdf --- /dev/null +++ b/pages/02_bayes_stats/index.html @@ -0,0 +1,38 @@ + What is Bayesian Statistics?
Bayesian Stats
Julia Logo
Code on GitHub
  1. What is Probability?
    1. Mathematical Definition
    2. Conditional Probability
    3. Joint Probability
    4. Bayes' Theorem
    5. Discrete vs Continuous Parameters
  2. Bayesian Statistics
  3. Frequentist Statistics
    1. pp-values
    2. What the pp-value is not
    3. Confidence Intervals
  4. Bayesian Statistics vs Frequentist Statistics
  5. Advantages of Bayesian Statistics
  6. The beginning of the end of Frequentist Statistics
  7. Turing
  8. Footnotes
  9. References

What is Bayesian Statistics?

Bayesian statistics is an approach to inferential statistics based on Bayes' theorem, where available knowledge about parameters in a statistical model is updated with the information in observed data. The background knowledge is expressed as a prior distribution and combined with observational data in the form of a likelihood function to determine the posterior distribution. The posterior can also be used for making predictions about future events.

Bayesian statistics is a departure from classical inferential statistics that prohibits probability statements about parameters and is based on asymptotically sampling infinite samples from a theoretical population and finding parameter values that maximize the likelihood function. Mostly notorious is null-hypothesis significance testing (NHST) based on p-values. Bayesian statistics incorporate uncertainty (and prior knowledge) by allowing probability statements about parameters, and the process of parameter value inference is a direct result of the Bayes' theorem.

Bayesian statistics is revolutionizing all fields of evidence-based science[1] (van de Schoot et al., 2021). Dissatisfaction with traditional methods of statistical inference (frequentist statistics) and the advent of computers with exponential growth in computing power[2] provided a rise in Bayesian statistics because it is an approach aligned with the human intuition of uncertainty, robust to scientific malpractices, but computationally intensive.

But before we get into Bayesian statistics, we have to talk about probability: the engine of Bayesian inference.

What is Probability?

PROBABILITY DOES NOT EXIST!

de Finetti (1974)[3]

These are the first words in the preface to the famous book by Bruno de Finetti (figure below), one of the most important probability mathematician and philosopher. Yes, probability does not exist. Or rather, probability as a physical quantity, objective chance, does NOT exist. De Finetti showed that, in a precise sense, if we dispense with the question of objective chance nothing is lost. The mathematics of inductive reasoning remains exactly the same.

De Finetti

Bruno de Finetti

Consider tossing a weighted coin. The attempts are considered independent and, as a result, exhibit another important property: the order does not matter. To say that order does not matter is to say that if you take any finite sequence of heads and tails and exchange the results however you want, the resulting sequence will have the same probability. We say that this probability is invariant under permutations.

Or, to put it another way, the only thing that matters is the relative frequency. Result that have the same frequency of heads and tails consequently have the same probability. The frequency is considered a sufficient statistic. Saying that order doesn't matter or saying that the only thing that matters is frequency are two ways of saying exactly the same thing. This property is called exchangeability by de Finetti. And it is the most important property of probability that makes it possible for us to manipulate it mathematically (or philosophically) even if it does not exist as a physical "thing".

Still developing the argument:

"Probabilistic reasoning - always understood as subjective - stems merely from our being uncertain about something. It makes no difference whether the uncertainty relates to an unforeseeable future [4], or to an unnoticed past, or to a past doubtfully reported or forgotten [5]... The only relevant thing is uncertainty - the extent of our own knowledge and ignorance. The actual fact of whether or not the events considered are in some sense determined, or known by other people, and so on, is of no consequence." (de Finetti, 1974)

In conclusion: no matter what the probability is, you can use it anyway, even if it is an absolute frequency (ex: probability that I will ride my bike naked is ZERO because the probability that an event that never occurred will occur in the future it is ZERO) or a subjective guess (ex: maybe the probability is not ZERO, but 0.00000000000001; very unlikely, but not impossible).

Mathematical Definition

With the philosophical intuition of probability elaborated, we move on to mathematical intuitions. The probability of an event is a real number[6], ∈R\in \mathbb{R} between 0 and 1, where, roughly, 0 indicates the impossibility of the event and 1 indicates the certainty of the event. The greater the likelihood of an event, the more likely it is that the event will occur. A simple example is the tossing of a fair (impartial) coin. Since the coin is fair, both results ("heads" and "tails") are equally likely; the probability of "heads" is equal to the probability of "tails"; and since no other result is possible[7], the probability of "heads" or "tails" is 12\frac{1}{2} (which can also be written as 0.5 or 50%).

Regarding notation, we define AA as an event and P(A)P(A) as the probability of event AA, thus:

{P(A)∈R:0≤P(A)≤1}.\{P(A) \in \mathbb{R} : 0 \leq P(A) \leq 1 \}.

This means the "probability of the event to occur is the set of all real numbers between 0 and 1; including 0 and 1". In addition, we have three axioms[8], originated from Kolmogorov(1933) (figure below):

  1. Non-negativity: For all AA, P(A)≥0P(A) \geq 0. Every probability is positive (greater than or equal to zero), regardless of the event.

  2. Additivity: For two mutually exclusive AA and BB (cannot occur at the same time[9]): P(A)=1−P(B)P(A) = 1 - P(B) and P(B)=1−P(A)P(B) = 1 - P(A).

  3. Normalization: The probability of all possible events A1,A2,…A_1, A_2, \dots must add up to 1: ∑n∈NP(An)=1\sum_{n \in \mathbb{N}} P(A_n) = 1.

Andrey Nikolaevich Kolmogorov

Andrey Nikolaevich Kolmogorov

With these three simple (and intuitive) axioms, we are able to derive and construct all the mathematics of probability.

Conditional Probability

An important concept is the conditional probability that we can define as the "probability that one event will occur if another has occurred or not". The notation we use is P(A∣B) P(A \mid B), which reads as "the probability that we have observed AA given we have already observed BB".

A good example is the Texas Hold'em Poker game, where the player receives two cards and can use five "community cards" to set up his "hand". The probability that you are dealt a King (KK) is 452\frac{4}{52}:

P(K)=(452)=(113). P(K) = \left(\frac{4}{52}\right) = \left(\frac{1}{13}\right) .

And the probability of being dealt an Ace is also the same as (2), 452\frac{4}{52}:

P(A)=(452)=(113). P(A) = \left(\frac{4}{52}\right) = \left(\frac{1}{13}\right) .

However, the probability that you are dealt a King as a second card since you have been dealt an Ace as a first card is:

P(K∣A)=(451). P(K \mid A) = \left(\frac{4}{51}\right) .

Since we have one less card (52−1=5152 - 1 = 51) because you have been dealt already an Ace (thus AA has been observed), we have 4 Kings still in the deck, so the (4) is 451\frac{4}{51}.

Joint Probability

Conditional probability leads us to another important concept: joint probability. Joint probability is the "probability that two events will both occur". Continuing with our Poker example, the probability that you will receive two cards, Ace (AA) and a King (KK) as two starting cards is:

P(A,K)=P(A)⋅P(K∣A)=P(113)⋅P(451)=P(451⋅13)≈0.006. \begin{aligned} P(A,K) &= P(A) \cdot P(K \mid A) \\ &= P \left(\frac{1}{13}\right) \cdot P \left(\frac{4}{51}\right)\\ &= P \left(\frac{4}{51 \cdot 13}\right) \\ &\approx 0.006 . \end{aligned}

Note that P(A,K)=P(K,A)P(A,K) = P(K,A):

P(K,A)=P(K)⋅P(A∣K)=P(113)⋅P(451)=P(451⋅13)≈0.006. \begin{aligned} P(K,A) &= P(K) \cdot P(A \mid K) \\ &= P \left(\frac{1}{13}\right) \cdot P \left(\frac{4}{51}\right)\\ &= P \left(\frac{4}{51 \cdot 13}\right) \\ &\approx 0.006 . \end{aligned}

But this symmetry does not always exist (in fact it very rarely exists). The identity we have is as follows:

P(A)⋅P(K∣A)=P(K)⋅P(A∣K). P(A) \cdot P(K \mid A) = P(K) \cdot P(A \mid K) .

So this symmetry only exists when the baseline rates for conditional events are equal:

P(A)=P(K). P(A) = P(K).

Which is what happens in our example.

Conditional Probability is not "commutative"

P(A∣B)≠P(B∣A) P(A \mid B) \neq P(B \mid A)

Let's see a practical example. For example, I’m feeling good and start coughing in line at the supermarket. What do you think will happen? Everyone will think I have COVID, which is equivalent to thinking about P(cough∣covid)P(\text{cough} \mid \text{covid}). Seeing the most common symptoms of COVID, if you have COVID, the chance of coughing is very high. But we actually cough a lot more often than we have COVID – P(cough)≠P(COVID)P(\text{cough}) \neq P(\text{COVID}), so:

P(COVID∣cough)≠P(cough∣COVID). P(\text{COVID} \mid \text{cough}) \neq P(\text{cough} \mid \text{COVID}) .

Bayes' Theorem

This is the last concept of probability that we need to address before diving into Bayesian statistics, but it is the most important. Note that it is not a semantic coincidence that Bayesian statistics and Bayes' theorem have the same prefix.

Thomas Bayes (1701 - 1761, figure below) was an English Presbyterian statistician, philosopher and minister known for formulating a specific case of the theorem that bears his name: Bayes' theorem. Bayes never published what would become his most famous accomplishment; his notes were edited and published after his death by his friend Richard Price[10]. In his later years, Bayes was deeply interested in probability. Some speculate that he was motivated to refute David Hume's argument against belief in miracles based on evidence from the testimony in "An Inquiry Concerning Human Understanding".

Thomas Bayes

Thomas Bayes

Let's move on to Theorem. Remember that we have the following identity in probability:

P(A,B)=P(B,A)P(A)⋅P(B∣A)=P(B)⋅P(A∣B). \begin{aligned} P(A,B) &= P(B,A) \\ P(A) \cdot P(B \mid A) &= P(B) \cdot P(A \mid B) . \end{aligned}

Ok, now move P(B)P(B) in the right of (11) to the left as a division:

P(A)⋅P(B∣A)=P(B)undefinedthis goes to ←⋅P(A∣B)P(A)⋅P(B∣A)P(B)=P(A∣B)P(A∣B)=P(A)⋅P(B∣A)P(B). \begin{aligned} P(A) \cdot P(B \mid A) &= \overbrace{P(B)}^{\text{this goes to $\leftarrow$}} \cdot P(A \mid B) \\ &\\ \frac{P(A) \cdot P(B \mid A)}{P(B)} &= P(A \mid B) \\ P(A \mid B) &= \frac{P(A) \cdot P(B \mid A)}{P(B)}. \end{aligned}

And the final result is:

P(A∣B)=P(A)⋅P(B∣A)P(B). P(A \mid B) = \frac{P(A) \cdot P(B \mid A)}{P(B)}.

Bayesian statistics uses this theorem as inference engine of parameters of a model conditioned on observed data.

Discrete vs Continuous Parameters

Everything that has been exposed so far is based on the assumption that the parameters are discrete. This was done in order to provide a better intuition of what is probability. We do not always work with discrete parameters. The parameters can be continuous, for example: age, height, weight, etc. But don't despair, all the rules and axioms of probability are also valid for continuous parameters. The only thing we have to do is to exchange all the sums ∑\sum for integrals ∫\int. For example, the third axiom of Normalization for continuous random variables becomes:

∫x∈Xp(x)dx=1. \int_{x \in X} p(x) dx = 1 .

Bayesian Statistics

Now that you know what probability is and what Bayes' theorem is, I will propose the following model:

P(θ∣y)undefinedPosterior=P(y∣θ)undefinedLikelihood⋅P(θ)undefinedPriorP(y)undefinedNormalizing Constant, \underbrace{P(\theta \mid y)}_{\text{Posterior}} = \frac{\overbrace{P(y \mid \theta)}^{\text{Likelihood}} \cdot \overbrace{P(\theta)}^{\text{Prior}}}{\underbrace{P(y)}_{\text{Normalizing Constant}}} ,

where:

  • θ\theta – parameter(s) of interest;

  • yy – observed data;

  • Prior – previous probability of the parameter value(s)[11] θ\theta;

  • Likelihood – probability of the observed data yy conditioned on the parameter value(s) θ\theta;

  • Posterior – posterior probability of the parameter value(s) θ\theta after observing the data yy; and

  • Normalizing Constant – P(y)P(y) does not make intuitive sense. This probability is transformed and can be interpreted as something that exists only so that the result of P(y∣θ)P(θ)P(y \mid \theta) P(\theta) is somewhere between 0 and 1 – a valid probability by the axioms. We will talk more about this constant in 5. Markov Chain Monte Carlo (MCMC).

Bayesian statistics allow us to directly quantify the uncertainty related to the value of one or more parameters of our model conditioned to the observed data. This is the main feature of Bayesian statistics, for we are directly estimating P(θ∣y)P (\theta \mid y) using Bayes' theorem. The resulting estimate is totally intuitive: it simply quantifies the uncertainty we have about the value of one or more parameters conditioned on the data, the assumptions of our model (likelihood) and the previous probability(prior) we have about such values.

Frequentist Statistics

To contrast with Bayesian statistics, let's look at the frequentist statistics, also known as "classical statistics". And already take notice: it is not something intuitive like the Bayesian statistics.

For frequentist statistics, the researcher is prohibited from making probabilistic conjectures about parameters. Because they are not uncertain, quite the contrary they are determined quantities. The only issue is that we do not directly observe the parameters, but they are deterministic and do not allow any margin of uncertainty. Therefore, for the frequentist approach, parameters are unobserved amounts of interest in which we do not make probabilistic conjectures.

What, then, is uncertain in frequentist statistics? Short answer: the observed data. For the frequentist approach, the sample is uncertain. Thus, we can only make probabilistic conjectures about our sample. Therefore, the uncertainty is expressed in the probability that I will obtain data similar to those that I obtained if I sampled from a population of interest infinite samples of the same size as my sample[12]. Uncertainty is conditioned by a frequentist approach, in other words, uncertainty only exists only if I consider an infinite sampling process and extract a frequency from that process. The probability only exists if it represents a frequency. Frequentist statistics is based on an "infinite sampling process of a population that I have never seen", strange that it may sounds.

For the frequentist approach, there is no posterior or prior probability since both involve parameters, and we saw that this is a no-no on frequentist soil. Everything that is necessary for statistical inference is contained within likelihood[13].

In addition, for reasons of ease of computation, since most of these methods were invented in the first half of the 20th century (without any help of a computer), only the parameter value(s) that maximize(s) the likelihood function is(are) computed[14]. From this optimization process we extracted the mode of the likelihood function (i.e. maximum value). The maximum likelihood estimate (MLE) is(are) the parameter value(s) so that a NN-sized sample randomly sampled from a population (i.e. the data you've collected) is the most likely NN-sized sample from that population. All other potential samples that could be extracted from this population will have a worse estimate than the sample you actually have[15]. In other words, we are conditioning the parameter value(s) on the observed data from the assumption that we are sampling infinite NN-sized samples from a theoretical population and treating the parameter values as fixed and our sample as random (or uncertain).

The mode works perfectly in the fairytale world, which assumes that everything follows a normal distribution, in which the mode is equal to the median and to the mean – mean=median=mode\text{mean} = \text{median} = \text{mode}. There is only one problem, this assumption is rarely true (see figure below), especially when we are dealing with multiple parameters with complex relationships between them (complex models).

Assumptions vs Reality

Assumptions vs Reality. Figure by Katherine Hoffman. Authorized Reproduction

A brief sociological and computational explanation is worthwhile of why frequentist (classical) statistics prohibit probabilistic conjectures about parameters and we are only left with optimizing (finding the maximum value of a function) rather than approximating or estimating a complete likelihood density (in other words, "to pull up the whole file" of the likelihood verisimilitude instead of just the mode).

On the sociological side of things, science at the beginning of the 20th century assumed that it should be strictly objective and all subjectivity must be banned. Therefore, since the estimation of the a posterior probability of parameters involves elucidating an a prior probability of parameters, such a method should not be allowed in science, as it brings subjectivity (we know today that nothing in human behavior is purely objective, and subjectivity permeates all human endeavors).

Regarding the computational side of things, in the 1930s without computers it was much easier to use strong assumptions about the data to get a value from a statistical estimation using mathematical derivations than to calculate the statistical estimation by hand without depending on such assumptions. For example: Student's famous tt test is a test that indicates when we can reject that the mean of a certain parameter of interest between two groups is equal (famous null hypothesis - H0H_0). This test starts from the assumption that if the parameter of interest is distributed according to a normal distribution (assumption 1 – normality of the dependent variable), if the variance of the parameter of interest varies homogeneously between groups (assumption 2 – homogeneity of the variances), and if the number of observations in the two groups are similar (assumption 3 – homogeneity of the size of the groups) the difference between the groups weighted by the variance of the groups follows a Student-tt distribution (hence the name of the test).

So statistical estimation comes down to calculating the average of two groups, the variance of both groups for a parameter of interest and looking for the associated pp-value in a table and see if we can reject the H0H_0. This was valid when everything we had to do was calculated by hand. Today, with a computer 1 million times more powerful than the Apollo 11 computer (one that took humanity to the moon) in your pocket [2], I don't know if it is still valid.

pp-values

pp-values are hard to understand, p<0.05p < 0.05.

p-values are hard to understand

Since I've mentioned the pp word, let me explain what it is. First, the correct[16] statistics textbook definition:

pp-value is the probability of obtaining test results at least as extreme as the results actually observed, under the assumption that the null hypothesis is correct.

Unfortunately with frequentist statistics you have to choose one of two qualities for explanations: intuitive or accurate[17].

If you write this definition in any test, book or scientific paper, you are 100% accurate and correct in defining what a pp-value is. Now, understanding this definition is complicated. For that, let's break this definition down into parts for a better understanding:

  • "probability of obtaining test results..": notice that pp-values are related your data and not your theory or hypothesis.

  • "...at least as extreme as the results actually observed...": "at least as extreme" implies defining a threshold for the characterization of some relevant finding, which is commonly called α\alpha. We generally stipulate alpha at 5% (α=0.05\alpha = 0.05) and anything more extreme than alpha (ie less than 5%) we characterize as significant.

  • "... under the assumption that the null hypothesis is correct.": every statistical test that has a pp-value has a null hypothesis (usually written as H0H_0). Null hypotheses, always have to do with some null effect. For example, the null hypothesis of the Shapiro-Wilk and Komolgorov-Smirnov test is "the data is distributed according to a Normal distribution" and that of the Levene test is "the group variances are equal". Whenever you see a pp-value, ask yourself: "What is the null hypothesis that this test assumes is correct?".

To understand the pp-value any statistical test first find out what is the null hypothesis behind that test. The definition of pp-value will not change. In every test it is always the same. What changes with the test is the null hypothesis. Each test has its H0H_0. For example, some common statistical tests (D∗\text{D}^* = data at least as extreme as the results actually observed):

  • tt Test: P(D∗∣the difference between groups are zero)P(\text{D}^* \mid \text{the difference between groups are zero})

  • ANOVA: P(D∗∣there is no difference between groups)P(\text{D}^* \mid \text{there is no difference between groups})

  • Regression: P(D∗∣the coefficient is zero)P(\text{D}^* \mid \text{the coefficient is zero})

  • Shapiro-Wilk: P(D∗∣the sample follows a normal distribution)P(\text{D}^* \mid \text{the sample follows a normal distribution})

pp-value is the probability of the data you obtained given that the null hypothesis is true. For those who like mathematical formalism: p=P(D∗∣H0)p = P(\text{D}^* \mid H_0). In English, this expression means "the probability of D∗\text{D}^* conditioned to H0H_0". Before moving on to some examples and attempts to formalize an intuition about pp-values, it is important to note that pp-values say something about data and not hypotheses. For pp-value, the null hypothesis is true, and we are only evaluating whether the data conforms to this null hypothesis or not. If you leave this tutorial armed with this intuition, the world will be rewarded with researchers better prepared to qualify and interpret evidence (p<0.05p <0.05).

⚠ Note

Intuitive Example of a pp-value:

Imagine that you have a coin that you suspect is biased towards a higher probability of flipping "heads" than "tails". (Your null hypothesis is then that the coin is fair.) You flip the coin 100 times and get more heads than tails. The pp-value will not tell you if the coin is fair, but it will tell you the probability that you will get at least as many heads as if the coin were fair. That's it - nothing more.

pp-values – A historical perspective

There is no way to understand pp-values ​​if we do not understand its origins and historical trajectory. The first mention of the term was made by statistician Ronald Fisher[18] (figure below). In 1925, Fisher (Fisher, 1925) defined the pp-value as an "index that measures the strength of the evidence against the null hypothesis". To quantify the strength of the evidence against the null hypothesis, Fisher defended "p<0.05p <0.05 (5% significance) as a standard level to conclude that there is evidence against the tested hypothesis, although not as an absolute rule". Fisher did not stop there but rated the strength of the evidence against the null hypothesis. He proposed "if pp is between 0.1 and 0.9 there is certainly no reason to suspect the hypothesis tested. If it is below 0.02 it is strongly indicated that the hypothesis fails to account for the whole of the facts. We shall not often be astray if we draw a conventional line at 0.05". Since Fisher made this statement almost 100 years ago, the 0.05 threshold has been used by researchers and scientists worldwide and it has become ritualistic to use 0.05 as a threshold as if other thresholds could not be used or even considered.

Ronald Fisher

Ronald Fisher

After that, the threshold of 0.05, now established as unquestionable, strongly influenced statistics and science. But there is no reason against adopting other thresholds (α\alpha) like 0.1 or 0.01 (Lakens et al., 2018). If well argued, the choice of thresholds other than 0.05 can be welcomed by editors, reviewers and advisors. Since the pp-value is a probability, it is a continuous quantity. There is no reason to differentiate a pp of 0.049 against a pp of 0.051. Robert Rosenthal, a psychologist already said "God loves pp 0.06 as much as a pp 0.05" (Rosnow & Rosenthal, 1989).

In the last year of his life, Fisher published an article (Fisher, 1962) examining the possibilities of Bayesian methods, but with a prior probabilities to be determined experimentally. Some authors even speculate (Jaynes, 2003) that if Fisher were alive today, he would probably be a "Bayesian".

What the pp-value is not

meme-pvalue

With the definition and a well-anchored intuition of what pp-value is, we can move on to what the pp-value is not!

  1. pp-value is not the probability of the null hypothesis - Famous confusion between P(D∣H0)P(D \mid H_0) and P(H0∣D)P(H_0 \mid D). pp-value is not the probability of the null hypothesis, but the probability of the data you obtained. To get the P(H0∣D)P(H_0 \mid D) you need Bayesian statistics.

  2. pp-value is not the probability of the data being produced by chance - No! Nobody said anything of "being produced by chance". Again: pp-value is the probability to get results at least as extreme as those that were observed, given that the null hypothesis is true.

  3. pp-value measures the size of the effect of a statistical test - Also wrong... pp-value does not say anything about the size of the effect. Just about whether the observed data differs from what is expected under the null hypothesis. It is clear that large effects are more likely to be statistically significant than small effects. But this is not a rule and never judge a finding by its pp-value, but by its effect size. In addition, pp-values ​​can be "hacked" in a number of ways (Head et al., 2015) and often their value is a direct consequence of the sample size.

Confidence Intervals

To conclude, let's talk about the famous confidence intervals, which are not a measure that quantifies the uncertainty of the value of a parameter (remember probabilistic conjectures about parameters are prohibited in frequentist-land). Wait for it! Here is the definition of confidence intervals:

"An X% confidence interval for a parameter θ\theta is an interval (L,U)(L, U) generated by a procedure that in repeated sampling has an X% probability of containing the true value of θ\theta, for all possible values of θ\theta."

Jerzy Neyman, the "father" of confidence intervals (see figure below) (Neyman, 1937).

Jerzy Neyman

Jerzy Neyman

Again the idea of sampling an infinite number of times from a population you have never seen. For example: let's say that you performed a statistical analysis to compare the effectiveness of a public policy in two groups and you obtained the difference between the average of those groups. You can express this difference as a confidence interval. We generally choose 95% confidence (since it is analogous as p<0.05p < 0.05). You then write in your paper that the "observed difference between groups is 10.5 - 23.5 (95% CI)." This means that approximately 95 studies out of 100 would compute a confidence interval that contains the true mean difference –- but it says nothing about which ones those are (whereas the data might). In other words, 95% is not the probability of obtaining data such that the estimate of the true parameter is contained in the interval that we obtained, it is the probability of obtaining data such that, if we compute another confidence interval in the same way, it contains the true parameter. The interval that we got in this particular instance is irrelevant and might as well be thrown away.

Confidence Intervals (Frequentist) vs Credible Intervals (Bayesian)

Bayesian statistics have a concept similar to the confidence intervals of frequentist statistics. This concept is called credibility interval. And, unlike the confidence interval, its definition is intuitive. Credibility interval measures an interval in which we are sure that the value of the parameter of interest is, based on the likelihood conditioned on the observed data - P(y∣θ)P(y \mid \theta); and the prior probability of the parameter of interest - P(θ)P(\theta). It is basically a "slice" of the posterior probability of the parameter restricted to a certain level of certainty. For example: a 95% credibility interval shows the interval that we are 95% sure that captures the value of our parameter of interest. That simple...

For example, see figure below, which shows a Log-Normal distribution with mean 0 and standard deviation 2. The green dot shows the maximum likelihood estimation (MLE) of the value of θ\theta which is simply the mode of distribution. And in the shaded area we have the 50% credibility interval of the value of θ\theta, which is the interval between the 25% percentile and the 75% percentile of the probability density of θ\theta. In this example, MLE leads to estimated values that are not consistent with the actual probability density of the value of θ\theta.

using CairoMakie
+using Distributions
+
+d = LogNormal(0, 2)
+range_d = 0:0.001:4
+q25 = quantile(d, 0.25)
+q75 = quantile(d, 0.75)
+credint = range(q25; stop=q75, length=100)
+f, ax, l = lines(
+    range_d,
+    pdf.(d, range_d);
+    linewidth=3,
+    axis=(; limits=(-0.2, 4.2, nothing, nothing), xlabel=L"\theta", ylabel="Density"),
+)
+scatter!(ax, mode(d), pdf(d, mode(d)); color=:green, markersize=12)
+band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5))

Log-Normal: Maximum Likelihood Estimate vs Credible Intervals

Now an example of a multimodal distribution[19]. The figure below shows a bimodal distribution with two modes 2 and 10[20] The green dot shows the maximum likelihood estimation (MLE) of the value of θ\theta which is the mode of distribution. See that even with 2 modes, maximum likelihood defaults to the highest mode[21]. And in the shaded area we have the 50% credibility interval of the value of θ\theta, which is the interval between the 25% percentile and the 75% percentile of the probability density of θ\theta. In this example, estimation by maximum likelihood again lead us to estimated values ​​that are not consistent with the actual probability density of the value of θ\theta.

d1 = Normal(10, 1)
+d2 = Normal(2, 1)
+mix_d = [0.4, 0.6]
+d = MixtureModel([d1, d2], mix_d)
+range_d = -2:0.01:14
+sim_d = rand(d, 10_000)
+q25 = quantile(sim_d, 0.25)
+q75 = quantile(sim_d, 0.75)
+credint = range(q25; stop=q75, length=100)
+
+f, ax, l = lines(
+    range_d,
+    pdf.(d, range_d);
+    linewidth=3,
+    axis=(;
+        limits=(-2, 14, nothing, nothing),
+        xticks=[0, 5, 10],
+        xlabel=L"\theta",
+        ylabel="Density",
+    ),
+)
+scatter!(ax, mode(d2), pdf(d, mode(d2)); color=:green, markersize=12)
+band!(ax, credint, 0.0, pdf.(d, credint); color=(:steelblue, 0.5))

Mixture: Maximum Likelihood Estimate vs Credible Intervals

Bayesian Statistics vs Frequentist Statistics

What we've seen so fat can be resumed in the table below:

Bayesian StatisticsFrequentist Statistics
DataFixed – Non-randomUncertain – Random
ParametersUncertain – RandomFixed – Non-random
InferenceUncertainty over parameter valuesUncertainty over a sampling procedure from an infinite population
ProbabilitySubjectiveObjective (but with strong model assumptions)
UncertaintyCredible Interval – P(θ∣y)P(\theta \mid y)Confidence Interval – P(y∣θ)P(y \mid \theta)

Advantages of Bayesian Statistics

Finally, I summarize the main advantages of Bayesian statistics:

  • Natural approach to express uncertainty

  • Ability to incorporate prior information

  • Greater model flexibility

  • Complete posterior distribution of parameters

    • Confidence Intervals vs Credibility Intervals

  • Natural propagation of uncertainty

And I believe that I also need to show the main disadvantage:

  • Slow model estimation speed (30 seconds instead of 3 seconds using the frequentist approach)

The beginning of the end of Frequentist Statistics

Götterdämmerung

Dear reader, know that you are at a time in history when Statistics is undergoing major changes. I believe that frequentist statistics, especially the way we qualify evidence and hypotheses with pp-values, will transform in a "significant" way. Five years ago, the American Statistical Association (ASA, the world's largest professional statistical organization) published a statement on pp-values (Wasserstein & Lazar, 2016). The statement says exactly what we talk about here. The main concepts of the null hypothesis significance test, and in particular pp-values, fail to provide what researchers require of them. Despite what many statistical books, teaching materials and published articles say, pp-values ​​below 0.05 do not "prove" the reality of anything. Nor, at this point, do the pp-values ​​above 0.05 refute anything. ASA's statement has more than 3,600 citations causing significant impact. As an example, an international symposium was held in 2017 that led to a special open access edition of The American Statistician dedicated to practical ways to abandon p<0.05p <0.05 (Wasserstein, Schirm & Lazar 2019).

Soon after, more attempts and claims followed. In September 2017, Nature Human Behavior published an editorial proposing that the significance level of the pp-value be reduced from 0.050.05 to 0.0050.005 (Benjamin et al., 2018). Several authors, including many highly influential and important statisticians, have argued that this simple step would help tackle the problem of the science replicability crisis, which many believe is the main consequence of the abusive use of pp-values (Ioannidis, 2019). In addition, many have gone a step further and suggest that science discard once and for all pp-values (Nature, 2019). Many suggest (myself included) that the main inference tool be Bayesian statistics (Amrhein, Greenland & McShane, 2019; Goodman, 2016; van de Schoot et al., 2021)

Turing

Turing (Ge, Xu & Ghahramani, 2018) is a probabilistic programming interface written in Julia (Bezanson, Edelman, Karpinski & Shah, 2017). It enables intuitive modeling syntax with flexible composable probabilistic programming inference. Turing supports a wide range of sampling based inference algorithms by combining model inference with differentiable programming interfaces in Julia. Yes, Julia is that amazing. The same differentiable stuff that you develop for optimization in neural networks you can plug it in to a probabilistic programming framework and it will work without much effort and boiler-plate code. Most importantly, Turing inference is composable: it combines Markov chain sampling operations on subsets of model variables, e.g. using a combination of a Hamiltonian Monte Carlo (HMC) engine and a particle Gibbs (PG) engine. This composable inference engine allows the user to easily switch between black-box style inference methods such as HMC, and customized inference methods.

I believe Turing is the most important and popular probabilistic language framework in Julia. It is what PyMC3 and Stan are for Python and R, but for Julia. Furthermore, you don't have to do "cartwheels" with Theano backends and tensors like in PyMC3 or learn a new language to declare your models like in Stan (or even have to debug C++ stuff). Turing is all Julia. It uses Julia arrays, Julia distributions, Julia autodiff, Julia plots, Julia random number generator, Julia MCMC algorithms etc. I think that developing and estimating Bayesian probabilistic models using Julia and Turing is powerful, intuitive, fun, expressive and allows easily new breakthroughs simply by being 100% Julia and embedded in Julia ecosystem. As discussed in 1. Why Julia?, having multiple dispatch with LLVM's JIT compilation allows us to combine code, types and algorithms in a very powerful and yet simple way. By using Turing in this context, a researcher (or a curious Joe) can develop new methods and extend the frontiers of Bayesian inference with new models, samplers, algorithms, or any mix-match of those.

Footnotes

[1] personally, like a good Popperian, I don't believe there is science without being evidence-based; what does not use evidence can be considered as logic, philosophy or social practices (no less or more important than science, just a demarcation of what is science and what is not; eg, mathematics and law).
[2] your smartphone (iPhone 12 - 4GB RAM) has 1,000,000x (1 million) more computing power than the computer that was aboard the Apollo 11 (4kB RAM) which took the man to the moon. Detail: this on-board computer was responsible for lunar module navigation, route and controls.
[3] if the reader wants an in-depth discussion see Nau (2001).
[4] my observation: related to the subjective Bayesian approach.
[5] my observation: related to the objective frequentist approach.
[6] a number that can be expressed as a point on a continuous line that originates from minus infinity and ends and plus infinity (−∞,+∞)(-\infty, +\infty); for those who like computing it is a floating point float ordouble.
[7] i.e. the events are "mutually exclusive". That means that only AA or BB can occur in the whole sample space.
[8] in mathematics, axioms are assumptions assumed to be true that serve as premises or starting points for the elaboration of arguments and theorems. Often the axioms are questionable, for example non-Euclidean geometry refutes Euclid's fifth axiom on parallel lines. So far there is no questioning that has supported the scrutiny of time and science about the three axioms of probability.
[9] for example, the result of a given coin is one of two mutually exclusive events: heads or tails.
[10] the formal name of the theorem is Bayes-Price-Laplace, as Thomas Bayes was the first to discover, Richard Price took his drafts, formalized in mathematical notation and presented to the Royal Society of London, and Pierre Laplace rediscovered the theorem without having had previous contact in the late 18th century in France by using probability for statistical inference with Census data in the Napoleonic era.
[11] I will cover prior probabilities in the content of tutorial 4. How to use Turing.
[12] I warned you that it was not intuitive...
[13] something worth noting: likelihood also carries a lot of subjectivity.
[14] for those who have a thing for mathematics (like myself), we calculate at which point of θ\theta the derivative of the likelihood function is zero - L′=0\mathcal{L}^\prime = 0. So we are talking really about an optimization problem that for some likelihood functions we can have a closed-form analytical solution.
[15] have I forgot to warn you that it is not so intuitive?
[16] there are several statistics textbooks that have wrong definitions of what a pp-value is. If you don't believe me, see Wasserstein & Lazar (2016).
[17] this duality is attributed to Andrew Gelman – Bayesian statistician.
[18] Ronald Fisher's personality and life controversy deserves a footnote. His contributions were undoubtedly crucial to the advancement of science and statistics. His intellect was brilliant and his talent already flourished young: before turning 33 years old he had proposed the maximum likelihood estimation method (MLE) (Stigler, 2007) and also created the concept of degrees of freedom when proposing a correction in Pearson's chi-square test (Baird, 1983). He also invented the Analysis of Variance (ANOVA) and was the first to propose randomization as a way of carrying out experiments, being considered the "father" of randomized clinical trials (RCTs). Not everything is golden in Fisher's life, he was a eugenicist and had a very strong view on ethnicity and race, advocating the superiority of certain ethnicities. Furthermore, he was extremely invariant, chasing, harming and mocking any critic of his theories and publications. What we see today in the monopoly of the Neyman-Pearson paradigm (Neyman & Pearson, 1933) with pp-values ​​and null hypotheses the result of this Fisherian effort to silence critics and let only his voice echo.
[19] which is not uncommon to see in the real world.
[20] for the curious it is a mixture of two normal distributions both with standard deviation 1, but with different means. To complete it assigns the weights of 60% for the distribution with an average of 2 and 40% for the distribution with an average of 10.
[21] to be more precise, estimation by maximum likelihood in non-convex functions cannot find an analytical solution and, if we are going to use another iterative maximization procedure, there is a risk of it becoming stuck in the second – lower-valued – mode of distribution.

References

Amrhein, V., Greenland, S., & McShane, B. (2019). Scientists rise up against statistical significance. Nature, 567(7748), 305–307. https://doi.org/10.1038/d41586-019-00857-9

Baird, D. (1983). The fisher/pearson chi-squared controversy: A turning point for inductive inference. The British Journal for the Philosophy of Science, 34(2), 105–118.

Benjamin, D. J., Berger, J. O., Johannesson, M., Nosek, B. A., Wagenmakers, E.-J., Berk, R., … Johnson, V. E. (2018). Redefine statistical significance. Nature Human Behaviour, 2(1), 6–10. https://doi.org/10.1038/s41562-017-0189-z

Bezanson, J., Edelman, A., Karpinski, S., & Shah, V. B. (2017). Julia: A fresh approach to numerical computing. SIAM Review, 59(1), 65–98.

de Finetti, B. (1974). Theory of Probability. New York: John Wiley & Sons.

Eckhardt, R. (1987). Stan Ulam, John von Neumann, and the Monte Carlo Method. Los Alamos Science, 15(30), 131–136.

Fisher, R. A. (1925). Statistical methods for research workers. Oliver; Boyd.

Fisher, R. A. (1962). Some Examples of Bayes’ Method of the Experimental Determination of Probabilities A Priori. Journal of the Royal Statistical Society. Series B (Methodological), 24(1), 118–124. Retrieved from https://www.jstor.org/stable/2983751

Ge, H., Xu, K., & Ghahramani, Z. (2018). Turing: A Language for Flexible Probabilistic Inference. International Conference on Artificial Intelligence and Statistics, 1682–1690. http://proceedings.mlr.press/v84/ge18b.html

Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian Data Analysis. Chapman and Hall/CRC.

Goodman, S. N. (2016). Aligning statistical and scientific reasoning. Science, 352(6290), 1180–1181. https://doi.org/10.1126/science.aaf5406

Head, M. L., Holman, L., Lanfear, R., Kahn, A. T., & Jennions, M. D. (2015). The extent and consequences of p-hacking in science. PLoS Biol, 13(3), e1002106.

Ioannidis, J. P. A. (2019). What Have We (Not) Learnt from Millions of Scientific Papers with <i>P</i> Values? The American Statistician, 73(sup1), 20–25. https://doi.org/10.1080/00031305.2018.1447512

It’s time to talk about ditching statistical significance. (2019). Nature, 567(7748, 7748), 283–283. https://doi.org/10.1038/d41586-019-00874-8

Jaynes, E. T. (2003). Probability theory: The logic of science. Cambridge university press.

Kolmogorov, A. N. (1933). Foundations of the Theory of Probability. Berlin: Julius Springer.

Lakens, D., Adolfi, F. G., Albers, C. J., Anvari, F., Apps, M. A. J., Argamon, S. E., … Zwaan, R. A. (2018). Justify your alpha. Nature Human Behaviour, 2(3), 168–171. https://doi.org/10.1038/s41562-018-0311-x

Nau, R. F. (2001). De Finetti was Right: Probability Does Not Exist. Theory and Decision, 51(2), 89–124. https://doi.org/10.1023/A:1015525808214

Neyman, J. (1937). Outline of a theory of statistical estimation based on the classical theory of probability. Philosophical Transactions of the Royal Society of London. Series A, Mathematical and Physical Sciences, 236(767), 333–380.

Neyman, J., & Pearson, E. S. (1933). On the problem of the most efficient tests of statistical hypotheses. Philosophical Transactions of the Royal Society of London. Series A, Containing Papers of a Mathematical or Physical Character, 231(694-706), 289–337.

Rosnow, R. L., & Rosenthal, R. (1989). Statistical procedures and the justification of knowledge in psychological science. American Psychologist, 44, 1276–1284.

Stigler, S. M. (2007). The epic story of maximum likelihood. Statistical Science, 22(4), 598–620.

van de Schoot, R., Depaoli, S., King, R., Kramer, B., Märtens, K., Tadesse, M. G., … Yau, C. (2021). Bayesian statistics and modelling. Nature Reviews Methods Primers, 1(1, 1), 1–26. https://doi.org/10.1038/s43586-020-00001-2

Wasserstein, R. L., & Lazar, N. A. (2016). The ASA’s Statement on p-Values: Context, Process, and Purpose. American Statistician, 70(2), 129–133. https://doi.org/10.1080/00031305.2016.1154108

Wasserstein, R. L., Schirm, A. L., & Lazar, N. A. (2019). Moving to a World Beyond "p < 0.05." American Statistician, 73, 1–19. https://doi.org/10.1080/00031305.2019.1583913

Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language.
\ No newline at end of file diff --git a/pages/03_prob_dist/index.html b/pages/03_prob_dist/index.html new file mode 100644 index 00000000..2d810ca0 --- /dev/null +++ b/pages/03_prob_dist/index.html @@ -0,0 +1,83 @@ + Common Probability Distributions
Bayesian Stats
Julia Logo
Code on GitHub
  1. Discrete
    1. Discrete Uniform
    2. Bernoulli
    3. Binomial
    4. Poisson
    5. Negative Binomial
  2. Continuous
    1. Normal / Gaussian
    2. Log-normal
    3. Exponential
    4. Student-tt distribution
    5. Beta Distribution
  3. Distribution Zoo
  4. References

Common Probability Distributions

Bayesian statistics uses probability distributions as the inference "engine" for the estimation of the parameter values along with their uncertainties.

Imagine that probability distributions are small pieces of "Lego". We can build whatever we want with these little pieces. We can make a castle, a house, a city; literally anything we want. The same is true for probabilistic models in Bayesian statistics. We can build models from the simplest to the most complex using probability distributions and their relationships to each other. In this tutorial we will give a brief overview of the main probabilistic distributions, their mathematical notation and their main uses in Bayesian statistics.

A probability distribution is the mathematical function that gives the probabilities of occurrence of different possible outcomes for an experiment. It is a mathematical description of a random phenomenon in terms of its sample space and the probabilities of events (subsets of the sample space).

We generally use the notation X ~ Dist (par1, par2, ...). Where X is the variable,Dist is the name of the distribution, and par are the parameters that define how the distribution behaves. Any probabilistic distribution can be "parameterized" by specifying parameters that allow us to shape some aspects of the distribution for some specific purpose.

Let's start with discrete distributions and then we'll address the continuous ones.

Discrete

Discrete probability distributions are those where the results are discrete numbers (also called whole numbers): …,−2,1,0,1,2,…,N\dots, -2, 1, 0, 1, 2, \dots, N and N∈ZN \in \mathbb{Z}. In discrete distributions we say the probability that a distribution takes certain values as "mass". The probability mass function PMF\text {PMF} is the function that specifies the probability of the random variable XX taking the value xx:

PMF(x)=P(X=x) \text{PMF}(x) = P(X = x)

Discrete Uniform

The discrete uniform distribution is a symmetric probability distribution in which a finite number of values are equally likely to be observed. Each of the nn values has an equal probability 1n\frac{1}{n}. Another way of saying "discrete uniform distribution" would be "a known and finite number of results equally likely to happen".

The discrete uniform distribution has two parameters and its notation is Unif(a,b)\text{Unif} (a, b):

  • Lower Bound (aa)

  • Upper Bound (bb)

Example: a 6-sided dice.

using CairoMakie
+using Distributions
+
+f, ax, b = barplot(
+    DiscreteUniform(1, 6);
+    axis=(;
+        title="6-sided Dice",
+        xlabel=L"\theta",
+        ylabel="Mass",
+        xticks=1:6,
+        limits=(nothing, nothing, 0, 0.3),
+    ),
+)

Discrete Uniform between 1 and 6

Bernoulli

Bernoulli's distribution describes a binary event of a successful experiment. We usually represent 00 as failure and 11 as success, so the result of a Bernoulli distribution is a binary variable Y∈{0,1}Y \in \{ 0, 1 \}.

The Bernoulli distribution is widely used to model discrete binary outcomes in which there are only two possible results.

Bernoulli's distribution has only a single parameter and its notation is Bernoulli(p)\text{Bernoulli}(p):

  • Success Probability (pp)

Example: Whether the patient survived or died or whether the customer completes their purchase or not.

f, ax1, b = barplot(
+    Bernoulli(0.5);
+    width=0.3,
+    axis=(;
+        title=L"p=0.5",
+        xlabel=L"\theta",
+        ylabel="Mass",
+        xticks=0:1,
+        limits=(nothing, nothing, 0, 1),
+    ),
+)
+ax2 = Axis(
+    f[1, 2]; title=L"p=0.2", xlabel=L"\theta", xticks=0:1, limits=(nothing, nothing, 0, 1)
+)
+barplot!(ax2, Bernoulli(0.2); width=0.3)
+linkaxes!(ax1, ax2)

Bernoulli with p={0.5,0.2}p = \{ 0.5, 0.2 \}

Binomial

The binomial distribution describes an event of the number of successes in a sequence of nnindependent experiment(s), each asking a yes-no question with a probability of success pp. Note that the Bernoulli distribution is a special case of the binomial distribution where the number of experiments is 11.

The binomial distribution has two parameters and its notation is Bin(n,p)\text{Bin} (n, p) or Binomial(n,p) \text{Binomial} (n, p):

  • Number of Experiment(s) (nn)

  • Probability of Success (pp)

Example: number of heads in 5 coin flips.

f, ax1, b = barplot(
+    Binomial(5, 0.5); axis=(; title=L"p=0.5", xlabel=L"\theta", ylabel="Mass")
+)
+ax2 = Axis(f[1, 2]; title=L"p=0.2", xlabel=L"\theta")
+barplot!(ax2, Binomial(5, 0.2))
+linkaxes!(ax1, ax2)

Binomial with n=5n=5 and p={0.5,0.2}p = \{ 0.5, 0.2 \}

Poisson

The Poisson distribution expresses the probability that a given number of events will occur in a fixed interval of time or space if those events occur with a known constant average rate and regardless of the time since the last event. The Poisson distribution can also be used for the number of events at other specified intervals, such as distance, area or volume.

The Poisson distribution has one parameter and its notation is Poisson(λ)\text{Poisson} (\lambda):

  • Rate (λ\lambda)

Example: Number of emails you receive daily. Number of holes you find on the street.

f, ax1, b = barplot(
+    Poisson(1); axis=(; title=L"\lambda=1", xlabel=L"\theta", ylabel="Mass")
+)
+ax2 = Axis(f[1, 2]; title=L"\lambda=4", xlabel=L"\theta")
+barplot!(ax2, Poisson(4))
+linkaxes!(ax1, ax2)

Poisson with λ={1,4}\lambda = \{ 1, 4 \}

Negative Binomial

The negative binomial distribution describes an event of the number of failures before the kkth success in a sequence of nn independent experiment(s), each asking a yes-no question with probability pp. Note that it becomes identical to the Poisson distribution at the limit of k→∞k \to \infty. This makes the negative binomial a robust option to replace a Poisson distribution to model phenomena with a overdispersion* (excess expected variation in data).

The negative binomial distribution has two parameters and its notation is NB(k,p)\text{NB} (k, p) or Negative-Binomial(k,p)\text{Negative-Binomial} (k, p):

  • Number of Success(es) (kk)

  • Probability of Success (pp)

Any phenomenon that can be modeled with a Poisson distribution, can be modeled with a negative binomial distribution (Gelman et al., 2013; 2020).

Example: Annual count of tropical cyclones.

f, ax1, b = barplot(
+    NegativeBinomial(1, 0.5); axis=(; title=L"k=1", xlabel=L"\theta", ylabel="Mass")
+)
+ax2 = Axis(f[1, 2]; title=L"k=2", xlabel=L"\theta")
+barplot!(ax2, NegativeBinomial(2, 0.5))
+linkaxes!(ax1, ax2)

Negative Binomial with p=0.5p=0.5 and r={1,2}r = \{ 1, 2 \}

Continuous

Continuous probability distributions are those where the results are values in a continuous range (also called real numbers): (−∞,+∞)∈R(-\infty, +\infty) \in \mathbb{R}. In continuous distributions we call the probability that a distribution takes certain values as "density". As we are talking about real numbers we are not able to obtain the probability that a random variable XX takes the value of xx. This will always be 00, as there is no way to specify an exact value of xx. xx lives in the real numbers line, so we need to specify the probability that XX takes values in a range [a,b][a,b]. The probability density function PDF\text {PDF} is defined as:

PDF(x)=P(a≤X≤b)=∫abf(x)dx \text{PDF}(x) = P(a \leq X \leq b) = \int_a^b f(x) dx

Normal / Gaussian

This distribution is generally used in the social and natural sciences to represent continuous variables in which its distributions are not known. This assumption is due to the central limit theorem. The central limit theorem states that, in some conditions, the average of many samples (observations) of a random variable with finite mean and variance is itself a random variable whose distribution converges to a normal distribution as the number of samples increases. Therefore, physical quantities that are expected to be the sum of many independent processes (such as measurement errors) often have distributions that are expected to be nearly normal.

The normal distribution has two parameters and its notation is Normal(μ,σ2)\text{Normal} (\mu, \sigma^2) or N(μ,σ2)\text{N}(\mu, \sigma^2):

  • Mean (μ\mu): distribution mean which is also both the mode and the median of the distribution

  • Standard Deviation (σ\sigma): the variance of the distribution (σ2\sigma^2) is a measure of the dispersion of the observations in relation to the mean

Example: Height, Weight, etc.

f, ax, l = lines(
+    Normal(0, 1);
+    label=L"\sigma=1",
+    linewidth=5,
+    axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)),
+)
+lines!(ax, Normal(0, 0.5); label=L"\sigma=0.5", linewidth=5)
+lines!(ax, Normal(0, 2); label=L"\sigma=2", linewidth=5)
+axislegend(ax)

Normal with μ=0\mu=0 and σ={1,0.5,2}\sigma = \{ 1, 0.5, 2 \}

Log-normal

The Log-normal distribution is a continuous probability distribution of a random variable whose logarithm is normally distributed. Thus, if a random variable XX is normally distributed by its natural log, then Y=log⁡(X)Y =\log(X) will have a normal distribution.

A random variable with logarithmic distribution accepts only positive real values. It is a convenient and useful model for measurements in the physical sciences and engineering, as well as medicine, economics and other fields, eg. for energies, concentrations, lengths, financial returns and other values.

A log-normal process is the statistical realization of the multiplicative product of many independent random variables, each one being positive.

The log-normal distribution has two parameters and its notation is Log-Normal(μ,σ2)\text{Log-Normal} (\mu, \sigma^2):

  • Mean (μ\mu): natural logarithm of the mean the distribution

  • Standard Deviation (σ\sigma): natural logarithm of the variance of the distribution (σ2\sigma^2) is a measure of the dispersion of the observations in relation to the mean

f, ax, l = lines(
+    LogNormal(0, 1);
+    label=L"\sigma=1",
+    linewidth=5,
+    axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 3, nothing, nothing)),
+)
+lines!(ax, LogNormal(0, 0.25); label=L"\sigma=0.25", linewidth=5)
+lines!(ax, LogNormal(0, 0.5); label=L"\sigma=0.5", linewidth=5)
+axislegend(ax)

Log-Normal with μ=0\mu=0 and σ={1,0.25,0.5}\sigma = \{ 1, 0.25, 0.5 \}

Exponential

The exponential distribution is the probability distribution of time between events that occur continuously and independently at a constant average rate.

The exponential distribution has one parameter and its notation is Exp(λ)\text{Exp} (\lambda):

  • Rate (λ\lambda)

Example: How long until the next earthquake. How long until the next bus arrives.

f, ax, l = lines(
+    Exponential(1);
+    label=L"\lambda=1",
+    linewidth=5,
+    axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 4.5, nothing, nothing)),
+)
+lines!(ax, Exponential(0.5); label=L"\lambda=0.5", linewidth=5)
+lines!(ax, Exponential(1.5); label=L"\lambda=2", linewidth=5)
+axislegend(ax)

Exponential with λ={1,0.5,1.5}\lambda = \{ 1, 0.5, 1.5 \}

Student-tt distribution

Student-tt distribution appears when estimating the average of a population normally distributed in situations where the sample size is small and the population standard deviation is unknown.

If we take a sample of nn observations from a normal distribution, then the distribution Student-tt with ν=n−1\nu = n-1 degrees of freedom can be defined as the distribution of the location of the sample mean relative to the true mean, divided by the standard deviation of the sample, after multiplying by the standardizing term n\sqrt{n}.

The Student-tt distribution is symmetrical and bell-shaped, like the normal distribution, but has longer tails, which means that it is more likely to produce values ​​that are far from its mean.

The Student-tt distribution has one parameter and its notation is Student-t(ν)\text{Student-$t$} (\nu):

  • Degrees of Freedom (ν\nu): controls how much it resembles a normal distribution

Example: A database full of outliers.

f, ax, l = lines(
+    TDist(2);
+    label=L"\nu=2",
+    linewidth=5,
+    axis=(; xlabel=L"\theta", ylabel="Density", limits=(-4, 4, nothing, nothing)),
+)
+lines!(ax, TDist(8); label=L"\nu=8", linewidth=5)
+lines!(ax, TDist(30); label=L"\nu=30", linewidth=5)
+axislegend(ax)

Student-tt with ν={2,8,30}\nu = \{ 2, 8, 30 \}

Beta Distribution

The beta distributions is a natural choice to model anything that is constrained to take values between 0 and 1. So it is a good candidate for probabilities and proportions.

The beta distribution has two parameters and its notation is Beta(a,b)\text{Beta} (a, b):

  • Shape parameter (aa or sometimes α\alpha): controls how much the shape is shifted towards 1

  • Shape parameter (bb or sometimes β\beta): controls how much the shape is shifted towards 0

Example: A basketball player has made already scored 5 free throws while missing 3 in a total of 8 attempts – Beta(3,5)\text{Beta}(3, 5).

f, ax, l = lines(
+    Beta(1, 1);
+    label=L"a=b=1",
+    linewidth=5,
+    axis=(; xlabel=L"\theta", ylabel="Density", limits=(0, 1, nothing, nothing)),
+)
+lines!(ax, Beta(3, 2); label=L"a=3, b=2", linewidth=5)
+lines!(ax, Beta(2, 3); label=L"a=2, b=3", linewidth=5)
+axislegend(ax)

Beta with different values of aa and bb

Distribution Zoo

I did not cover all existing distributions. There is a whole plethora of probabilistic distributions.

To access the entire "distribution zoo" use this tool from Ben Lambert (statistician from Imperial College of London): https://ben18785.shinyapps.io/distribution-zoo/

References

Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian Data Analysis. Chapman and Hall/CRC.

Gelman, A., Hill, J., & Vehtari, A. (2020). Regression and other stories. Cambridge University Press.

Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language.
\ No newline at end of file diff --git a/pages/04_Turing/index.html b/pages/04_Turing/index.html new file mode 100644 index 00000000..ee355c71 --- /dev/null +++ b/pages/04_Turing/index.html @@ -0,0 +1,174 @@ + How to use Turing
Bayesian Stats
Julia Logo
Code on GitHub
  1. Probabilistic Programming
  2. Turing's Ecosystem
  3. Turing.jl
    1. Simulating Data
    2. Visualizations
  4. Prior and Posterior Predictive Checks
  5. Conclusion
  6. Footnotes
  7. References

How to use Turing

Turing is an ecosystem of Julia packages for Bayesian Inference using probabilistic programming. Turing provides an easy and intuitive way of specifying models.

Probabilistic Programming

What is probabilistic programming (PP)? It is a programming paradigm in which probabilistic models are specified and inference for these models is performed automatically (Hardesty, 2015). In more clear terms, PP and PP Languages (PPLs) allows us to specify variables as random variables (like Normal, Binomial etc.) with known or unknown parameters. Then, we construct a model using these variables by specifying how the variables related to each other, and finally automatic inference of the variables' unknown parameters is then performed.

In a Bayesian approach this means specifying priors, likelihoods and letting the PPL compute the posterior. Since the denominator in the posterior is often intractable, we use Markov Chain Monte Carlo[1] and some fancy algorithm that uses the posterior geometry to guide the MCMC proposal using Hamiltonian dynamics called Hamiltonian Monte Carlo (HMC) to approximate the posterior. This involves, besides a suitable PPL, automatic differentiation, MCMC chains interface, and also an efficient HMC algorithm implementation. In order to provide all of these features, Turing has a whole ecosystem to address each and every one of these components.

Turing's Ecosystem

Before we dive into how to specify models in Turing, let's discuss Turing's ecosystem. We have several Julia packages under Turing's GitHub organization TuringLang, but I will focus on 6 of those:

The first one is Turing.jl (Ge, Xu & Ghahramani, 2018) itself, the main package that we use to interface with all the Turing ecosystem of packages and the backbone of the PPL Turing.

The second, MCMCChains.jl, is an interface to summarizing MCMC simulations and has several utility functions for diagnostics and visualizations.

The third package is DynamicPPL.jl (Tarek, Xu, Trapp, Ge & Ghahramani, 2020) which specifies a domain-specific language and backend for Turing (which itself is a PPL). The main feature of DynamicPPL.jl is that is is entirely written in Julia and also it is modular.

AdvancedHMC.jl (Xu, Ge, Tebbutt, Tarek, Trapp & Ghahramani, 2020) provides a robust, modular and efficient implementation of advanced HMC algorithms. The state-of-the-art HMC algorithm is the No-U-Turn Sampling (NUTS)[1] (Hoffman & Gelman, 2011) which is available in AdvancedHMC.jl.

The fourth package, DistributionsAD.jl defines the necessary functions to enable automatic differentiation (AD) of the logpdf function from Distributions.jl using the packages Tracker.jl, Zygote.jl, ForwardDiff.jl and ReverseDiff.jl. The main goal of DistributionsAD.jl is to make the output of logpdf differentiable with respect to all continuous parameters of a distribution as well as the random variable in the case of continuous distributions. This is the package that guarantees the "automatic inference" part of the definition of a PPL.

Finally, Bijectors.jl implements a set of functions for transforming constrained random variables (e.g. simplexes, intervals) to Euclidean space. Note that Bijectors.jl is still a work-in-progress and in the future we'll have better implementation for more constraints, e.g. positive ordered vectors of random variables.

Most of the time we will not be dealing with these packages directly, since Turing.jl will take care of the interfacing for us. So let's talk about Turing.jl.

Turing.jl

Turing.jl is the main package in the Turing ecosystem and the backbone that glues all the other packages together. Turing's "workflow" begin with a model specification. We specify the model inside a macro @model where we can assign variables in two ways:

  • using ~: which means that a variable follows some probability distribution (Normal, Binomial etc.) and its value is random under that distribution

  • using =: which means that a variable does not follow a probability distribution and its value is deterministic (like the normal = assignment in programming languages)

Turing will perform automatic inference on all variables that you specify using ~. Here is a simple example of how we would model a six-sided dice. Note that a "fair" dice will be distributed as a discrete uniform probability with the lower bound as 1 and the upper bound as 6:

X∼Uniform(1,6) X \sim \text{Uniform}(1,6)

Note that the expectation of a random variable X∼Uniform(a,b)X \sim \text{Uniform}(a,b) is:

E(X)=a+b2=72=3.5 E(X) = \frac{a+b}{2} = \frac{7}{2} = 3.5

Graphically this means:

using CairoMakie
+using Distributions
+
+dice = DiscreteUniform(1, 6)
+f, ax, b = barplot(
+    dice;
+    label="six-sided Dice",
+    axis=(; xlabel=L"\theta", ylabel="Mass", xticks=1:6, limits=(nothing, nothing, 0, 0.3)),
+)
+vlines!(ax, [mean(dice)]; linewidth=5, color=:red, label=L"E(\theta)")
+axislegend(ax)

A "fair" six-sided Dice: Discrete Uniform between 1 and 6

So let's specify our first Turing model. It will be named dice_throw and will have a single parameter y which is a NN-dimensional vector of integers representing the observed data, i.e. the outcomes of NN six-sided dice throws:

using Turing
+
+@model function dice_throw(y)
+    #Our prior belief about the probability of each result in a six-sided dice.
+    #p is a vector of length 6 each with probability p that sums up to 1.
+    p ~ Dirichlet(6, 1)
+
+    #Each outcome of the six-sided dice has a probability p.
+    for i in eachindex(y)
+        y[i] ~ Categorical(p)
+    end
+end;

Here we are using the Dirichlet distribution which is the multivariate generalization of the Beta distribution. The Dirichlet distribution is often used as the conjugate prior for Categorical or Multinomial distributions. Our dice is modelled as a Categorical distribution with six possible results y∈{1,2,3,4,5,6}y \in \{ 1, 2, 3, 4, 5, 6 \} with some probability vector p=(p1,…,p6)\mathbf{p} = (p_1, \dots, p_6). Since all mutually exclusive outcomes must sum up to 1 to be a valid probability, we impose the constraint that all pps must sum up to 1 – ∑i=1npi=1\sum^n_{i=1} p_i = 1. We could have used a vector of six Beta random variables but it would be hard and inefficient to enforce this constraint. Instead, I've opted for a Dirichlet with a weekly informative prior towards a "fair" dice which is encoded as a Dirichlet(6,1). This is translated as a 6-dimensional vector of elements that sum to one:

mean(Dirichlet(6, 1))
6-element Fill{Float64}, with entries equal to 0.16666666666666666
+

And, indeed, it sums up to one:

+
sum(mean(Dirichlet(6, 1)))
1.0
+

Also, since the outcome of a Categorical distribution is an integer and y is a NN-dimensional vector of integers we need to apply some sort of broadcasting here. We could use the familiar dot . broadcasting operator in Julia: y .~ Categorical(p) to signal that all elements of y are distributed as a Categorical distribution. But doing that does not allow us to do predictive checks (more on this below). So, instead we use a for-loop.

+

Simulating Data

+

Now let's set a seed for the pseudo-random number generator and simulate 1,000 throws of a six-sided dice:

+
using Random
+
+Random.seed!(123);
+
+my_data = rand(DiscreteUniform(1, 6), 1_000);
+

The vector my_data is a 1,000-length vector of Ints ranging from 1 to 6, just like how a regular six-sided dice outcome would be:

+
first(my_data, 5)
5-element Vector{Int64}:
+ 4
+ 4
+ 6
+ 2
+ 4
+

Once the model is specified we instantiate the model with the single parameter y as the simulated my_data:

+
model = dice_throw(my_data);
+

Next, we call Turing's sample() function that takes a Turing model as a first argument, along with a sampler as the second argument, and the third argument is the number of iterations. Here, I will use the NUTS() sampler from AdvancedHMC.jl and 1,000 iterations. Please note that, as default, Turing samplers will discard the first thousand (1,000) iterations as warmup. So the sampler will output 1,000 samples starting from sample 1,001 until sample 2,000:

+
chain = sample(model, NUTS(), 1_000);
+

Now let's inspect the chain. We can do that with the function describe() that will return a 2-element vector of ChainDataFrame (this is the type defined by MCMCChains.jl to store Markov chain's information regarding the inferred parameters). The first ChainDataFrame has information regarding the parameters' summary statistics (mean, std, r_hat, ...) and the second is the parameters' quantiles. Since describe(chain) returns a 2-element vector, I will assign the output to two variables:

+
summaries, quantiles = describe(chain);
+

We won't be focusing on quantiles, so let's put it aside for now. Let's then take a look at the parameters' summary statistics:

+
summaries
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64    Float64   Float64       Float64
+
+        p[1]    0.1571    0.0124    0.0003   2069.1239   657.7236    1.0047      536.1814
+        p[2]    0.1449    0.0115    0.0002   2261.9424   741.2396    1.0061      586.1473
+        p[3]    0.1401    0.0108    0.0002   1911.4282   868.5186    1.0069      495.3170
+        p[4]    0.1808    0.0120    0.0003   2087.7925   842.7605    1.0005      541.0190
+        p[5]    0.1971    0.0126    0.0003   1720.0950   673.6157    1.0019      445.7359
+        p[6]    0.1800    0.0126    0.0003   1786.2325   742.7542    1.0029      462.8745
+
+

Here p is a 6-dimensional vector of probabilities, which each one associated with a mutually exclusive outcome of a six-sided dice throw. As we expected, the probabilities are almost equal to 16\frac{1}{6}, like a "fair" six-sided dice that we simulated data from (sampling from DiscreteUniform(1, 6)). Indeed, just for a sanity check, the mean of the estimates of p sums up to 1:

+
sum(summaries[:, :mean])
1.0
+

In the future if you have some crazy huge models and you just want a subset of parameters from your chains? Just do group(chain, :parameter) or index with chain[:, 1:6, :]:

+
summarystats(chain[:, 1:3, :])
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64    Float64   Float64       Float64
+
+        p[1]    0.1571    0.0124    0.0003   2069.1239   657.7236    1.0047      536.1814
+        p[2]    0.1449    0.0115    0.0002   2261.9424   741.2396    1.0061      586.1473
+        p[3]    0.1401    0.0108    0.0002   1911.4282   868.5186    1.0069      495.3170
+
+

or chain[[:parameters,...]]:

+
summarystats(chain[[:var"p[1]", :var"p[2]"]])
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64    Float64   Float64       Float64
+
+        p[1]    0.1571    0.0124    0.0003   2069.1239   657.7236    1.0047      536.1814
+        p[2]    0.1449    0.0115    0.0002   2261.9424   741.2396    1.0061      586.1473
+
+

And, finally let's compute the expectation of the estimated six-sided dice, E(X~)E(\tilde{X}), using the standard expectation definition of expectation for a discrete random variable:

+E(X)=∑x∈Xx⋅P(x) E(X) = \sum_{x \in X} x \cdot P(x) +
sum([idx * i for (i, idx) in enumerate(summaries[:, :mean])])
3.655798033295476
+

Bingo! The estimated expectation is very close to the theoretical expectation of 72=3.5\frac{7}{2} = 3.5, as we've show in (2).

+

Visualizations

+

Note that the type of our chain is a Chains object from MCMCChains.jl:

+
typeof(chain)
MCMCChains.Chains{Float64, AxisArrays.AxisArray{Float64, 3, Array{Float64, 3}, Tuple{AxisArrays.Axis{:iter, StepRange{Int64, Int64}}, AxisArrays.Axis{:var, Vector{Symbol}}, AxisArrays.Axis{:chain, UnitRange{Int64}}}}, Missing, @NamedTuple{parameters::Vector{Symbol}, internals::Vector{Symbol}}, @NamedTuple{varname_to_symbol::OrderedCollections.OrderedDict{AbstractPPL.VarName, Symbol}, start_time::Float64, stop_time::Float64}}
+

Since Chains is a Tables.jl-compatible data structure, we can use all of the plotting capabilities from AlgebraOfGraphics.jl.

+
using AlgebraOfGraphics
+using AlgebraOfGraphics: density
+#exclude additional information such as log probability
+params = names(chain, :parameters)
+chain_mapping =
+    mapping(params .=> "sample value") *
+    mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params))
+plt1 = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines)
+plt2 = data(chain) * chain_mapping * density()
+f = Figure(; resolution=(800, 600))
+draw!(f[1, 1], plt1)
+draw!(f[1, 2], plt2; axis=(; ylabel="density"))
+

Visualization of a MCMC Chain simulation

+

On the figure above we can see, for each parameter in the model, on the left the parameter's traceplot and on the right the parameter's density[2].

+

Prior and Posterior Predictive Checks

+

Predictive checks are a great way to validate a model. The idea is to generate data from the model using parameters from draws from the prior or posterior. Prior predictive check is when we simulate data using model parameter values drawn from the prior distribution, and posterior predictive check is is when we simulate data using model parameter values drawn from the posterior distribution.

+

The workflow we do when specifying and sampling Bayesian models is not linear or acyclic (Gelman et al., 2020). This means that we need to iterate several times between the different stages in order to find a model that captures best the data generating process with the desired assumptions. The figure below demonstrates the workflow [3].

+

Bayesian Workflow

+

Bayesian Workflow. Adapted from Gelman et al. (2020)

+

This is quite easy in Turing. Our six-sided dice model already has a posterior distribution which is the object chain. We need to create a prior distribution for our model. To accomplish this, instead of supplying a MCMC sampler like NUTS(), we supply the "sampler" Prior() inside Turing's sample() function:

+
prior_chain = sample(model, Prior(), 2_000);
+

Now we can perform predictive checks using both the prior (prior_chain) or posterior (chain) distributions. To draw from the prior and posterior predictive distributions we instantiate a "predictive model", i.e. a Turing model but with the observations set to missing, and then calling predict() on the predictive model and the previously drawn samples. First let's do the prior predictive check:

+
missing_data = similar(my_data, Missing) # vector of `missing`
+model_missing = dice_throw(missing_data) # instantiate the "predictive model
+prior_check = predict(model_missing, prior_chain);
+

Here we are creating a missing_data object which is a Vector of the same length as the my_data and populated with type missing as values. We then instantiate a new dice_throw model with the missing_data vector as the y argument. Finally, we call predict() on the predictive model and the previously drawn samples, which in our case are the samples from the prior distribution (prior_chain).

+

Note that predict() returns a Chains object from MCMCChains.jl:

+
typeof(prior_check)
MCMCChains.Chains{Float64, AxisArrays.AxisArray{Float64, 3, Array{Float64, 3}, Tuple{AxisArrays.Axis{:iter, StepRange{Int64, Int64}}, AxisArrays.Axis{:var, Vector{Symbol}}, AxisArrays.Axis{:chain, UnitRange{Int64}}}}, Missing, @NamedTuple{parameters::Vector{Symbol}, internals::Vector{Symbol}}, @NamedTuple{varname_to_symbol::OrderedCollections.OrderedDict{AbstractPPL.VarName, Symbol}}}
+

And we can call summarystats():

+
summarystats(prior_check[:, 1:5, :]) # just the first 5 prior samples
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64    Float64   Float64       Missing
+
+        y[1]    3.4265    1.7433    0.0396   1966.1989        NaN    1.0016       missing
+        y[2]    3.4815    1.6974    0.0414   1672.3452        NaN    1.0024       missing
+        y[3]    3.4620    1.7149    0.0396   1878.7524        NaN    1.0022       missing
+        y[4]    3.4905    1.7177    0.0387   1960.3938        NaN    1.0006       missing
+        y[5]    3.5095    1.7101    0.0386   2002.7276        NaN    0.9998       missing
+
+

We can do the same with chain for a posterior predictive check:

+
posterior_check = predict(model_missing, chain);
+summarystats(posterior_check[:, 1:5, :]) # just the first 5 posterior samples
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64    Float64   Float64       Missing
+
+        y[1]    3.8050    1.6637    0.0510   1065.1417        NaN    0.9992       missing
+        y[2]    3.6280    1.6638    0.0583    773.8349        NaN    1.0000       missing
+        y[3]    3.6430    1.7007    0.0554    921.8019        NaN    1.0044       missing
+        y[4]    3.5720    1.6845    0.0518   1052.7732        NaN    0.9993       missing
+        y[5]    3.6390    1.7400    0.0546    988.2165        NaN    1.0012       missing
+
+

Conclusion

+

This is the basic overview of Turing usage. I hope that I could show you how simple and intuitive is to specify probabilistic models using Turing. First, specify a model with the macro @model, then sample from it by specifying the data, sampler and number of interactions. All probabilistic parameters (the ones that you've specified using ~) will be inferred with a full posterior density. Finally, you inspect the parameters' statistics like mean and standard deviation, along with convergence diagnostics like r_hat. Conveniently, you can plot stuff easily if you want to. You can also do predictive checks using either the posterior or prior model's distributions.

+

Footnotes

+

+ +
[1] + see 5. Markov Chain Monte Carlo (MCMC). + +
+ + +
[2] + we'll cover those plots and diagnostics in 5. Markov Chain Monte Carlo (MCMC). + +
+ + +
[3] + note that this workflow is a extremely simplified adaptation from the original workflow on which it was based. I suggest the reader to consult the original workflow of Gelman et al. (2020). + +
+

+

References

+

Ge, H., Xu, K., & Ghahramani, Z. (2018). Turing: A Language for Flexible Probabilistic Inference. International Conference on Artificial Intelligence and Statistics, 1682–1690. http://proceedings.mlr.press/v84/ge18b.html

+

Gelman, A., Vehtari, A., Simpson, D., Margossian, C. C., Carpenter, B., Yao, Y., … Modr’ak, M. (2020, November 3). Bayesian Workflow. Retrieved February 4, 2021, from http://arxiv.org/abs/2011.01808

+

Hardesty (2015). "Probabilistic programming does in 50 lines of code what used to take thousands". phys.org. April 13, 2015. Retrieved April 13, 2015. https://phys.org/news/2015-04-probabilistic-lines-code-thousands.html

+

Hoffman, M. D., & Gelman, A. (2011). The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo. Journal of Machine Learning Research, 15(1), 1593–1623. Retrieved from http://arxiv.org/abs/1111.4246

+

Tarek, M., Xu, K., Trapp, M., Ge, H., & Ghahramani, Z. (2020). DynamicPPL: Stan-like Speed for Dynamic Probabilistic Models. ArXiv:2002.02702 [Cs, Stat]. http://arxiv.org/abs/2002.02702

+

Xu, K., Ge, H., Tebbutt, W., Tarek, M., Trapp, M., & Ghahramani, Z. (2020). AdvancedHMC.jl: A robust, modular and efficient implementation of advanced HMC algorithms. Symposium on Advances in Approximate Bayesian Inference, 1–10. http://proceedings.mlr.press/v118/xu20a.html

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/05_MCMC/index.html b/pages/05_MCMC/index.html new file mode 100644 index 00000000..d4efd108 --- /dev/null +++ b/pages/05_MCMC/index.html @@ -0,0 +1,856 @@ + Markov Chain Monte Carlo (MCMC)
Bayesian Stats
Julia Logo
Code on GitHub
  1. What is the denominator P(data)P(\text{data}) for?
  2. What if we remove this denominator?
  3. Markov Chain Monte Carlo (MCMC)
    1. Monte Carlo Method
    2. Simulations
    3. Metropolis and Metropolis-Hastings
    4. Gibbs
    5. What happens when we run Markov chains in parallel?
  4. Hamiltonian Monte Carlo – HMC
    1. Momentum Distribution – P(ϕ)P(\phi)
    2. HMC Algorithm
    3. HMC – Implementation
    4. HMC – Complex Topologies
  5. "I understood nothing..."
  6. MCMC Metrics
    1. What looks like when your model doesn't converge
    2. MCMC Visualizations
  7. How to make your Markov chains converge
  8. Footnotes
  9. References

Markov Chain Monte Carlo (MCMC)

The main computational barrier for Bayesian statistics is the denominator P(data)P(\text{data}) of the Bayes formula:

P(θ∣data)=P(θ)⋅P(data∣θ)P(data) P(\theta \mid \text{data})=\frac{P(\theta) \cdot P(\text{data} \mid \theta)}{P(\text{data})}

In discrete cases we can turn the denominator into a sum of all parameters using the chain rule of probability:

P(A,B∣C)=P(A∣B,C)×P(B∣C) P(A,B \mid C)=P(A \mid B,C) \times P(B \mid C)

This is also called marginalization:

P(data)=∑θP(data∣θ)×P(θ) P(\text{data})=\sum_{\theta} P(\text{data} \mid \theta) \times P(\theta)

However, in the continuous cases the denominator P(data)P(\text{data}) becomes a very large and complicated integral to calculate:

P(data)=∫θP(data∣θ)×P(θ)dθ P(\text{data})=\int_{\theta} P(\text{data} \mid \theta) \times P(\theta)d \theta

In many cases this integral becomes intractable (incalculable) and therefore we must find other ways to calculate the posterior probability P(θ∣data)P(\theta \mid \text{data}) in (1) without using the denominator P(data)P(\text{data}).

What is the denominator P(data)P(\text{data}) for?

Quick answer: to normalize the posterior in order to make it a valid probability distribution. This means that the sum of all probabilities of the possible events in the probability distribution must be equal to 1:

  • in the case of discrete probability distribution: ∑θP(θ∣data)=1\sum_{\theta} P(\theta \mid \text{data}) = 1

  • in the case of continuous probability distribution: ∫θP(θ∣data)dθ=1\int_{\theta} P(\theta \mid \text{data})d \theta = 1

What if we remove this denominator?

When we remove the denominator (data)(\text{data}) we have that the posterior P(θ∣data)P(\theta \mid \text{data}) is proportional to the prior multiplied by the likelihood P(θ)⋅P(data∣θ)P(\theta) \cdot P(\text{data} \mid \theta)[1].

P(θ∣data)∝P(θ)⋅P(data∣θ) P(\theta \mid \text{data}) \propto P(\theta) \cdot P(\text{data} \mid \theta)

This YouTube video explains the denominator problem very well, see below:

Markov Chain Monte Carlo (MCMC)

This is where Markov Chain Monte Carlo comes in. MCMC is a broad class of computational tools for approximating integrals and generating samples from a posterior probability (Brooks, Gelman, Jones & Meng, 2011). MCMC is used when it is not possible to sample θ\theta directly from the subsequent probabilistic distribution P(θ∣data)P(\theta \mid \text{data}). Instead, we sample in an iterative manner such that at each step of the process we expect the distribution from which we sample P∗(θ∗∣data)P^* (\theta^* \mid \text{data}) (here ∗* means simulated) becomes increasingly similar to the posterior P(θ∣data)P(\theta \mid \text{data}). All of this is to eliminate the (often impossible) calculation of the denominator P(data)P(\text{data}).

The idea is to define an ergodic Markov chain (that is to say that there is a single stationary distribution) of which the set of possible states is the sample space and the stationary distribution is the distribution to be approximated (or sampled). Let X0,X1,…,XnX_0, X_1, \dots, X_n be a simulation of the chain. The Markov chain converges to the stationary distribution of any initial state X0X_0 after a large enough number of iterations rr, the distribution of the state XrX_r will be similar to the stationary distribution, so we can use it as a sample. Markov chains have a property that the probability distribution of the next state depends only on the current state and not on the sequence of events that preceded: P(Xn+1=x∣X0,X1,X2,…,Xn)=P(Xn+1=x∣Xn)P(X_{n+1}=x \mid X_{0},X_{1},X_{2},\ldots ,X_{n}) = P(X_{n+1}=x \mid X_{n}). This property is called Markovian, after the mathematician Andrey Markov (see figure below). Similarly, repeating this argument with XrX_r as the starting point, we can use X2rX_{2r} as a sample, and so on. We can then use the state sequence Xr,X2r,X3r,…X_r, X_{2r}, X_{3r}, \dots as almost independent samples of the stationary distribution of the Markov chain.

Andrey Markov

Andrey Markov

The effectiveness of this approach depends on:

  1. how big rr must be to ensure a suitably good sample; and

  2. computational power required for each iteration of the Markov chain.

In addition, it is customary to discard the first iterations of the algorithm as they are usually not representative of the distribution to be approximated. In the initial iterations of MCMC algorithms, generally the Markov current is in a warm-up process[2] and its state is far from ideal to start a reliable sampling. It is generally recommended to discard half of the iterations (Gelman, Carlin, Stern, Dunson, Vehtari, & Rubin, 2013a). For example: if the Markov chain has 4,000 iterations, we discard the first 2,000 as warm-up.

Monte Carlo Method

Stanislaw Ulam (figure below), who participated in the Manhattan project and when trying to calculate the neutron diffusion process for the hydrogen bomb ended up creating a class of methods called Monte Carlo.

Stanislaw Ulam

Stanislaw Ulam

Monte Carlo methods have the underlying concept of using randomness to solve problems that can be deterministic in principle. They are often used in physical and mathematical problems and are most useful when it is difficult or impossible to use other approaches. Monte Carlo methods are used mainly in three classes of problems: optimization, numerical integration and generating sample from a probability distribution.

The idea for the method came to Ulam while playing solitaire during his recovery from surgery, as he thought about playing hundreds of games to statistically estimate the probability of a successful outcome. As he himself mentions in Eckhardt (1987):

"The first thoughts and attempts I made to practice [the Monte Carlo method] were suggested by a question which occurred to me in 1946 as I was convalescing from an illness and playing solitaires. The question was what are the chances that a Canfield solitaire laid out with 52 cards will come out successfully? After spending a lot of time trying to estimate them by pure combinatorial calculations, I wondered whether a more practical method than "abstract thinking" might not be to lay it out say one hundred times and simply observe and count the number of successful plays. This was already possible to envisage with the beginning of the new era of fast computers, and I immediately thought of problems of neutron diffusion and other questions of mathematical physics, and more generally how to change processes described by certain differential equations into an equivalent form interpretable as a succession of random operations. Later... [in 1946, I ] described the idea to John von Neumann and we began to plan actual calculations."

Because it was secret, von Neumann and Ulam's work required a codename. A colleague of von Neumann and Ulam, Nicholas Metropolis (figure below), suggested using the name "Monte Carlo", which refers to Casino Monte Carlo in Monaco, where Ulam's uncle (Michał Ulam) borrowed money from relatives to gamble.

Nicholas Metropolis

Nicholas Metropolis

The applications of the Monte Carlo method are numerous: physical sciences, engineering, climate change, computational biology, computer graphics, applied statistics, artificial intelligence, search and rescue, finance and business and law. In the scope of these tutorials we will focus on applied statistics and specifically in the context of Bayesian inference: providing a random sample of the posterior distribution.

Simulations

I will do some simulations to illustrate MCMC algorithms and techniques. So, here's the initial setup:

using CairoMakie
+using Distributions
+using Random
+
+Random.seed!(123);

Let's start with a toy problem of a multivariate normal distribution of XX and YY, where

[XY]∼Multivariate Normal([μXμY],Σ)Σ∼(σX2σXσYρσXσYρσY2) \begin{bmatrix} X \\ Y \end{bmatrix} \sim \text{Multivariate Normal} \left( \begin{bmatrix} \mu_X \\ \mu_Y \end{bmatrix}, \mathbf{\Sigma} \right) \\ \mathbf{\Sigma} \sim \begin{pmatrix} \sigma^2_{X} & \sigma_{X}\sigma_{Y} \rho \\ \sigma_{X}\sigma_{Y} \rho & \sigma^2_{Y} \end{pmatrix}

If we assign μX=μY=0\mu_X = \mu_Y = 0 and σX=σY=1\sigma_X = \sigma_Y = 1 (mean 0 and standard deviation 1 for both XX and YY), we have the following formulation from (6):

[XY]∼Multivariate Normal([00],Σ),Σ∼(1ρρ1) \begin{bmatrix} X \\ Y \end{bmatrix} \sim \text{Multivariate Normal} \left( \begin{bmatrix} 0 \\ 0 \end{bmatrix}, \mathbf{\Sigma} \right), \\ \mathbf{\Sigma} \sim \begin{pmatrix} 1 & \rho \\ \rho & 1 \end{pmatrix}

All that remains is to assign a value for ρ\rho in (7) for the correlation between XX and YY. For our example we will use correlation of 0.8 (ρ=0.8\rho = 0.8):

Σ∼(10.80.81) \mathbf{\Sigma} \sim \begin{pmatrix} 1 & 0.8 \\ 0.8 & 1 \end{pmatrix} 
const N = 100_000
+const μ = [0, 0]
+const Σ = [1 0.8; 0.8 1]
+
+const mvnormal = MvNormal(μ, Σ)
FullNormal(
+dim: 2
+μ: [0.0, 0.0]
+Σ: [1.0 0.8; 0.8 1.0]
+)
+

In the figure below it is possible to see a contour plot of the PDF of a multivariate normal distribution composed of two normal variables XX and YY, both with mean 0 and standard deviation 1. The correlation between XX and YY is ρ=0.8\rho = 0.8:

x = -3:0.01:3
+y = -3:0.01:3
+dens_mvnormal = [pdf(mvnormal, [i, j]) for i in x, j in y]
+f, ax, c = contourf(x, y, dens_mvnormal; axis=(; xlabel=L"X", ylabel=L"Y"))
+Colorbar(f[1, 2], c)

Countour Plot of the PDF of a Multivariate Normal Distribution

Also a surface plot can be seen below for you to get a 3-D intuition of what is going on:

f, ax, s = surface(
+    x,
+    y,
+    dens_mvnormal;
+    axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 8),
+)

Surface Plot of the PDF of a Multivariate Normal Distribution

Metropolis and Metropolis-Hastings

The first MCMC algorithm widely used to generate samples from Markov chain originated in physics in the 1950s (in a very close relationship with the atomic bomb at the Manhattan project) and is called Metropolis (Metropolis, Rosenbluth, Rosenbluth, Teller, & Teller, 1953) in honor of the first author Nicholas Metropolis (figure above). In summary, the Metropolis algorithm is an adaptation of a random walk with an acceptance/rejection rule to converge to the target distribution.

The Metropolis algorithm uses a proposal distribution Jt(θ∗)J_t(\theta^*) (JJ stands for jumping distribution and tt indicates which state of the Markov chain we are in) to define next values of the distribution P∗(θ∗∣data)P^*(\theta^* \mid \text{data}). This distribution must be symmetrical:

Jt(θ∗∣θt−1)=Jt(θt−1∣θ∗) J_t (\theta^* \mid \theta^{t-1}) = J_t(\theta^{t-1} \mid \theta^*)

In the 1970s, a generalization of the Metropolis algorithm emerged that does not require that the proposal distributions be symmetric. The generalization was proposed by Wilfred Keith Hastings (Hastings, 1970) (figure below) and is called Metropolis-Hastings algorithm.

Wilfred Hastings

Wilfred Hastings

Metropolis Algorithm

The essence of the algorithm is a random walk through the parameters' sample space, where the probability of the Markov chain changing state is defined as:

Pchange=min⁡(P(θproposed)P(θcurrent),1) P_{\text{change}} = \min \left( {\frac{P (\theta_{\text{proposed}})}{P (\theta_{\text{current}})}}, 1 \right)

This means that the Markov chain will only change to a new state under two conditions:

  1. When the probability of the parameters proposed by the random walk P(θproposed)P(\theta_{\text{proposed}}) is greater than the probability of the parameters of the current state P(θcurrent)P(\theta_{\text{current}}), we change with 100% probability. Note that if P(θproposed)>P(θcurrent)P(\theta_{\text{proposed}}) > P(\theta_{\text{current}}) then the function min⁡\min chooses the value 1 which means 100%.

  2. When the probability of the parameters proposed by the random walk P(θproposed)P(\theta_{\text{proposed}}) is less than the probability of the parameters of the current state P(θcurrent)P(\theta_{\text{current}}), we changed with a probability equal to the proportion of that difference. Note that if P(θproposed)<P(θcurrent)P(\theta_{\text{proposed}}) < P(\theta_{\text{current}}) then the function min⁡\mindoes not choose the value 1, but the value P(θproposed)P(θcurrent)\frac{P(\theta_{\text{proposed}})}{P(\theta_{\text{current}})} which equates the proportion of the probability of the proposed parameters to the probability of the parameters at the current state.

Anyway, at each iteration of the Metropolis algorithm, even if the Markov chain changes state or not, we sample the parameter θ\theta anyway. That is, if the chain does not change to a new state, θ\theta will be sampled twice (or more if the current is stationary in the same state).

The Metropolis-Hastings algorithm can be described in the following way [3] (θ\theta is the parameter, or set of parameters, of interest and yy is the data):

  1. Define a starting point θ0\theta^0 of which p(θ0∣y)>0p(\theta^0 \mid y) > 0, or sample it from an initial distribution p0(θ)p_0(\theta). p0(θ)p_0(\theta) can be a normal distribution or a prior distribution of θ\theta (p(θ)p(\theta)).

  2. For t=1,2,…t = 1, 2, \dots:

    • Sample a proposed θ∗\theta^* from a proposal distribution in time tt, Jt(θ∗∣θt−1)J_t (\theta^* \mid \theta^{t-1}).

    • Calculate the ratio of probabilities:

      • Metropolis: r=p(θ∗∣y)p(θt−1∣y)r = \frac{p(\theta^* \mid y)}{p(\theta^{t-1} \mid y)}

      • Metropolis-Hastings: r=p(θ∗∣y)Jt(θ∗∣θt−1)p(θt−1∣y)Jt(θt−1∣θ∗)r = \frac{\frac{p(\theta^* \mid y)}{J_t(\theta^* \mid \theta^{t-1})}}{\frac{p(\theta^{t-1} \mid y)}{J_t(\theta^{t-1} \mid \theta^*)}}

    • Assign:

      θt={θ∗with probability min⁡(r,1)θt−1otherwise \theta^t = \begin{cases} \theta^* & \text{with probability } \min (r, 1) \\ \theta^{t-1} & \text{otherwise} \end{cases}

Limitations of the Metropolis Algorithm

The limitations of the Metropolis-Hastings algorithm are mainly computational. With randomly generated proposals, it usually takes a large number of iterations to enter areas of higher (more likely) posterior densities. Even efficient Metropolis-Hastings algorithms sometimes accept less than 25% of the proposals (Roberts, Gelman & Gilks, 1997). In lower-dimensional situations, the increased computational power can compensate for the lower efficiency to some extent. But in higher-dimensional and more complex modeling situations, bigger and faster computers alone are rarely enough to overcome the challenge.

Metropolis – Implementation

In our toy example we will assume that Jt(θ∗∣θt−1)J_t (\theta^* \mid \theta^{t-1}) is symmetric, thus Jt(θ∗∣θt−1)=Jt(θt−1∣θ∗)J_t(\theta^* \mid \theta^{t-1}) = J_t (\theta^{t-1} \mid \theta^*), so I'll just implement the Metropolis algorithm (not the Metropolis-Hastings algorithm).

Below I created a Metropolis sampler for our toy example. At the end it prints the acceptance rate of the proposals. Here I am using the same proposal distribution for both XX and YY: a uniform distribution parameterized with a width parameter:

X∼Uniform(X−width2,X+width2)Y∼Uniform(Y−width2,Y+width2) X \sim \text{Uniform} \left( X - \frac{\text{width}}{2}, X + \frac{\text{width}}{2} \right) \\ Y \sim \text{Uniform} \left( Y - \frac{\text{width}}{2}, Y + \frac{\text{width}}{2} \right)

I will use the already known Distributions.jl MvNormal from the plots above along with the logpdf() function to calculate the PDF of the proposed and current θ\thetas. It is easier to work with probability logs than with the absolute values[4]. Mathematically we will compute:

r=PDF⁡(Multivariate Normal([xproposedyproposed])∣Multivariate Normal([μXμY],Σ))PDF⁡(Multivariate Normal([xcurrentycurrent])∣Multivariate Normal([μXμY],Σ))=PDF⁡proposedPDF⁡current=exp⁡(log⁡(PDF⁡proposed)−log⁡(PDF⁡current)) \begin{aligned} r &= \frac{ \operatorname{PDF}\left( \text{Multivariate Normal} \left( \begin{bmatrix} x_{\text{proposed}} \\ y_{\text{proposed}} \end{bmatrix} \right) \Bigg| \text{Multivariate Normal} \left( \begin{bmatrix} \mu_X \\ \mu_Y \end{bmatrix}, \mathbf{\Sigma} \right) \right)} { \operatorname{PDF}\left( \text{Multivariate Normal} \left( \begin{bmatrix} x_{\text{current}} \\ y_{\text{current}} \end{bmatrix} \right) \Bigg| \text{Multivariate Normal} \left( \begin{bmatrix} \mu_X \\ \mu_Y \end{bmatrix}, \mathbf{\Sigma} \right) \right)}\\ &=\frac{\operatorname{PDF}_{\text{proposed}}}{\operatorname{PDF}_{\text{current}}}\\ &= \exp\Big( \log\left(\operatorname{PDF}_{\text{proposed}}\right) - \log\left(\operatorname{PDF}_{\text{current}}\right) \Big) \end{aligned}

Here is a simple implementation in Julia:

function metropolis(
+    S::Int64,
+    width::Float64,
+    ρ::Float64;
+    μ_x::Float64=0.0,
+    μ_y::Float64=0.0,
+    σ_x::Float64=1.0,
+    σ_y::Float64=1.0,
+    start_x=-2.5,
+    start_y=2.5,
+    seed=123,
+)
+    rgn = MersenneTwister(seed)
+    binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y])
+    draws = Matrix{Float64}(undef, S, 2)
+    accepted = 0::Int64
+    x = start_x
+    y = start_y
+    @inbounds draws[1, :] = [x y]
+    for s in 2:S
+        x_ = rand(rgn, Uniform(x - width, x + width))
+        y_ = rand(rgn, Uniform(y - width, y + width))
+        r = exp(logpdf(binormal, [x_, y_]) - logpdf(binormal, [x, y]))
+
+        if r > rand(rgn, Uniform())
+            x = x_
+            y = y_
+            accepted += 1
+        end
+        @inbounds draws[s, :] = [x y]
+    end
+    println("Acceptance rate is: $(accepted / S)")
+    return draws
+end
metropolis (generic function with 1 method)
+

Now let's run our metropolis() algorithm for the bivariate normal case with S = 10_000, width = 2.75 and ρ = 0.8:

+
const S = 10_000
+const width = 2.75
+const ρ = 0.8
+
+X_met = metropolis(S, width, ρ);
Acceptance rate is: 0.2116
+
+

Take a quick peek into X_met, we'll see it's a matrix of XX and YY values as columns and the time tt as rows:

+
X_met[1:10, :]
10×2 Matrix{Float64}:
+ -2.5        2.5
+ -2.5        2.5
+ -3.07501    1.47284
+ -2.60189   -0.990426
+ -2.60189   -0.990426
+ -0.592119  -0.34422
+ -0.592119  -0.34422
+ -0.592119  -0.34422
+ -0.592119  -0.34422
+ -0.592119  -0.34422
+

Also note that the acceptance of the proposals was 21%, the expected for Metropolis algorithms (around 20-25%) (Roberts et. al, 1997).

+

We can construct Chains object using MCMCChains.jl[5] by passing a matrix along with the parameters names as symbols inside the Chains() constructor:

+
using MCMCChains
+
+chain_met = Chains(X_met, [:X, :Y]);
+

Then we can get summary statistics regarding our Markov chain derived from the Metropolis algorithm:

+
summarystats(chain_met)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Missing
+
+           X   -0.0314    0.9984    0.0313   1022.8896   1135.7248    0.9999       missing
+           Y   -0.0334    0.9784    0.0305   1028.0976   1046.3325    0.9999       missing
+
+

Both of X and Y have mean close to 0 and standard deviation close to 1 (which are the theoretical values). Take notice of the ess (effective sample size - ESS) is approximate 1,000. So let's calculate the efficiency of our Metropolis algorithm by dividing the ESS by the number of sampling iterations that we've performed:

+efficiency=ESSiterations \text{efficiency} = \frac{\text{ESS}}{\text{iterations}} +
mean(summarystats(chain_met)[:, :ess_tail]) / S
0.10910286584687137
+

Our Metropolis algorithm has around 10.2% efficiency. Which, in my honest opinion, sucks...(😂)

+
Metropolis – Visual Intuition
+

I believe that a good visual intuition, even if you have not understood any mathematical formula, is the key for you to start a fruitful learning journey. So I made some animations!

+

The animation in figure below shows the first 100 simulations of the Metropolis algorithm used to generate X_met. Note that in several iterations the proposal is rejected and the algorithm samples the parameters θ1\theta_1 and θ2\theta_2 from the previous state (which becomes the current one, since the proposal is refused). The blue ellipsis represents the 90% HPD of our toy example's bivariate normal distribution.

+

Note: HPD stands for Highest Probability Density (which in our case the posterior's 90% probability range).

+
using LinearAlgebra: eigvals, eigvecs
+#source: https://discourse.julialang.org/t/plot-ellipse-in-makie/82814/4
+function getellipsepoints(cx, cy, rx, ry, θ)
+    t = range(0, 2 * pi; length=100)
+    ellipse_x_r = @. rx * cos(t)
+    ellipse_y_r = @. ry * sin(t)
+    R = [cos(θ) sin(θ); -sin(θ) cos(θ)]
+    r_ellipse = [ellipse_x_r ellipse_y_r] * R
+    x = @. cx + r_ellipse[:, 1]
+    y = @. cy + r_ellipse[:, 2]
+    return (x, y)
+end
+function getellipsepoints(μ, Σ; confidence=0.95)
+    quant = sqrt(quantile(Chisq(2), confidence))
+    cx = μ[1]
+    cy = μ[2]
+
+    egvs = eigvals(Σ)
+    if egvs[1] > egvs[2]
+        idxmax = 1
+        largestegv = egvs[1]
+        smallesttegv = egvs[2]
+    else
+        idxmax = 2
+        largestegv = egvs[2]
+        smallesttegv = egvs[1]
+    end
+
+    rx = quant * sqrt(largestegv)
+    ry = quant * sqrt(smallesttegv)
+
+    eigvecmax = eigvecs(Σ)[:, idxmax]
+    θ = atan(eigvecmax[2] / eigvecmax[1])
+    if θ < 0
+        θ += 2 * π
+    end
+
+    return getellipsepoints(cx, cy, rx, ry, θ)
+end
+
+f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+record(f, joinpath(@OUTPUT, "met_anim.gif"); framerate=5) do frame
+    for i in 1:100
+        scatter!(ax, (X_met[i, 1], X_met[i, 2]); color=(:red, 0.5))
+        linesegments!(X_met[i:(i + 1), 1], X_met[i:(i + 1), 2]; color=(:green, 0.5))
+        recordframe!(frame)
+    end
+end;
+

Animation of the First 100 Samples Generated from the Metropolis Algorithm

+

Now let's take a look how the first 1,000 simulations were, excluding 1,000 initial iterations as warm-up.

+
const warmup = 1_000
+
+f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+scatter!(
+    ax,
+    X_met[warmup:(warmup + 1_000), 1],
+    X_met[warmup:(warmup + 1_000), 2];
+    color=(:red, 0.3),
+)
+

First 1,000 Samples Generated from the Metropolis Algorithm after warm-up

+

And, finally, lets take a look in the all 9,000 samples generated after the warm-up of 1,000 iterations.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+scatter!(ax, X_met[warmup:end, 1], X_met[warmup:end, 2]; color=(:red, 0.3))
+

All 9,000 Samples Generated from the Metropolis Algorithm after warm-up

+

Gibbs

+

To circumvent the problem of low acceptance rate of the Metropolis (and Metropolis-Hastings) algorithm, the Gibbs algorithm was developed, which does not have an acceptance/rejection rule for new proposals to change the state of the Markov chain. All proposals are accepted.

+

Gibbs' algorithm had an original idea conceived by physicist Josiah Willard Gibbs (figure below), in reference to an analogy between a sampling algorithm and statistical physics (a branch of physics which is based on statistical mechanics). The algorithm was described by brothers Stuart and Donald Geman in 1984 (Geman & Geman, 1984), about eight decades after Gibbs's death.

+

Josiah Gibbs

+

Josiah Gibbs

+

The Gibbs algorithm is very useful in multidimensional sample spaces (in which there are more than 2 parameters to be sampled for the posterior probability). It is also known as alternating conditional sampling, since we always sample a parameter conditioned to the probability of the other parameters.

+

The Gibbs algorithm can be seen as a special case of the Metropolis-Hastings algorithm because all proposals are accepted (Gelman, 1992).

+

Gibbs Algorithm

+

The essence of Gibbs' algorithm is an iterative sampling of parameters conditioned to other parameters P(θ1∣θ2,…θn)P(\theta_1 \mid \theta_2, \dots \theta_n).

+

Gibbs's algorithm can be described in the following way[6] (θ\theta is the parameter, or set of parameters, of interest and yy is the data):

+
    +

  1. Define P(θ1),P(θ2),…,P(θn)P(\theta_1), P(\theta_2), \dots, P(\theta_n): the prior probability of each of the θn\theta_n parameters.

    + +

  2. Sample a starting point θ10,θ20,…,θn0\theta^0_1, \theta^0_2, \dots, \theta^0_n. We usually sample from a normal distribution or from a distribution specified as the prior distribution of θn\theta_n.

    + +

  3. For t=1,2,…t = 1, 2, \dots:

    +θ1t∼p(θ1∣θ20,…,θn0)θ2t∼p(θ2∣θ1t−1,…,θn0)⋮θnt∼p(θn∣θ1t−1,…,θn−1t−1)\begin{aligned} + \theta^t_1 &\sim p(\theta_1 \mid \theta^0_2, \dots, \theta^0_n) \\ + \theta^t_2 &\sim p(\theta_2 \mid \theta^{t-1}_1, \dots, \theta^0_n) \\ + &\vdots \\ + \theta^t_n &\sim p(\theta_n \mid \theta^{t-1}_1, \dots, \theta^{t-1}_{n-1}) + \end{aligned} + +
+

Limitations of the Gibbs Algorithm

+

The main limitation of the Gibbs algorithm is with regard to alternative conditional sampling.

+

If we compare with the Metropolis algorithm (and consequently Metropolis-Hastings) we have random proposals from a proposal distribution in which we sample each parameter unconditionally to other parameters. In order for the proposals to take us to the posterior probability's correct locations to sample, we have an acceptance/rejection rule for these proposals, otherwise the samples of the Metropolis algorithm would not approach the target distribution of interest. The state changes of the Markov chain are then carried out multidimensionally [7]. As you saw in the Metropolis' figures, in a 2-D space (as is our example's bivariate normal distribution), when there is a change of state in the Markov chain, the new proposal location considers both θ1\theta_1 and θ2\theta_2, causing diagonal movement in space 2-D sample. In other words, the proposal is done regarding all dimensions of the parameter space.

+

In the case of the Gibbs algorithm, in our toy example, this movement occurs only in a single parameter, i.e single dimension, as we sample sequentially and conditionally to other parameters. This causes horizontal movements (in the case of θ1\theta_1) and vertical movements (in the case of θ2\theta_2), but never diagonal movements like the ones we saw in the Metropolis algorithm.

+

Gibbs – Implementation

+

Here are some new things compared to the Metropolis algorithm implementation. First to conditionally sample the parameters P(θ1∣θ2)P(\theta_1 \mid \theta_2) and P(θ2∣θ1)P(\theta_2 \mid \theta_1), we need to create two new variables β and λ. These variables represent the correlation between XX and YY (θ1\theta_1 and θ2\theta_2 respectively) scaled by the ratio of the variance of XX and YY. And then we use these variables in the sampling of θ1\theta_1 and θ2\theta_2:

+β=ρ⋅σYσX=ρλ=ρ⋅σXσY=ρσYX=1−ρ2σXY=1−ρ2θ1∼Normal(μX+λ⋅(y∗−μY),σXY)θ2∼Normal(μy+β⋅(x∗−μX),σYX) +\begin{aligned} +\beta &= \rho \cdot \frac{\sigma_Y}{\sigma_X} = \rho \\ +\lambda &= \rho \cdot \frac{\sigma_X}{\sigma_Y} = \rho \\ +\sigma_{YX} &= 1 - \rho^2\\ +\sigma_{XY} &= 1 - \rho^2\\ +\theta_1 &\sim \text{Normal} \bigg( \mu_X + \lambda \cdot (y^* - \mu_Y), \sigma_{XY} \bigg) \\ +\theta_2 &\sim \text{Normal} \bigg( \mu_y + \beta \cdot (x^* - \mu_X), \sigma_{YX} \bigg) +\end{aligned} + +
function gibbs(
+    S::Int64,
+    ρ::Float64;
+    μ_x::Float64=0.0,
+    μ_y::Float64=0.0,
+    σ_x::Float64=1.0,
+    σ_y::Float64=1.0,
+    start_x=-2.5,
+    start_y=2.5,
+    seed=123,
+)
+    rgn = MersenneTwister(seed)
+    binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y])
+    draws = Matrix{Float64}(undef, S, 2)
+    x = start_x
+    y = start_y
+    β = ρ * σ_y / σ_x
+    λ = ρ * σ_x / σ_y
+    sqrt1mrho2 = sqrt(1 - ρ^2)
+    σ_YX = σ_y * sqrt1mrho2
+    σ_XY = σ_x * sqrt1mrho2
+    @inbounds draws[1, :] = [x y]
+    for s in 2:S
+        if s % 2 == 0
+            y = rand(rgn, Normal(μ_y + β * (x - μ_x), σ_YX))
+        else
+            x = rand(rgn, Normal(μ_x + λ * (y - μ_y), σ_XY))
+        end
+        @inbounds draws[s, :] = [x y]
+    end
+    return draws
+end
gibbs (generic function with 1 method)
+

Generally a Gibbs sampler is not implemented in this way. Here I coded the Gibbs algorithm so that it samples a parameter for each iteration. To be more computationally efficient we would sample all parameters are on each iteration. I did it on purpose because I want to show in the animations the real trajectory of the Gibbs sampler in the sample space (vertical and horizontal, not diagonal). So to remedy this I will provide gibbs() double the amount of S (20,000 in total). Also take notice that we are now proposing new parameters' values conditioned on other parameters, so there is not an acceptance/rejection rule here.

+
X_gibbs = gibbs(S * 2, ρ);
+

As before lets' take a quick peek into X_gibbs, we'll see it's a matrix of XX and YY values as columns and the time tt as rows:

+
X_gibbs[1:10, :]
10×2 Matrix{Float64}:
+ -2.5         2.5
+ -2.5        -1.28584
+  0.200236   -1.28584
+  0.200236    0.84578
+  0.952273    0.84578
+  0.952273    0.523811
+  0.0202213   0.523811
+  0.0202213   0.604758
+  0.438516    0.604758
+  0.438516    0.515102
+

Again, we construct a Chains object by passing a matrix along with the parameters names as symbols inside the Chains() constructor:

+
chain_gibbs = Chains(X_gibbs, [:X, :Y]);
+

Then we can get summary statistics regarding our Markov chain derived from the Gibbs algorithm:

+
summarystats(chain_gibbs)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Missing
+
+           X    0.0161    1.0090    0.0217   2152.7581   3985.5939    1.0005       missing
+           Y    0.0097    1.0071    0.0219   2114.7477   4038.1490    1.0003       missing
+
+

Both of X and Y have mean close to 0 and standard deviation close to 1 (which are the theoretical values). Take notice of the ess (effective sample size - ESS) that is around 2,100. Since we used S * 2 as the number of samples, in order for we to compare with Metropolis, we would need to divide the ESS by 2. So our ESS is between 1,000, which is similar to Metropolis' ESS. Now let's calculate the efficiency of our Gibbs algorithm by dividing the ESS by the number of sampling iterations that we've performed also accounting for the S * 2:

+
(mean(summarystats(chain_gibbs)[:, :ess_tail]) / 2) / S
0.20059357221090376
+

Our Gibbs algorithm has around 10.6% efficiency. Which, in my honest opinion, despite the small improvement still sucks...(😂)

+
Gibbs – Visual Intuition
+

Oh yes, we have animations for Gibbs also!

+

The animation in figure below shows the first 100 simulations of the Gibbs algorithm used to generate X_gibbs. Note that all proposals are accepted now, so the at each iteration we sample new parameters values. The blue ellipsis represents the 90% HPD of our toy example's bivariate normal distribution.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+record(f, joinpath(@OUTPUT, "gibbs_anim.gif"); framerate=5) do frame
+    for i in 1:200
+        scatter!(ax, (X_gibbs[i, 1], X_gibbs[i, 2]); color=(:red, 0.5))
+        linesegments!(X_gibbs[i:(i + 1), 1], X_gibbs[i:(i + 1), 2]; color=(:green, 0.5))
+        recordframe!(frame)
+    end
+end;
+

Animation of the First 100 Samples Generated from the Gibbs Algorithm

+

Now let's take a look how the first 1,000 simulations were, excluding 1,000 initial iterations as warm-up.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+scatter!(
+    ax,
+    X_gibbs[(2 * warmup):(2 * warmup + 1_000), 1],
+    X_gibbs[(2 * warmup):(2 * warmup + 1_000), 2];
+    color=(:red, 0.3),
+)
+

First 1,000 Samples Generated from the Gibbs Algorithm after warm-up

+

And, finally, lets take a look in the all 9,000 samples generated after the warm-up of 1,000 iterations.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+scatter!(ax, X_gibbs[(2 * warmup):end, 1], X_gibbs[(2 * warmup):end, 2]; color=(:red, 0.3))
+

All 9,000 Samples Generated from the Gibbs Algorithm after warm-up

+

What happens when we run Markov chains in parallel?

+

Since the Markov chains are independent, we can run them in parallel. The key to this is defining different starting points for each Markov chain (if you use a sample of a previous distribution of parameters as a starting point this is not a problem). We will use the same toy example of a bivariate normal distribution XX and YY that we used in the previous examples, but now with 4 Markov chains in parallel with different starting points[8].

+

First, let's defined 4 different pairs of starting points using a nice Cartesian product from Julia's Base.Iterators:

+
const starts = collect(Iterators.product((-2.5, 2.5), (2.5, -2.5)))
2×2 Matrix{Tuple{Float64, Float64}}:
+ (-2.5, 2.5)  (-2.5, -2.5)
+ (2.5, 2.5)   (2.5, -2.5)
+

Also, I will restrict this simulation to 100 samples:

+
const S_parallel = 100;
+

Additionally, note that we are using different seeds:

+
X_met_1 = metropolis(
+    S_parallel, width, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1])
+);
+X_met_2 = metropolis(
+    S_parallel, width, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2])
+);
+X_met_3 = metropolis(
+    S_parallel, width, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3])
+);
+X_met_4 = metropolis(
+    S_parallel, width, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4])
+);
Acceptance rate is: 0.24
+Acceptance rate is: 0.18
+Acceptance rate is: 0.18
+Acceptance rate is: 0.23
+
+

There have been some significant changes in the approval rate for Metropolis proposals. All were around 13%-24%, this is due to the low number of samples (only 100 for each Markov chain), if the samples were larger we would see these values converge to close to 20% according to the previous example of 10,000 samples with a single stream (Roberts et. al, 1997).

+

Now let's take a look on how those 4 Metropolis Markov chains sample the parameter space starting from different positions. Each chain will have its own marker and path color, so that we can see their different behavior:

+
using Colors
+const logocolors = Colors.JULIA_LOGO_COLORS;
+f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+record(f, joinpath(@OUTPUT, "parallel_met.gif"); framerate=5) do frame
+    for i in 1:99
+        scatter!(ax, (X_met_1[i, 1], X_met_1[i, 2]); color=(logocolors.blue, 0.5))
+        linesegments!(
+            X_met_1[i:(i + 1), 1], X_met_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5)
+        )
+        scatter!(ax, (X_met_2[i, 1], X_met_2[i, 2]); color=(logocolors.red, 0.5))
+        linesegments!(
+            X_met_2[i:(i + 1), 1], X_met_2[i:(i + 1), 2]; color=(logocolors.red, 0.5)
+        )
+        scatter!(ax, (X_met_3[i, 1], X_met_3[i, 2]); color=(logocolors.green, 0.5))
+        linesegments!(
+            X_met_3[i:(i + 1), 1], X_met_3[i:(i + 1), 2]; color=(logocolors.green, 0.5)
+        )
+        scatter!(ax, (X_met_4[i, 1], X_met_4[i, 2]); color=(logocolors.purple, 0.5))
+        linesegments!(
+            X_met_4[i:(i + 1), 1], X_met_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5)
+        )
+        recordframe!(frame)
+    end
+end;
+

Animation of 4 Parallel Metropolis Markov Chains

+

Now we'll do the the same for Gibbs, taking care to provide also different seeds and starting points:

+
X_gibbs_1 = gibbs(
+    S_parallel * 2, ρ; seed=124, start_x=first(starts[1]), start_y=last(starts[1])
+);
+X_gibbs_2 = gibbs(
+    S_parallel * 2, ρ; seed=125, start_x=first(starts[2]), start_y=last(starts[2])
+);
+X_gibbs_3 = gibbs(
+    S_parallel * 2, ρ; seed=126, start_x=first(starts[3]), start_y=last(starts[3])
+);
+X_gibbs_4 = gibbs(
+    S_parallel * 2, ρ; seed=127, start_x=first(starts[4]), start_y=last(starts[4])
+);
+

Now let's take a look on how those 4 Gibbs Markov chains sample the parameter space starting from different positions. Each chain will have its own marker and path color, so that we can see their different behavior:

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+record(f, joinpath(@OUTPUT, "parallel_gibbs.gif"); framerate=5) do frame
+    for i in 1:199
+        scatter!(ax, (X_gibbs_1[i, 1], X_gibbs_1[i, 2]); color=(logocolors.blue, 0.5))
+        linesegments!(
+            X_gibbs_1[i:(i + 1), 1], X_gibbs_1[i:(i + 1), 2]; color=(logocolors.blue, 0.5)
+        )
+        scatter!(ax, (X_gibbs_2[i, 1], X_gibbs_2[i, 2]); color=(logocolors.red, 0.5))
+        linesegments!(
+            X_gibbs_2[i:(i + 1), 1], X_gibbs_2[i:(i + 1), 2]; color=(logocolors.red, 0.5)
+        )
+        scatter!(ax, (X_gibbs_3[i, 1], X_gibbs_3[i, 2]); color=(logocolors.green, 0.5))
+        linesegments!(
+            X_gibbs_3[i:(i + 1), 1], X_gibbs_3[i:(i + 1), 2]; color=(logocolors.green, 0.5)
+        )
+        scatter!(ax, (X_gibbs_4[i, 1], X_gibbs_4[i, 2]); color=(logocolors.purple, 0.5))
+        linesegments!(
+            X_gibbs_4[i:(i + 1), 1], X_gibbs_4[i:(i + 1), 2]; color=(logocolors.purple, 0.5)
+        )
+        recordframe!(frame)
+    end
+end;
+

Animation of 4 Parallel Gibbs Markov Chains

+

Hamiltonian Monte Carlo – HMC

+

The problems of low acceptance rates of proposals for Metropolis techniques and the low performance of the Gibbs algorithm in multidimensional problems (in which the posterior's topology is quite complex) led to the emergence of a new MCMC technique using Hamiltonian dynamics (in honor of the Irish physicist William Rowan Hamilton (1805-1865) (figure below). The name for this technique is Hamiltonian Monte Carlo – HMC.

+

William Rowan Hamilton

+

William Rowan Hamilton

+

The HMC is an adaptation of the Metropolis technique and employs a guided scheme for generating new proposals: this improves the proposal's acceptance rate and, consequently, efficiency. More specifically, the HMC uses the posterior log gradient to direct the Markov chain to regions of higher posterior density, where most samples are collected. As a result, a Markov chain with the well-adjusted HMC algorithm will accept proposals at a much higher rate than the traditional Metropolis algorithm (Roberts et. al, 1997).

+

HMC was first described in the physics literature (Duane, Kennedy, Pendleton & Roweth, 1987) (which they called the "Hybrid" Monte Carlo – HMC). Soon after, HMC was applied to statistical problems by Neal (1994) (which he called Hamiltonean Monte Carlo – HMC). For an in-depth discussion regarding HMC (which is not the focus of this tutorial), I recommend Neal (2011) and Betancourt (2017).

+

HMC uses Hamiltonian dynamics applied to particles exploring the topology of a posterior density. In some simulations Metropolis has an acceptance rate of approximately 23%, while HMC 65% (Gelman et al., 2013b). In addition to better exploring the posterior's topology and tolerating complex topologies, HMC is much more efficient than Metropolis and does not suffer from the Gibbs' parameter correlation problem.

+

For each component θj\theta_j, the HMC adds a momentum variable ϕj\phi_j. The subsequent density P(θ∣y)P(\theta \mid y) is increased by an independent distribution P(ϕ)P(\phi) of the momentum, thus defining a joint distribution:

+P(θ,ϕ∣y)=P(ϕ)⋅P(θ∣y) P(\theta, \phi \mid y) = P(\phi) \cdot P(\theta \mid y) +

HMC uses a proposal distribution that changes depending on the current state in the Markov chain. The HMC discovers the direction in which the posterior distribution increases, called gradient, and distorts the distribution of proposals towards the gradient. In the Metropolis algorithm, the distribution of the proposals would be a (usually) Normal distribution centered on the current position, so that jumps above or below the current position would have the same probability of being proposed. But the HMC generates proposals quite differently.

+

You can imagine that for high-dimensional posterior densities that have narrow diagonal valleys and even curved valleys, the HMC dynamics will find proposed positions that are much more promising than a naive symmetric proposal distribution, and more promising than the Gibbs sampling, which can get stuck in diagonal walls.

+

The probability of the Markov chain changing state in the HMC algorithm is defined as:

+Pchange=min⁡(P(θproposed)⋅P(ϕproposed)P(θcurrent)⋅P(ϕcurrent),1) +P_{\text{change}} = \min\left({\frac{P(\theta_{\text{proposed}}) \cdot +P(\phi_{\text{proposed}})}{P(\theta_{\text{current}})\cdot P(\phi_{\text{current}})}}, 1\right) + +

where ϕ\phi is the momentum.

+

Momentum Distribution – P(ϕ)P(\phi)

+

We normally give ϕ\phi a normal multivariate distribution with a mean of 0 and a covariance of M\mathbf{M}, a "mass matrix". To keep things a little bit simpler, we use a diagonal mass matrix M\mathbf{M}. This makes the components of ϕ\phi independent with ϕj∼Normal(0,Mjj)\phi_j \sim \text{Normal}(0, M_{jj})

+

HMC Algorithm

+

The HMC algorithm is very similar to the Metropolis algorithm but with the inclusion of the momentum ϕ\phi as a way of quantifying the gradient of the posterior distribution:

+
    +

  1. Sample ϕ\phi from a Normal(0,M)\text{Normal}(0, \mathbf{M})

    + +

  2. Simultaneously sample θ\theta and ϕ\phi with LLleapfrog steps each scaled by a ϵ\epsilon factor. In a leapfrog step, both θ\theta and ϕ\phi are changed, in relation to each other. Repeat the following steps LL times:

    + +
+

2.1. Use the gradient of log posterior of θ\theta to produce a half-step of ϕ\phi:

+ϕ←ϕ+12ϵdlog⁡p(θ∣y)dθ\phi \leftarrow \phi + \frac{1}{2} \epsilon \frac{d \log p(\theta \mid y)}{d \theta} +

2.2 Use the momentum vector ϕ\phi to update the parameter vector θ\theta:

+θ←θ+ϵM−1ϕ \theta \leftarrow \theta + \epsilon \mathbf{M}^{-1} \phi +

2.3. Use again the gradient of log posterior of θ\theta to another half-step of ϕ\phi:

+ϕ←ϕ+12ϵdlog⁡p(θ∣y)dθ \phi \leftarrow \phi + \frac{1}{2} \epsilon \frac{d \log p(\theta \mid y)}{d \theta} +
    +

  1. Assign θt−1\theta^{t-1} and ϕt−1\phi^{t-1} as the values of the parameter vector and the momentum vector, respectively, at the beginning of the leapfrog process (step 2) and θ∗\theta^* and ϕ∗\phi^* as the values after LL steps. As an acceptance/rejection rule calculate:

    + +
+r=p(θ∗∣y)p(ϕ∗)p(θt−1∣y)p(ϕ−1) r = \frac{p(\theta^* \mid y) p(\phi^*)}{p(\theta^{t-1} \mid y) p(\phi^{-1})} +
    +

  1. Assign:

    + +
+θt={θ∗with probability min⁡(r,1)θt−1otherwise\theta^t = + \begin{cases} + \theta^* & \text{with probability } \min (r, 1) \\ + \theta^{t-1} & \text{otherwise} + \end{cases} +

HMC – Implementation

+

Alright let's code the HMC algorithm for our toy example's bivariate normal distribution:

+
using ForwardDiff: gradient
+function hmc(
+    S::Int64,
+    width::Float64,
+    ρ::Float64;
+    L=40,
+    ϵ=0.001,
+    μ_x::Float64=0.0,
+    μ_y::Float64=0.0,
+    σ_x::Float64=1.0,
+    σ_y::Float64=1.0,
+    start_x=-2.5,
+    start_y=2.5,
+    seed=123,
+)
+    rgn = MersenneTwister(seed)
+    binormal = MvNormal([μ_x; μ_y], [σ_x ρ; ρ σ_y])
+    draws = Matrix{Float64}(undef, S, 2)
+    accepted = 0::Int64
+    x = start_x
+    y = start_y
+    @inbounds draws[1, :] = [x y]
+    M = [1.0 0.0; 0.0 1.0]
+    ϕ_d = MvNormal([0.0, 0.0], M)
+    for s in 2:S
+        x_ = rand(rgn, Uniform(x - width, x + width))
+        y_ = rand(rgn, Uniform(y - width, y + width))
+        ϕ = rand(rgn, ϕ_d)
+        kinetic = sum(ϕ .^ 2) / 2
+        log_p = logpdf(binormal, [x, y]) - kinetic
+        ϕ += 0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_])
+        for l in 1:L
+            x_, y_ = [x_, y_] + (ϵ * M * ϕ)
+            ϕ += +0.5 * ϵ * gradient(x -> logpdf(binormal, x), [x_, y_])
+        end
+        ϕ = -ϕ # make the proposal symmetric
+        kinetic = sum(ϕ .^ 2) / 2
+        log_p_ = logpdf(binormal, [x_, y_]) - kinetic
+        r = exp(log_p_ - log_p)
+
+        if r > rand(rgn, Uniform())
+            x = x_
+            y = y_
+            accepted += 1
+        end
+        @inbounds draws[s, :] = [x y]
+    end
+    println("Acceptance rate is: $(accepted / S)")
+    return draws
+end
hmc (generic function with 1 method)
+

In the hmc() function above I am using the gradient() function from ForwardDiff.jl (Revels, Lubin & Papamarkou, 2016) which is Julia's package for forward mode auto differentiation (autodiff). The gradient() function accepts a function as input and an array X\mathbf{X}. It literally evaluates the function ff at X\mathbf{X} and returns the gradient ∇f(X)\nabla f(\mathbf{X}). This is one the advantages of Julia: I don't need to implement an autodiff for logpdf()s of any distribution, it will be done automatically for any one from Distributions.jl. You can also try reverse mode autodiff with ReverseDiff.jl if you want to; and it will also very easy to get a gradient. Now, we've got carried away with Julia's amazing autodiff potential... Let me show you an example of a gradient of a log PDF evaluated at some value. I will use our mvnormal bivariate normal distribution as an example and evaluate its gradient at x=1x = 1 and y=−1y = -1:

+
gradient(x -> logpdf(mvnormal, x), [1, -1])
2-element Vector{Float64}:
+ -5.000000000000003
+  5.000000000000003
+

So the gradient tells me that the partial derivative of x=1x = 1 with respect to our mvnormal distribution is -5 and the partial derivative of y=−1y = -1 with respect to our mvnormal distribution is 5.

+

Now let's run our HMC Markov chain. We are going to use L=40L = 40 and (don't ask me how I found out) ϵ=0.0856\epsilon = 0.0856:

+
X_hmc = hmc(S, width, ρ; ϵ=0.0856, L=40);
Acceptance rate is: 0.2079
+
+

Our acceptance rate is 20.79%. As before lets' take a quick peek into X_hmc, we'll see it's a matrix of XX and YY values as columns and the time tt as rows:

+
X_hmc[1:10, :]
10×2 Matrix{Float64}:
+ -2.5        2.5
+ -2.5        2.5
+ -1.50904    1.52822
+ -1.50904    1.52822
+ -1.85829   -0.464346
+ -0.739967  -1.15873
+ -0.739967  -1.15873
+ -0.739967  -1.15873
+ -0.739967  -1.15873
+ -0.739967  -1.15873
+

Again, we construct a Chains object by passing a matrix along with the parameters names as symbols inside the Chains() constructor:

+
chain_hmc = Chains(X_hmc, [:X, :Y]);
+

Then we can get summary statistics regarding our Markov chain derived from the HMC algorithm:

+
summarystats(chain_hmc)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Missing
+
+           X    0.0169    0.9373    0.0222   1738.2585   1537.0420    1.0014       missing
+           Y    0.0073    0.9475    0.0237   1556.0589   1640.5275    1.0003       missing
+
+

Both of X and Y have mean close to 0 and standard deviation close to 1 (which are the theoretical values). Take notice of the ess (effective sample size - ESS) that is around 1,600. Now let's calculate the efficiency of our HMC algorithm by dividing the ESS by the number of sampling iterations:

+
mean(summarystats(chain_hmc)[:, :ess_tail]) / S
0.15887847790933807
+

We see that a simple naïve (and not well-calibrated[9]) HMC has 70% more efficiency from both Gibbs and Metropolis. ≈ 10% versus ≈ 17%. Great! 😀

+

HMC – Visual Intuition

+

This wouldn't be complete without animations for HMC!

+

The animation in figure below shows the first 100 simulations of the HMC algorithm used to generate X_hmc. Note that we have a gradient-guided proposal at each iteration, so the animation would resemble more like a very lucky random-walk Metropolis [10]. The blue ellipsis represents the 90% HPD of our toy example's bivariate normal distribution.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+record(f, joinpath(@OUTPUT, "hmc_anim.gif"); framerate=5) do frame
+    for i in 1:100
+        scatter!(ax, (X_hmc[i, 1], X_hmc[i, 2]); color=(:red, 0.5))
+        linesegments!(X_hmc[i:(i + 1), 1], X_hmc[i:(i + 1), 2]; color=(:green, 0.5))
+        recordframe!(frame)
+    end
+end;
+

Animation of the First 100 Samples Generated from the HMC Algorithm

+

As usual, let's take a look how the first 1,000 simulations were, excluding 1,000 initial iterations as warm-up.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+scatter!(
+    ax,
+    X_hmc[warmup:(warmup + 1_000), 1],
+    X_hmc[warmup:(warmup + 1_000), 2];
+    color=(:red, 0.3),
+)
+

First 1,000 Samples Generated from the HMC Algorithm after warm-up

+

And, finally, lets take a look in the all 9,000 samples generated after the warm-up of 1,000 iterations.

+
f, ax, l = lines(
+    getellipsepoints(μ, Σ; confidence=0.9)...;
+    label="90% HPD",
+    linewidth=2,
+    axis=(; limits=(-3, 3, -3, 3), xlabel=L"\theta_1", ylabel=L"\theta_2"),
+)
+axislegend(ax)
+scatter!(ax, X_hmc[warmup:end, 1], X_hmc[warmup:end, 2]; color=(:red, 0.3))
+

All 9,000 Samples Generated from the HMC Algorithm after warm-up

+

HMC – Complex Topologies

+

There are cases where HMC will be much better than Metropolis or Gibbs. In particular, these cases focus on complicated and difficult-to-explore posterior topologies. In these contexts, an algorithm that can guide the proposals to regions of higher density (such as the case of the HMC) is able to explore much more efficient (less iterations for convergence) and effective (higher rate of acceptance of the proposals).

+

See figure below for an example of a bimodal posterior density with also the marginal histogram of XX and YY:

+X=Normal([102],[1001]),Y=Normal([00],[8.42.02.01.7]) +X = \text{Normal} \left( +\begin{bmatrix} +10 \\ +2 +\end{bmatrix}, +\begin{bmatrix} +1 & 0 \\ +0 & 1 +\end{bmatrix} +\right), \quad +Y = \text{Normal} \left( +\begin{bmatrix} +0 \\ +0 +\end{bmatrix}, +\begin{bmatrix} +8.4 & 2.0 \\ +2.0 & 1.7 +\end{bmatrix} +\right) + +
d1 = MvNormal([10, 2], [1 0; 0 1])
+d2 = MvNormal([0, 0], [8.4 2.0; 2.0 1.7])
+
+d = MixtureModel([d1, d2])
+
+x = -6:0.01:15
+y = -2.5:0.01:4.2
+dens_mixture = [pdf(d, [i, j]) for i in x, j in y]
+
+f, ax, s = surface(
+    x,
+    y,
+    dens_mixture;
+    axis=(type=Axis3, xlabel=L"X", ylabel=L"Y", zlabel="PDF", azimuth=pi / 4),
+)
+

Multivariate Bimodal Normal

+

And to finish an example of Neal's funnel (Neal, 2003) in the figure below. This is a very difficult posterior to be sampled even for HMC, as it varies in geometry in the dimensions XX and YY. This means that the HMC sampler has to change the leapfrog steps LL and the scaling factor ϵ\epsilon every time, since at the top of the image (the top of the funnel) a large value of LL is needed along with a small ϵ\epsilon; and at the bottom (at the bottom of the funnel) the opposite: small LL and large ϵ\epsilon.

+
funnel_y = rand(Normal(0, 3), 10_000)
+funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2)
+
+f, ax, s = scatter(
+    funnel_x,
+    funnel_y;
+    color=(:steelblue, 0.3),
+    axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)),
+)
+

Neal's Funnel

+

"I understood nothing..."

+

If you haven't understood anything by now, don't despair. Skip all the formulas and get the visual intuition of the algorithms. See the limitations of Metropolis and Gibbs and compare the animations and figures with those of the HMC. The superiority of efficiency (more samples with low autocorrelation - ESS) and effectiveness (more samples close to the most likely areas of the target distribution) is self-explanatory by the images.

+

In addition, you will probably never have to code your HMC algorithm (Gibbs, Metropolis or any other MCMC) by hand. For that there are packages like Turing's AdvancedHMC.jl In addition, AdvancedHMC has a modified HMC with a technique called No-U-Turn Sampling (NUTS)[11] (Hoffman & Gelman, 2011) that selects automatically the values ​​of ϵ\epsilon (scaling factor) and LL (leapfrog steps). The performance of the HMC is highly sensitive to these two "hyperparameters" (parameters that must be specified by the user). In particular, if LL is too small, the algorithm exhibits undesirable behavior of a random walk, while if LL is too large, the algorithm wastes computational efficiency. NUTS uses a recursive algorithm to build a set of likely candidate points that span a wide range of proposal distribution, automatically stopping when it starts to go back and retrace its steps (why it doesn't turn around - No U-turn), in addition NUTS also automatically calibrates simultaneously LL and ϵ\epsilon.

+

MCMC Metrics

+

All Markov chains have some convergence and diagnostics metrics that you should be aware of. Note that this is still an ongoing area of intense research and new metrics are constantly being proposed (e.g. Lambert & Vehtari (2020) or Vehtari, Gelman., Simpson, Carpenter & Bürkner (2021)) To show MCMC metrics let me recover our six-sided dice example from 4. How to use Turing:

+
using Turing
+
+@model function dice_throw(y)
+    #Our prior belief about the probability of each result in a six-sided dice.
+    #p is a vector of length 6 each with probability p that sums up to 1.
+    p ~ Dirichlet(6, 1)
+
+    #Each outcome of the six-sided dice has a probability p.
+    return y ~ filldist(Categorical(p), length(y))
+end;
+

Let's once again generate 1,000 throws of a six-sided dice:

+
data_dice = rand(DiscreteUniform(1, 6), 1_000);
+

Like before we'll sample using NUTS() and 1,000 iterations. Note that you can use Metropolis with MH(), Gibbs with Gibbs() and HMC with HMC() if you want to. You can check all Turing's different MCMC samplers in Turing's documentation.

+
model = dice_throw(data_dice)
+chain = sample(model, NUTS(), 1_000);
+summarystats(chain)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64    Float64   Float64       Float64
+
+        p[1]    0.1689    0.0118    0.0003   1965.3374   876.7033    0.9995      613.7843
+        p[2]    0.1661    0.0119    0.0003   1419.7589   739.3257    1.0001      443.3975
+        p[3]    0.1600    0.0108    0.0003   1594.9950   873.7550    0.9990      498.1246
+        p[4]    0.1700    0.0116    0.0003   1998.9151   884.6355    1.0045      624.2708
+        p[5]    0.1710    0.0117    0.0003   1670.1179   813.8896    0.9994      521.5858
+        p[6]    0.1641    0.0120    0.0003   2055.8182   792.1886    1.0011      642.0419
+
+

We have the following columns that outputs some kind of MCMC summary statistics:

+
    +

  • mcse: Monte Carlo Standard Error, the uncertainty about a statistic in the sample due to sampling error.

    + +

  • ess: Effective Sample Size, a rough approximation of the number of effective samples sampled by the MCMC estimated by the value of rhat.

    + +

  • rhat: a metric of convergence and stability of the Markov chain.

    + +
+

The most important metric to take into account is the rhat which is a metric that measures whether the Markov chains are stable and converged to a value during the total progress of the sampling procedure. It is basically the proportion of variation when comparing two halves of the chains after discarding the warmups. A value of 1 implies convergence and stability. By default, rhat must be less than 1.01 for the Bayesian estimation to be valid (Brooks & Gelman, 1998; Gelman & Rubin, 1992).

+

Note that all of our model's parameters have achieve a nice ess and, more important, rhat in the desired range, a solid indicator that the Markov chain is stable and has converged to the estimated parameter values.

+

What looks like when your model doesn't converge

+

Depending on the model and data, it is possible that HMC (even with NUTS) will not achieve convergence. NUTS will not converge if it encounters divergences either due to a very pathological posterior density topology or if you supply improper parameters. To exemplify let me run a "bad" chain by passing the NUTS() constructor a target acceptance rate of 0.3 with only 500 sampling iterations (including warmup of 1,000 iters):

+
bad_chain = sample(model, NUTS(0.3), 500)
+summarystats(bad_chain)
Summary Statistics
+  parameters      mean       std      mcse   ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64    Float64    Float64   Float64       Float64
+
+        p[1]    0.1684    0.0098    0.0029     8.6088    16.0774    1.1119      128.4895
+        p[2]    0.1657    0.0130    0.0029    12.6348        NaN    1.0553      188.5793
+        p[3]    0.1637    0.0112    0.0025    20.4216    26.6760    1.2401      304.7994
+        p[4]    0.1719    0.0102    0.0022    27.0245    23.5726    1.0613      403.3510
+        p[5]    0.1728    0.0181    0.0056    13.6728    13.2203    1.1900      204.0718
+        p[6]    0.1575    0.0175    0.0059     6.4771     3.5114    1.1389       96.6725
+
+

Here we can see that the ess and rhat of the bad_chain are really bad! There will be several divergences that we can access in the column numerical_error of a Chains object. Here we have 64 divergences.

+
sum(bad_chain[:numerical_error])
17.0
+

Also we can see the Markov chain acceptance rate in the column acceptance_rate. This is the bad_chain acceptance rate:

+
mean(bad_chain[:acceptance_rate])
0.018655899447476025
+

And now the "good" chain:

+
mean(chain[:acceptance_rate])
0.7934378228300696
+

What a difference huh? 80% versus 1.5%.

+

MCMC Visualizations

+

Besides the rhat values, we can also visualize the Markov chain with a traceplot. The traceplot is the overlap of the MCMC chain sampling for each estimated parameter (vertical axis). The idea is that the chains mix and that there is no slope or visual pattern along the iterations (horizontal axis). This demonstrates that the chains have mixed converged to a certain value of the parameter and remained in that region during a good part (or all) of the Markov chain sampling. We can do that with the MCMCChains.jl's function traceplot(). Let's look the "good" chain first:

+
using AlgebraOfGraphics
+params = names(chain, :parameters)
+chain_mapping =
+    mapping(params .=> "sample value") *
+    mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params))
+plt = data(chain) * mapping(:iteration) * chain_mapping * visual(Lines)
+f = Figure(; resolution=(1200, 900))
+draw!(f[1, 1], plt)
+

Traceplot for chain

+

And now the bad_chain:

+
params = names(bad_chain, :parameters)
+chain_mapping =
+    mapping(params .=> "sample value") *
+    mapping(; color=:chain => nonnumeric, row=dims(1) => renamer(params))
+plt = data(bad_chain) * mapping(:iteration) * chain_mapping * visual(Lines)
+f = Figure(; resolution=(1200, 900))
+draw!(f[1, 1], plt)
+

Traceplot for bad_chain

+

We can see that the bad_chain is all over the place and has definitely not converged or became stable.

+

How to make your Markov chains converge

+

First: Before making fine adjustments to the number of chains or the number of iterations (among others ...) know that Turing's NUTS sampler is very efficient and effective in exploring the most diverse and complex "crazy" topologies of posteriors' target distributions. The standard arguments NUTS() work perfectly for 99% of cases (even in complex models). That said, most of the time when you have sampling and computational problems in your Bayesian model, the problem is in the model specification and not in the MCMC sampling algorithm. This is known as Folk Theorem (Gelman, 2008). In the words of Andrew Gelman:

+
+

"When you have computational problems, often there's a problem with your model".

Andrew Gelman (Gelman, 2008)

+
+

If your Bayesian model has problems with convergence there are some steps that can be tried[12]. Listed here from the simplest to the most complex:

+
    +

  1. Increase the number of iterations and chains: First option is to increase the number of MCMC iterations and it is also possible to increase the number of parallel chains to be sampled.

    + +

  2. Model reparameterization: the second option is to reparameterize the model. There are two ways to parameterize the model: the first with centered parameterization (CP) and the second with non-centered parameterization (NCP). NCP is most useful in Multilevel Models, therefore we will cover NCP in 10. Multilevel Models.

    + +

  3. Collect more data: sometimes the model is too complex and we need a larger sample to get stable estimates.

    + +

  4. Rethink the model: convergence failure when we have adequate sampling is usually due to a specification of priors and likelihood function that are not compatible with the data. In this case, it is necessary to rethink the data's generative process in which the model's assumptions are anchored.

    + +
+

Footnotes

+

+ +
[1] + the symbol ∝\propto (\propto) should be read as "proportional to". + +
+ + +
[2] + some references call this process burnin. + +
+ + +
[3] + if you want a better explanation of the Metropolis and Metropolis-Hastings algorithms I suggest to see Chib & Greenberg (1995). + +
+ + +
[4] + Due to easier computational complexity and to avoid numeric overflow we generally use sum of logs instead of multiplications, specially when dealing with probabilities, i.e. R∈[0,1]\mathbb{R} \in [0, 1]. + +
+ + +
[5] + this is one of the packages of Turing's ecosystem. I recommend you to take a look into 4. How to use Turing. + +
+ + +
[6] + if you want a better explanation of the Gibbs algorithm I suggest to see Casella & George (1992). + +
+ + +
[7] + this will be clear in the animations and images. + +
+ + +
[8] + note that there is some shenanigans here to take care. You would also want to have different seeds for the random number generator in each Markov chain. This is why metropolis() and gibbs() have a seed parameter. + +
+ + +
[9] + as detailed in the following sections, HMC is quite sensitive to the choice of LL and ϵ\epsilon and I haven't tried to get the best possible combination of those. + +
+ + +
[10] + or a not-drunk random-walk Metropolis 😂. + +
+ + +
[11] + NUTS is an algorithm that uses the symplectic leapfrog integrator and builds a binary tree composed of leaf nodes that are simulations of Hamiltonian dynamics using 2j2^jleapfrog steps in forward or backward directions in time where jj is the integer representing the iterations of the construction of the binary tree. Once the simulated particle starts to retrace its trajectory, the tree construction is interrupted and the ideal number of LLleapfrog steps is defined as 2j2^j in time j−1j-1 from the beginning of the retrogression of the trajectory. So the simulated particle never "turns around" so "No U-turn". For more details on the algorithm and how it relates to Hamiltonian dynamics see Hoffman & Gelman (2011). + +
+ + +
[12] + furthermore, if you have high-correlated variables in your model, you can try a QR decomposition of the data matrix XX. I will cover QR decomposition and other Turing's computational "tricks of the trade" in 11. Computational Tricks with Turing. + +
+

+

References

+

Betancourt, M. (2017, January 9). A Conceptual Introduction to Hamiltonian Monte Carlo. Retrieved November 6, 2019, from http://arxiv.org/abs/1701.02434

+

Brooks, S., Gelman, A., Jones, G., & Meng, X.-L. (2011). Handbook of Markov Chain Monte Carlo. Retrieved from https://books.google.com?id=qfRsAIKZ4rIC

+

Brooks, S. P., & Gelman, A. (1998). General Methods for Monitoring Convergence of Iterative Simulations. Journal of Computational and Graphical Statistics, 7(4), 434–455. https://doi.org/10.1080/10618600.1998.10474787

+

Casella, G., & George, E. I. (1992). Explaining the gibbs sampler. The American Statistician, 46(3), 167–174. https://doi.org/10.1080/00031305.1992.10475878

+

Chib, S., & Greenberg, E. (1995). Understanding the Metropolis-Hastings Algorithm. The American Statistician, 49(4), 327–335. https://doi.org/10.1080/00031305.1995.10476177

+

Duane, S., Kennedy, A. D., Pendleton, B. J., & Roweth, D. (1987). Hybrid Monte Carlo. Physics Letters B, 195(2), 216–222. https://doi.org/10.1016/0370-2693(87)91197-X

+

Eckhardt, R. (1987). Stan Ulam, John von Neumann, and the Monte Carlo Method. Los Alamos Science, 15(30), 131–136.

+

Gelman, A. (1992). Iterative and Non-Iterative Simulation Algorithms. Computing Science and Statistics (Interface Proceedings), 24, 457–511. PROCEEDINGS PUBLISHED BY VARIOUS PUBLISHERS.

+

Gelman, A. (2008). The folk theorem of statistical computing. Retrieved from https://statmodeling.stat.columbia.edu/2008/05/13/thefolktheore/

+

Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013a). Basics of Markov Chain Simulation. In Bayesian Data Analysis. Chapman and Hall/CRC.

+

Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013b). Bayesian Data Analysis. Chapman and Hall/CRC.

+

Gelman, A., & Rubin, D. B. (1992). Inference from Iterative Simulation Using Multiple Sequences. Statistical Science, 7(4), 457–472. https://doi.org/10.1214/ss/1177011136

+

Geman, S., & Geman, D. (1984). Stochastic Relaxation, Gibbs Distributions, and the Bayesian Restoration of Images. IEEE Transactions on Pattern Analysis and Machine Intelligence, PAMI-6(6), 721–741. https://doi.org/10.1109/TPAMI.1984.4767596

+

Hastings, W. K. (1970). Monte Carlo sampling methods using Markov chains and their applications. Biometrika, 57(1), 97–109. https://doi.org/10.1093/biomet/57.1.97

+

Hoffman, M. D., & Gelman, A. (2011). The No-U-Turn Sampler: Adaptively Setting Path Lengths in Hamiltonian Monte Carlo. Journal of Machine Learning Research, 15(1), 1593–1623. Retrieved from http://arxiv.org/abs/1111.4246

+

Lambert, B., & Vehtari, A. (2020). R∗R^*: A robust MCMC convergence diagnostic with uncertainty using decision tree classifiers. ArXiv:2003.07900 [Stat]. http://arxiv.org/abs/2003.07900

+

Metropolis, N., Rosenbluth, A. W., Rosenbluth, M. N., Teller, A. H., & Teller, E. (1953). Equation of State Calculations by Fast Computing Machines. The Journal of Chemical Physics, 21(6), 1087–1092. https://doi.org/10.1063/1.1699114

+

Neal, Radford M. (1994). An Improved Acceptance Procedure for the Hybrid Monte Carlo Algorithm. Journal of Computational Physics, 111(1), 194–203. https://doi.org/10.1006/jcph.1994.1054

+

Neal, Radford M. (2003). Slice Sampling. The Annals of Statistics, 31(3), 705–741. Retrieved from https://www.jstor.org/stable/3448413

+

Neal, Radford M. (2011). MCMC using Hamiltonian dynamics. In S. Brooks, A. Gelman, G. L. Jones, & X.-L. Meng (Eds.), Handbook of markov chain monte carlo.

+

Revels, J., Lubin, M., & Papamarkou, T. (2016). Forward-Mode Automatic Differentiation in Julia. ArXiv:1607.07892 [Cs]. http://arxiv.org/abs/1607.07892

+

Roberts, G. O., Gelman, A., & Gilks, W. R. (1997). Weak convergence and optimal scaling of random walk Metropolis algorithms. Annals of Applied Probability, 7(1), 110–120. https://doi.org/10.1214/aoap/1034625254

+

Vehtari, A., Gelman, A., Simpson, D., Carpenter, B., & Bürkner, P.-C. (2021). Rank-Normalization, Folding, and Localization: An Improved Rˆ for Assessing Convergence of MCMC. Bayesian Analysis, 1(1), 1–28. https://doi.org/10.1214/20-BA1221

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/06_linear_reg/index.html b/pages/06_linear_reg/index.html new file mode 100644 index 00000000..dc309ed1 --- /dev/null +++ b/pages/06_linear_reg/index.html @@ -0,0 +1,49 @@ + Bayesian Linear Regression
Bayesian Stats
Julia Logo
Code on GitHub
  1. Linear Regression
  2. Example - Children's IQ Score
  3. References

Bayesian Linear Regression

"All models are wrong but some are useful"

George Box (Box, 1976)

This tutorial begins with a very provocative quote from the statistician George Box (figure below) on statistical models. Yes, all models are somehow wrong. But they are very useful. The idea is that the reality is too complex for us to understand when analyzing it in a naked and raw way. We need to somehow simplify it into individual components and analyze their relationships. But there is a danger here: any simplification of reality promotes loss of information in some way. Therefore, we always have a delicate balance between simplifications of reality through models and the inherent loss of information. Now you ask me: "how are they useful?" Imagine that you are in total darkness and you have a very powerful flashlight but with a narrow beam. Are you going to throw the flashlight away because it can't light everything around you and stay in the dark? You must use the flashlight to aim at interesting places in the darkness in order to illuminate them. You will never find a flashlight that illuminates everything with the clarity you need to analyze all the fine details of reality. Just as you will never find a unique model that will explain the whole reality around you. You need different flashlights just like you need different models. Without them you will be in total darkness.

George Box

George Box

Linear Regression

Let's talk about a class of model known as linear regression. The idea here is to model a continuous dependent variable with a linear combination of independent variables.

y=α+Xβ+ϵ \mathbf{y} = \alpha + \mathbf{X} \boldsymbol{\beta} + \epsilon

where:

  • y\mathbf{y} – dependent variable

  • α\alpha – intercept

  • β\boldsymbol{\beta} – coefficient vector

  • X\mathbf{X} – data matrix

  • ϵ\epsilon – model error

To estimate the β\boldsymbol{\beta} coefficients we use a Gaussian/normal likelihood function. Mathematically the Bayesian regression model is:

y∼Normal(α+X⋅β,σ)α∼Normal(μα,σα)β∼Normal(μβ,σβ)σ∼Exponential(λσ) \begin{aligned} \mathbf{y} &\sim \text{Normal}\left( \alpha + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ \sigma &\sim \text{Exponential}(\lambda_\sigma) \end{aligned}

Here we see that the likelihood function P(y∣θ)P(\mathbf{y} \mid \boldsymbol{\theta}) is a normal distribution in which y\mathbf{y} depends on the parameters of the model α\alpha and β\boldsymbol{\beta}, in addition to having an error σ\sigma. We condition y\mathbf{y} onto the observed data X\mathbf{X} by inserting α+X⋅β\alpha + \mathbf{X} \cdot \boldsymbol{\beta} as the linear predictor of the model (the mean μ\mu parameter of the model's Normal likelihood function, and σ\sigma is the variance parameter). What remains is to specify which are the priors of the model parameters:

  • Prior Distribution of α\alpha – Knowledge we possess regarding the model's intercept.

  • Prior Distribution of β\boldsymbol{\beta} – Knowledge we possess regarding the model's independent variables' coefficients.

  • Prior Distribution of σ\sigma – Knowledge we possess regarding the model's error. Important that the error can only be positive. In addition, it is intuitive to place a distribution that gives greater weight to values close to zero, but that also allows values that are far from zero, so a distribution with a long tail is welcome. Candidate distributions are Exponential\text{Exponential} which is only supported on positive real numbers (so it solves the question of negative errors) or Cauchy+\text{Cauchy}^+ truncated to only positive numbers (remembering that the distribution Cauchy is Student's tt with degrees of freedom ν=1\nu = 1).

Our goal is to instantiate a linear regression with the observed data (y\mathbf{y} and X\mathbf{X}) and find the posterior distribution of our model's parameters of interest (α\alpha and β\boldsymbol{\beta}). This means to find the full posterior distribution of:

P(θ∣y)=P(α,β,σ∣y) P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta}, \sigma \mid \mathbf{y})

This is easily accomplished with Turing:

using Turing
+using LinearAlgebra: I
+using Statistics: mean, std
+using Random: seed!
+seed!(123)
+
+@model function linreg(X, y; predictors=size(X, 2))
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))
+    β ~ filldist(TDist(3), predictors)
+    σ ~ Exponential(1)
+
+    #likelihood
+    return y ~ MvNormal(α .+ X * β, σ^2 * I)
+end;

Here I am specifying very weakly informative priors:

  • α∼Normal(yˉ,2.5⋅σy)\alpha \sim \text{Normal}(\bar{\mathbf{y}}, 2.5 \cdot \sigma_{\mathbf{y}}) – This means a normal distribution centered on y's mean with a standard deviation 2.5 times the standard deviation of y. That prior should with ease cover all possible values of α\alpha. Remember that the normal distribution has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty).

  • β∼Student-t(0,1,3)\boldsymbol{\beta} \sim \text{Student-}t(0,1,3) – The predictors all have a prior distribution of a Student-tt distribution centered on 0 with variance 1 and degrees of freedom ν=3\nu = 3. That wide-tailed tt distribution will cover all possible values for our coefficients. Remember the Student-tt also has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). Also the filldist() is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution.

  • σ∼Exponential(1)\sigma \sim \text{Exponential}(1) – A wide-tailed-positive-only distribution perfectly suited for our model's error.

Also, we are using the MvNormal construction where we specify both a vector of means (first positional argument) and a covariance matrix (second positional argument). Regarding the covariance matrix σ^2 * I, it uses the model's errors σ, here parameterized as a standard deviation, squares it to produce a variance paramaterization, and multiplies by I, which is Julia's LinearAlgebra standard module implementation to represent an identity matrix of any size.

Example - Children's IQ Score

For our example, I will use a famous dataset called kidiq (Gelman & Hill, 2007), which is data from a survey of adult American women and their respective children. Dated from 2007, it has 434 observations and 4 variables:

  • kid_score: child's IQ

  • mom_hs: binary/dummy (0 or 1) if the child's mother has a high school diploma

  • mom_iq: mother's IQ

  • mom_age: mother's age

Ok let's read our data with CSV.jl and output into a DataFrame from DataFrames.jl:

using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv"
+kidiq = CSV.read(HTTP.get(url).body, DataFrame)
+describe(kidiq)
4×7 DataFrame
+ Row │ variable   mean        min      median   max      nmissing  eltype
+     │ Symbol     Float64     Real     Float64  Real     Int64     DataType
+─────┼──────────────────────────────────────────────────────────────────────
+   1 │ kid_score   86.7972    20       90.0     144             0  Int64
+   2 │ mom_hs       0.785714   0        1.0       1             0  Int64
+   3 │ mom_iq     100.0       71.0374  97.9153  138.893         0  Float64
+   4 │ mom_age     22.7857    17       23.0      29             0  Int64

As you can see from the describe() output, the mean children's IQ is around 87 while the mother's is 100. Also the mother's range from 17 to 29 years with mean of around 23 years old. Finally, note that 79% of mothers have a high school diploma.

Now let's us instantiate our model with the data:

X = Matrix(select(kidiq, Not(:kid_score)))
+y = kidiq[:, :kid_score]
+model = linreg(X, y);

And, finally, we will sample from the Turing model. We will be using the default NUTS() sampler with 1_000 samples, but now we will sample from 4 Markov chains using multiple threads MCMCThreads():

chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4)
+summarystats(chain)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α   21.5126    8.6720    0.2217   1528.8886   1868.2612    1.0009       24.7261
+        β[1]    2.0014    1.7943    0.0419   2281.1940   1734.6512    1.0016       36.8928
+        β[2]    0.5788    0.0584    0.0013   2163.9754   2292.8814    1.0006       34.9971
+        β[3]    0.2566    0.3092    0.0074   1762.0214   2135.6795    1.0010       28.4965
+           σ   17.8859    0.6033    0.0106   3271.1669   2347.2435    1.0008       52.9033
+

We had no problem with the Markov chains as all the rhat are well below 1.01 (or above 0.99). Our model has an error σ of around 18. So it estimates IQ±9. The intercept α is the basal child's IQ. So each child has 22±9 IQ before we add the coefficients multiplied by the child's independent variables. And from our coefficients β\boldsymbol{\beta}, we can see that the quantile() tells us the uncertainty around their estimates:

quantile(chain)
Quantiles
+  parameters      2.5%     25.0%     50.0%     75.0%     97.5%
+      Symbol   Float64   Float64   Float64   Float64   Float64
+
+           α    4.7278   15.7633   21.2942   27.4322   38.4426
+        β[1]   -0.5876    0.7324    1.6761    2.9919    6.3388
+        β[2]    0.4662    0.5392    0.5793    0.6184    0.6924
+        β[3]   -0.3477    0.0440    0.2588    0.4733    0.8490
+           σ   16.7525   17.4685   17.8796   18.2703   19.1238
+

  • β[1] – first column of X, mom_hs, has 95% credible interval that is all over the place, including zero. So its effect on child's IQ is inconclusive.

  • β[2] – second column of X, mom_iq, has a 95% credible interval from 0.46 to 0.69. So we expect that every increase in the mother's IQ is associated with a 0.46 to 0.69 increase in the child's IQ.

  • β[3] – third column of X, mom_age, has also 95% credible interval that is all over the place, including zero. Like mom_hs, its effect on child's IQ is inconclusive.

That's how you interpret 95% credible intervals from a quantile() output of a linear regression Chains object.

References

Box, G. E. P. (1976). Science and Statistics. Journal of the American Statistical Association, 71(356), 791–799. https://doi.org/10.2307/2286841

Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press.

Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language.
\ No newline at end of file diff --git a/pages/07_logistic_reg/index.html b/pages/07_logistic_reg/index.html new file mode 100644 index 00000000..9960c22a --- /dev/null +++ b/pages/07_logistic_reg/index.html @@ -0,0 +1,135 @@ + Bayesian Logistic Regression
Bayesian Stats
Julia Logo
Code on GitHub
  1. Comparison with Linear Regression
  2. Bayesian Logistic Regression
    1. Using Bernoulli Likelihood
    2. Using Binomial Likelihood
  3. Example - Contaminated Water Wells
  4. Footnotes
  5. References

Bayesian Logistic Regression

Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The first is logistic regression (also called binomial regression).

A logistic regression behaves exactly like a linear model: it makes a prediction simply by computing a weighted sum of the independent variables X\mathbf{X} by the estimated coefficients β\boldsymbol{\beta}, plus an intercept α\alpha. However, instead of returning a continuous value yy, such as linear regression, it returns the logistic function of yy:

Logistic(x)=11+e(−x) \text{Logistic}(x) = \frac{1}{1 + e^{(-x)}}

We use logistic regression when our dependent variable is binary. It has only two distinct values, usually encoded as 00 or 11. See the figure below for a graphical intuition of the logistic function:

using CairoMakie
+
+function logistic(x)
+    return 1 / (1 + exp(-x))
+end
+
+f, ax, l = lines(-10 .. 10, logistic; axis=(; xlabel=L"x", ylabel=L"\mathrm{Logistic}(x)"))
+f

Logistic Function

As we can see, the logistic function is basically a mapping of any real number to a real number in the range between 0 and 1:

Logistic(x)={R∈[−∞,+∞]}→{R∈(0,1)} \text{Logistic}(x) = \{ \mathbb{R} \in [- \infty , + \infty] \} \to \{ \mathbb{R} \in (0, 1) \}

That is, the logistic function is the ideal candidate for when we need to convert something continuous without restrictions to something continuous restricted between 0 and 1. That is why it is used when we need a model to have a probability as a dependent variable (remembering that any real number between 0 and 1 is a valid probability). In the case of a binary dependent variable, we can use this probability as the chance of the dependent variable taking a value of 0 or 1.

Comparison with Linear Regression

Linear regression follows the following mathematical formulation:

Linear=θ0+θ1x1+θ2x2+⋯+θnxn \text{Linear} = \theta_0 + \theta_1 x_1 + \theta_2 x_2 + \dots + \theta_n x_n

  • θ\theta - model parameters

    • θ0\theta_0 - intercept

    • θ1,θ2,…\theta_1, \theta_2, \dots - independent variables x1,x2,…x_1, x_2, \dots coefficients

  • nn - total number of independent variables

Logistic regression would add the logistic function to the linear term:

  • p^=Logistic(Linear)=11+e−Linear⁡\hat{p} = \text{Logistic}(\text{Linear}) = \frac{1}{1 + e^{-\operatorname{Linear}}} - predicted probability of the observation being the value 1

  • y^={0 if p^<0.51 if p^≥0.5\hat{\mathbf{y}}=\left\{\begin{array}{ll} 0 & \text { if } \hat{p} < 0.5 \\ 1 & \text { if } \hat{p} \geq 0.5 \end{array}\right. - predicted discrete value of y\mathbf{y}

Example:

Probability of Death=Logistic(−10+10⋅cancer+12⋅diabetes+8⋅obesity) \text{Probability of Death} = \text{Logistic} \big(-10 + 10 \cdot \text{cancer} + 12 \cdot \text{diabetes} + 8 \cdot \text{obesity} \big)

Bayesian Logistic Regression

We can model logistic regression in two ways. The first option with a Bernoulli likelihood function and the second option with a binomial likelihood function.

With the Bernoulli likelihood we model a binary dependent variable yy which is the result of a Bernoulli trial with a certain probability pp.

In a binomial likelihood, we model a continuous dependent variable yy which is the number of successes of nn independent Bernoulli trials.

Using Bernoulli Likelihood

y∼Bernoulli(p)p∼Logistic(α+X⋅β)α∼Normal(μα,σα)β∼Normal(μβ,σβ) \begin{aligned} \mathbf{y} &\sim \text{Bernoulli}\left( p \right) \\ \mathbf{p} &\sim \text{Logistic}(\alpha + \mathbf{X} \cdot \boldsymbol{\beta}) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \end{aligned}

where:

  • y\mathbf{y} – binary dependent variable.

  • p\mathbf{p} – probability of y\mathbf{y} taking the value of y\mathbf{y} – success of an independent Bernoulli trial.

  • Logistic\text{Logistic} – logistic function.

  • α\alpha – intercept.

  • β\boldsymbol{\beta} – coefficient vector.

  • X\mathbf{X} – data matrix.

Using Binomial Likelihood

y∼Binomial(n,p)p∼Logistic(α+X⋅β)α∼Normal(μα,σα)β∼Normal(μβ,σβ) \begin{aligned} \mathbf{y} &\sim \text{Binomial}\left( n, p \right) \\ \mathbf{p} &\sim \text{Logistic}(\alpha + \mathbf{X} \cdot \boldsymbol{\beta}) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \end{aligned}

where:

  • y\mathbf{y} – binary dependent variable – successes of nn independent Bernoulli trials.

  • nn – number of independent Bernoulli trials.

  • p\mathbf{p} – probability of y\mathbf{y} taking the value of y\mathbf{y} – success of an independent Bernoulli trial.

  • Logistic\text{Logistic} – logistic function.

  • α\alpha – intercept.

  • β\boldsymbol{\beta} – coefficient vector.

  • X\mathbf{X} – data matrix.

In both likelihood options, what remains is to specify the model parameters' prior distributions:

  • Prior Distribution of α\alpha – Knowledge we possess regarding the model's intercept.

  • Prior Distribution of β\boldsymbol{\beta} – Knowledge we possess regarding the model's independent variables' coefficients.

Our goal is to instantiate a logistic regression with the observed data (y\mathbf{y} and X\mathbf{X}) and find the posterior distribution of our model's parameters of interest (α\alpha and β\boldsymbol{\beta}). This means to find the full posterior distribution of:

P(θ∣y)=P(α,β∣y) P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta} \mid \mathbf{y})

Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error parameter σ\sigma in our logistic regression. This is due to neither the Bernoulli nor binomial distributions having a "scale" parameter such as the σ\sigma parameter in the Gaussian/normal distribution.

Also note that the Bernoulli distribution is a special case of the binomial distribution where n=1n = 1:

Bernoulli(p)=Binomial(1,p) \text{Bernoulli}(p) = \text{Binomial}(1, p)

This is easily accomplished with Turing:

using Turing
+using LazyArrays
+using Random: seed!
+seed!(123)
+
+@model function logreg(X, y; predictors=size(X, 2))
+    #priors
+    α ~ Normal(0, 2.5)
+    β ~ filldist(TDist(3), predictors)
+
+    #likelihood
+    return y ~ arraydist(LazyArray(@~ BernoulliLogit.(α .+ X * β)))
+end;

Here I am specifying very weakly informative priors:

  • α∼Normal(0,2.5)\alpha \sim \text{Normal}(0, 2.5) – This means a normal distribution centered on 0 with a standard deviation of 2.5. That prior should with ease cover all possible values of α\alpha. Remember that the normal distribution has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty).

  • β∼Student-t(0,1,3)\boldsymbol{\beta} \sim \text{Student-}t(0,1,3) – The predictors all have a prior distribution of a Student-tt distribution centered on 0 with variance 1 and degrees of freedom ν=3\nu = 3. That wide-tailed tt distribution will cover all possible values for our coefficients. Remember the Student-tt also has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). Also the filldist() is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution.

Turing's arraydist() function wraps an array of distributions returning a new distribution sampling from the individual distributions. And the LazyArrays' LazyArray() constructor wrap a lazy object that wraps a computation producing an array to an array. Last, but not least, the macro @~ creates a broadcast and is a nice short hand for the familiar dot . broadcasting operator in Julia. This is an efficient way to tell Turing that our y vector is distributed lazily as a BernoulliLogit broadcasted to α added to the product of the data matrix X and β coefficient vector.

If your dependent variable y is continuous and represents the number of successes of nn independent Bernoulli trials you can use the binomial likelihood in your model:

y ~ arraydist(LazyArray(@~ BinomialLogit.(n, α .+ X * β)))
+

Example - Contaminated Water Wells

+

For our example, I will use a famous dataset called wells (Gelman & Hill, 2007), which is data from a survey of 3,200 residents in a small area of Bangladesh suffering from arsenic contamination of groundwater. Respondents with elevated arsenic levels in their wells had been encouraged to switch their water source to a safe public or private well in the nearby area and the survey was conducted several years later to learn which of the affected residents had switched wells. It has 3,200 observations and the following variables:

+
    +

  • switch – binary/dummy (0 or 1) for well-switching.

    + +

  • arsenic – arsenic level in respondent's well.

    + +

  • dist – distance (meters) from the respondent's house to the nearest well with safe drinking water.

    + +

  • association – binary/dummy (0 or 1) if member(s) of household participate in community organizations.

    + +

  • educ – years of education (head of household).

    + +
+

Ok let's read our data with CSV.jl and output into a DataFrame from DataFrames.jl:

+
using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/wells.csv"
+wells = CSV.read(HTTP.get(url).body, DataFrame)
+describe(wells)
5×7 DataFrame
+ Row │ variable  mean       min    median   max      nmissing  eltype
+     │ Symbol    Float64    Real   Float64  Real     Int64     DataType
+─────┼──────────────────────────────────────────────────────────────────
+   1 │ switch     0.575166  0       1.0       1             0  Int64
+   2 │ arsenic    1.65693   0.51    1.3       9.65          0  Float64
+   3 │ dist      48.3319    0.387  36.7615  339.531         0  Float64
+   4 │ assoc      0.422848  0       0.0       1             0  Int64
+   5 │ educ       4.82848   0       5.0      17             0  Int64
+

As you can see from the describe() output 58% of the respondents switched wells and 42% percent of respondents somehow are engaged in community organizations. The average years of education of the household's head is approximate 5 years and ranges from 0 (no education at all) to 17 years. The distance to safe drinking water is measured in meters and averages 48m ranging from less than 1m to 339m. Regarding arsenic levels I cannot comment because the only thing I know that it is toxic and you probably would never want to have your well contaminated with it. Here, we believe that all of those variables somehow influence the probability of a respondent switch to a safe well.

+

Now let's us instantiate our model with the data:

+
X = Matrix(select(wells, Not(:switch)))
+y = wells[:, :switch]
+model = logreg(X, y);
+

And, finally, we will sample from the Turing model. We will be using the default NUTS() sampler with 1_000 samples, with 4 Markov chains using multiple threads MCMCThreads():

+
chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4)
+summarystats(chain)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α   -0.1537    0.1008    0.0026   1487.6006   2091.0499    1.0020       26.5174
+        β[1]    0.4664    0.0419    0.0009   2237.3707   2405.0432    1.0010       39.8825
+        β[2]   -0.0090    0.0010    0.0000   4269.6543   3192.4775    1.0009       76.1093
+        β[3]   -0.1226    0.0777    0.0019   1680.2431   1877.4329    1.0002       29.9514
+        β[4]    0.0424    0.0096    0.0002   2656.3110   2520.0618    1.0020       47.3504
+
+

We had no problem with the Markov chains as all the rhat are well below 1.01 (or above 0.99). Note that the coefficients are in log-odds scale. They are the natural log of the odds[1], and odds is defined as:

+odds=p1−p \text{odds} = \frac{p}{1-p} +

where pp is a probability. So log-odds is defined as:

+log⁡(odds)=log⁡(p1−x) \log(\text{odds}) = \log \left( \frac{p}{1-x} \right) +

So in order to get odds from a log-odds we must undo the log operation with a exponentiation. This translates to:

+odds=exp⁡(log⁡(odds)) \text{odds} = \exp ( \log ( \text{odds} )) +

We can do this with a transformation in a DataFrame constructed from a Chains object:

+
using Chain
+
+@chain quantile(chain) begin
+    DataFrame
+    select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false)
+end
5×6 DataFrame
+ Row │ parameters  2.5%      25.0%     50.0%     75.0%     97.5%
+     │ Symbol      Float64   Float64   Float64   Float64   Float64
+─────┼──────────────────────────────────────────────────────────────
+   1 │ α           0.700906  0.801908  0.859547  0.91731   1.04274
+   2 │ β[1]        1.47317   1.54865   1.59402   1.63747   1.7351
+   3 │ β[2]        0.989048  0.990354  0.991039  0.991766  0.993111
+   4 │ β[3]        0.75859   0.84017   0.885127  0.931194  1.03098
+   5 │ β[4]        1.0235    1.03671   1.04353   1.0502    1.06242
+

Our interpretation of odds is the same as in betting games. Anything below 1 signals a unlikely probability that yy will be 11. And anything above 1 increases the probability of yy being 11, while 1 itself is a neutral odds for yy being either 11 or 00. Since I am not a gambling man, let's talk about probabilities. So I will create a function called logodds2prob() that converts log-odds to probabilities:

+
function logodds2prob(logodds::Float64)
+    return exp(logodds) / (1 + exp(logodds))
+end
+
+@chain quantile(chain) begin
+    DataFrame
+    select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false)
+end
5×6 DataFrame
+ Row │ parameters  2.5%      25.0%     50.0%     75.0%     97.5%
+     │ Symbol      Float64   Float64   Float64   Float64   Float64
+─────┼──────────────────────────────────────────────────────────────
+   1 │ α           0.412078  0.445033  0.462234  0.478436  0.510462
+   2 │ β[1]        0.59566   0.607635  0.614497  0.620848  0.634383
+   3 │ β[2]        0.497247  0.497577  0.49775   0.497933  0.498272
+   4 │ β[3]        0.431363  0.456572  0.469532  0.482186  0.507626
+   5 │ β[4]        0.505807  0.509012  0.510649  0.512243  0.515133
+

There you go, much better now. Let's analyze our results. The intercept α is the basal switch probability which has a median value of 46%. All coefficients whose 95% credible intervals captures the value 12=0.5\frac{1}{2} = 0.5 tells that the effect on the propensity of switch is inconclusive. It is pretty much similar to a 95% credible interval that captures the 0 in the linear regression coefficients. So this rules out β[3] which is the third column of X – assoc. The other remaining 95% credible intervals can be interpreted as follows:

+
    +

  • β[1] – first column of X, arsenic, has 95% credible interval 0.595 to 0.634. This means that each increase in one unit of arsenic is related to an increase of 9.6% to 13.4% propension of switch being 1.

    + +

  • β[2] – second column of X, dist, has a 95% credible interval from 0.497 to 0.498. So we expect that each increase in one meter of dist is related to a decrease of 0.01% propension of switch being 1.

    + +

  • β[4] – fourth column of X, educ, has a 95% credible interval from 0.506 to 0.515. Each increase in one year of educ is related to an increase of 0.6% to 1.5% propension of switch being 1.

    + +
+

That's how you interpret 95% credible intervals from a quantile() output of a logistic regression Chains object converted from log-odds to probability.

+

Footnotes

+ + +
[1] + actually the logit function or the log-odds is the logarithm of the odds p1−p\frac{p}{1-p} where pp is a probability. + +
+ +

References

+

Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press.

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/08_ordinal_reg/index.html b/pages/08_ordinal_reg/index.html new file mode 100644 index 00000000..6ef37be3 --- /dev/null +++ b/pages/08_ordinal_reg/index.html @@ -0,0 +1,194 @@ + Bayesian Ordinal Regression
Bayesian Stats
Julia Logo
Code on GitHub
  1. How to deal with Ordered Discrete Dependent Variable?
    1. Cumulative Distribution Function (CDF)
    2. Log-cumulative-odds
    3. K−1K-1 Intercepts
  2. Adding Coefficients β\boldsymbol{\beta}
  3. Likelihood
  4. Bayesian Ordinal Regression
  5. Example - Esoph
  6. Footnotes
  7. References

Bayesian Ordinal Regression

Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The second is ordinal regression.

A ordinal regression behaves exactly like a linear model: it makes a prediction simply by computing a weighted sum of the independent variables X\mathbf{X} by the estimated coefficients β\boldsymbol{\beta}, but now we have not only one intercept but several intercepts αk\alpha_k for k∈Kk \in K.

We use ordinal regression when our dependent variable is ordinal. That means it has different that have a "natural order"**. Most important, the distance between values is not the same. For example, imagine a pain score scale that goes from 1 to 10. The distance between 1 and 2 is different from the distance 9 to 10. Another example is opinion pools with their ubiquitous disagree-agree range of plausible values. These are also known as Likert scale variables. The distance between "disagree" to "not agree or disagree" is different than the distance between "agree" and "strongly agree".

This assumption is what we call the "metric" assumption, also called as the equidistant assumption. Almost always when we model an ordinal dependent variable this assumption is violated. Thus, we cannot blindly employ linear regression here.

How to deal with Ordered Discrete Dependent Variable?

So, how we deal with ordered discrete responses in our dependent variable? This is similar with the previous logistic regression approach. We have to do a non-linear transformation of the dependent variable.

Cumulative Distribution Function (CDF)

In the case of ordinal regression, we need to first transform the dependent variable into a cumulative scale. We need to calculate the cumulative distribution function (CDF) of our dependent variable:

P(Y≤y)=∑i=yminyP(Y=i)P(Y \leq y) = \sum^y_{i=y_{\text{min}}} P(Y = i)

The CDF is a monotonic increasing function that depicts the probability of our random variable YY taking values less than a certain value. In our case, the discrete ordinal case, these values can be represented as positive integers ranging from 1 to the length of possible values. For instance, a 6-categorical ordered response variable will have 6 possible values, and all their CDFs will lie between 0 and 1. Furthermore, their sum will be 1; since it represents the total probability of the variable taking any of the possible values, i.e. 100%.

Log-cumulative-odds

That is still not enough, we need to apply the logit function to the CDF:

logit(x)=logistic−1(x)=ln⁡(x1−x)\mathrm{logit}(x) = \mathrm{logistic}^{-1}(x) = \ln\left(\frac{x}{1 -x}\right)

where ln⁡\ln is the natural logarithm function.

The logit is the inverse of the logistic transformation, it takes as a input any number between 0 and 1 (where a probability is the perfect candidate) and outputs a real number, which we call the log-odds.

Since we are taking the log-odds of the CDF, we can call this complete transformation as log-odds of the CDF, or log-cumulative-odds.

K−1K-1 Intercepts

Now, the next question is: what do we do with the log-cumulative-odds?

We need the log-cumulative-odds because it allows us to construct different intercepts for the possible values our ordinal dependent variable. We create an unique intercept for each possible outcome k∈Kk \in K.

Notice that the highest probable value of YY will always have a log-cumulative-odds of ∞\infty, since for p=1p=1:

ln⁡p1−p=ln⁡11−1=ln⁡0=∞\ln \frac{p}{1-p} = \ln \frac{1}{1-1} = \ln 0 = \infty

Thus, we only need K−1K-1 intercepts for a KK possible dependent variables' response values. These are known as cut points.

Each intercept implies a CDF for each value KK. This allows us to violate the equidistant assumption absent in most ordinal variables.

Each intercept implies a log-cumulative-odds for each k∈Kk \in K. We also need to undo the cumulative nature of the K−1K-1 intercepts. We can accomplish this by first converting a log-cumulative-odds back to a cumulative probability. This is done by undoing the logit transformation and applying the logistic function:

logit−1(x)=logistic(x)=11+e−x\mathrm{logit}^{-1}(x) = \mathrm{logistic}(x) = \frac{1}{1 + e^{-x}}

Then, finally, we can remove the cumulative from "cumulative probability" by subtraction of each of the kk cut points by their previous k−1k-1 cut point:

P(Y=k)=P(Y≤k)−P(Y≤k−1)P(Y=k) = P(Y \leq k) - P(Y \leq k-1)

where YY is the dependent variable and k∈Kk \in K are the cut points for each intercept.

Let me show you an example with some synthetic data.

using DataFrames
+using CairoMakie
+using AlgebraOfGraphics
+using Distributions
+using StatsFuns: logit

Here we have a discrete variable x with 6 possible ordered values as response. The values range from 1 to 6 having probability, respectively: 10%, 15%, 33%, 25%, 10%, and 7%; represented with the probs vector. The underlying distribution is represented by a Categorical distribution from Distributions.jl, which takes a vector of probabilities as parameters (probs).

For each value we are calculating:

  1. Probability Mass Function with the pdf function

  2. Cumulative Distribution Function with the cdf function

  3. Log-cumulative-odds with the logit transformation of the CDF

In the plot below there are 3 subplots:

  • Upper corner: histogram of x

  • Left lower corner: CDF of x

  • Right lower corner: log-cumulative-odds of x

let
+    probs = [0.10, 0.15, 0.33, 0.25, 0.10, 0.07]
+    dist = Categorical(probs)
+    x = 1:length(probs)
+    x_pmf = pdf.(dist, x)
+    x_cdf = cdf.(dist, x)
+    x_logodds_cdf = logit.(x_cdf)
+    df = DataFrame(; x, x_pmf, x_cdf, x_logodds_cdf)
+    labels = ["CDF", "Log-cumulative-odds"]
+    f = Figure()
+    plt1 = data(df) * mapping(:x, :x_pmf) * visual(BarPlot)
+    plt2 =
+        data(df) *
+        mapping(:x, [:x_cdf, :x_logodds_cdf]; col=dims(1) => renamer(labels)) *
+        visual(ScatterLines)
+    axis = (; xticks=1:6)
+    draw!(f[1, 2:3], plt1; axis)
+    draw!(f[2, 1:4], plt2; axis, facet=(; linkyaxes=:none))
+    f
+end
CairoMakie.Screen{SVG}
+

Ordinal Dependent Variable

As we can see, we have K−1K-1 (in our case 6−1=56-1=5) intercept values in log-cumulative-odds. You can carly see that these values they violate the equidistant assumption for metric response values. The spacing between the cut points are not the same, they vary.

Adding Coefficients β\boldsymbol{\beta}

Ok, the K−1K-1 intercepts α\boldsymbol{\alpha} are done. Now let's add coefficients to act as covariate effects in our ordinal regression model.

We transformed everything into log-odds scale so that we can add effects (coefficients multiplying a covariate) or basal rates (intercepts) together. For each k∈K−1k \in K-1, we calculate:

ϕk=αk+βixi\phi_k = \alpha_k + \beta_i x_i

where αk\alpha_k is the log-cumulative-odds for the k∈K−1k \in K-1 intercepts, βi\beta_i is the coefficient for the iith covariate xx. Finally, ϕk\phi_k represents the linear predictor for the kkth intercept.

Observe that the coefficient β\beta is being added to a log-cumulative-odds, such that, it will be expressed in a log-cumulative-odds also.

We can express it in matrix form:

ϕ=α+X⋅β\boldsymbol{\phi} = \boldsymbol{\alpha} + \mathbf{X} \cdot \boldsymbol{\beta}

where ϕ\boldsymbol{\phi}, α\boldsymbol{\alpha} and β\boldsymbol{\beta} are vectors and X\mathbf{X} is the data matrix where each row is an observation and each column a covariate.

This still obeys the ordered constraint on the dependent variable possible values.

How to Interpret Coefficient β\boldsymbol{\beta}?

Now, suppose we have found our ideal value for our β\boldsymbol{\beta}. How we would interpret our β\boldsymbol{\beta} estimated values?

First, to recap, β\boldsymbol{\beta} measures the strength of association between the covariate x\mathbf{x} and dependent variable y\mathbf{y}. But, β\boldsymbol{\beta} is nested inside a non-linear transformation called logistic function:

logistic(β)=11+e−β\mathrm{logistic}(\boldsymbol{\beta}) = \frac{1}{1 + e^{-\boldsymbol{\beta}}}

So, our first step is to undo the logistic function. There is a function that is called logit function that is the inverse of the logistic function:

logistic−1(x)=logit(x)=ln⁡(x1−x)\mathrm{logistic}^{-1}(x) = \mathrm{logit}(x) = \ln\left(\frac{x}{1 -x}\right)

where ln⁡\ln is the natural logarithm function.

If we analyze closely the logit function we can find that inside the ln⁡\ln there is a disguised odds in the x1−x\frac{x}{1 -x}. Hence, our β\boldsymbol{\beta} can be interpreted as the logarithm of the odds, or in short form: the log-odds.

We already saw how odds, log-odds, and probability are related in the previous logistic regression tutorial. So you might want to go back there to get the full explanation.

The log-odds are the key to interpret coefficient β\boldsymbol{\beta}**. Any positive value of β\beta means that there exists a positive association between xx and yy, while any negative values indicate a negative association between xx and yy. Values close to 0 demonstrates the lack of association between xx and yy.

Likelihood

We have almost everything we need for our ordinal regression. We are only missing a final touch. Currently our logistic function outputs a vector of probabilities that sums to 1.

All of the intercepts αk\alpha_k along with the coefficients βi\beta_i are in log-cumulative-odds scale. If we apply the logistic function to the linear predictors ϕk\phi_k we get K−1K-1 probabilities: one for each ϕk\phi_k

We need a likelihood that can handle a vector of probabilities and outputs a single discrete value. The categorical distribution is the perfect candidate.

Bayesian Ordinal Regression

Now we have all the components for our Bayesian ordinal regression specification:

y∼Categorical(p)p=Logistic(ϕ)ϕ=α+X⋅βα1=CDF(y1)αk=CDF(yk)−CDF(yk−1)for1<k<K−1αK−1=1−CDF(yK−1) \begin{aligned} \mathbf{y} &\sim \text{Categorical}(\mathbf{p}) \\ \mathbf{p} &= \text{Logistic}(\boldsymbol{\phi}) \\ \boldsymbol{\phi} &= \boldsymbol{\alpha} + \mathbf{X} \cdot \boldsymbol{\beta} \\ \alpha_1 &= \text{CDF}(y_1) \\ \alpha_k &= \text{CDF}(y_k) - \text{CDF}(y_{k-1}) \quad \text{for} \quad 1 < k < K-1 \\ \alpha_{K-1} &= 1 - \text{CDF}(y_{K-1}) \end{aligned}

where:

  • y\mathbf{y} – ordered discrete dependent variable.

  • p\mathbf{p} – probability vector of length KK.

  • KK – number of possible values y\mathbf{y} can take, i.e. number of ordered discrete values.

  • ϕ\boldsymbol{\phi} – log-cumulative-odds, i.e. cut points considering the intercepts and covariates effect.

  • αk\alpha_k – intercept in log-cumulative-odds for each k∈K−1k \in K-1.

  • X\mathbf{X} – covariate data matrix.

  • β\boldsymbol{\beta} – coefficient vector of the same length as the number of columns in X\mathbf{X}.

  • logistic\mathrm{logistic} – logistic function.

  • CDF\mathrm{CDF} – cumulative distribution function.

What remains is to specify the model parameters' prior distributions:

  • Prior Distribution of α\boldsymbol{\alpha} – Knowledge we possess regarding the model's intercepts.

  • Prior Distribution of β\boldsymbol{\beta} – Knowledge we possess regarding the model's independent variables' coefficients.

Our goal is to instantiate an ordinal regression with the observed data (y\mathbf{y} and X\mathbf{X}) and find the posterior distribution of our model's parameters of interest (α\boldsymbol{\alpha} and β\boldsymbol{\beta}). This means to find the full posterior distribution of:

P(θ∣y)=P(α,β∣y) P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\boldsymbol{\alpha}, \boldsymbol{\beta} \mid \mathbf{y})

Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error parameter σ\sigma in our ordinal regression. This is due to the Categorical distribution not having a "scale" parameter such as the σ\sigma parameter in the Gaussian/normal distribution.

This is easily accomplished with Turing:

using Turing
+using Bijectors
+using LazyArrays
+using LinearAlgebra
+using Random: seed!
+using Bijectors: transformed, OrderedBijector
+seed!(123)
+
+@model function ordreg(X, y; predictors=size(X, 2), ncateg=maximum(y))
+    #priors
+    cutpoints ~ transformed(filldist(TDist(3) * 5, ncateg - 1), OrderedBijector())
+    β ~ filldist(TDist(3) * 2.5, predictors)
+
+    #likelihood
+    return y ~ arraydist([OrderedLogistic(X[i, :] ⋅ β, cutpoints) for i in 1:length(y)])
+end;

First, let's deal with the new stuff in our model: the Bijectors.jl's transformed and OrderedBijector. As I've said in the 4. How to use Turing, Turing has a rich ecosystem of packages. Bijectors implements a set of functions for transforming constrained random variables (e.g. simplexes, intervals) to Euclidean space. Here we are defining cutpoints as a ncateg - 1 vector of Student-tt distributions with mean 0, standard deviation 5 and degrees of freedom ν=3\nu = 3. Remember that we only need K−1K-1 cutpoints for all of our KK intercepts. And we are also constraining it to be an ordered vector with transformed(d, OrderedBijector), such that for all cutpoints cic_i we have c1<c2<...ck−1c_1 < c_2 < ... c_{k-1}.

As before, we are giving β\boldsymbol{\beta} a very weakly informative priors of a Student-tt distribution centered on 0 with variance 1 and degrees of freedom ν=3\nu = 3. That wide-tailed tt distribution will cover all possible values for our coefficients. Remember the Student-tt also has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). Also the filldist() is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution.

Finally, in the likelihood, Turing's arraydist() function wraps an array of distributions returning a new distribution sampling from the individual distributions. And we use some indexing inside an array literal.

Example - Esoph

For our example, I will use a famous dataset called esoph (Breslow & Day, 1980), which is data from a case-control study of (o)esophageal cancer in Ille-et-Vilaine, France. It has records for 88 age/alcohol/tobacco combinations:

  • agegp: Age group

    • 1: 25-34 years

    • 2: 35-44 years

    • 3: 45-54 years

    • 4: 55-64 years

    • 5: 65-74 years

    • 6: 75+ years

  • alcgp: Alcohol consumption

    • 1: 0-39 g/day

    • 2: 40-79 g/day

    • 3: 80-119 g/day

    • 4: 120+ g/day

  • tobgp: Tobacco consumption

    • 1: 0-9 g/day

    • 2: 10-19 g/day

    • 3: 20-29 g/day

    • 4: 30+ g/day

  • ncases: Number of cases

  • ncontrols: Number of controls

Ok let's read our data with CSV.jl and output into a DataFrame from DataFrames.jl:

using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/esoph.csv"
+esoph = CSV.read(HTTP.get(url).body, DataFrame)
88×5 DataFrame
+ Row │ agegp    alcgp      tobgp     ncases  ncontrols
+     │ String7  String15   String15  Int64   Int64
+─────┼─────────────────────────────────────────────────
+   1 │ 25-34    0-39g/day  0-9g/day       0         40
+   2 │ 25-34    0-39g/day  10-19          0         10
+   3 │ 25-34    0-39g/day  20-29          0          6
+   4 │ 25-34    0-39g/day  30+            0          5
+   5 │ 25-34    40-79      0-9g/day       0         27
+   6 │ 25-34    40-79      10-19          0          7
+   7 │ 25-34    40-79      20-29          0          4
+   8 │ 25-34    40-79      30+            0          7
+   9 │ 25-34    80-119     0-9g/day       0          2
+  10 │ 25-34    80-119     10-19          0          1
+  11 │ 25-34    80-119     30+            0          2
+  12 │ 25-34    120+       0-9g/day       0          1
+  13 │ 25-34    120+       10-19          1          0
+  14 │ 25-34    120+       20-29          0          1
+  15 │ 25-34    120+       30+            0          2
+  16 │ 35-44    0-39g/day  0-9g/day       0         60
+  17 │ 35-44    0-39g/day  10-19          1         13
+  18 │ 35-44    0-39g/day  20-29          0          7
+  19 │ 35-44    0-39g/day  30+            0          8
+  20 │ 35-44    40-79      0-9g/day       0         35
+  21 │ 35-44    40-79      10-19          3         20
+  22 │ 35-44    40-79      20-29          1         13
+  23 │ 35-44    40-79      30+            0          8
+  24 │ 35-44    80-119     0-9g/day       0         11
+  25 │ 35-44    80-119     10-19          0          6
+  26 │ 35-44    80-119     20-29          0          2
+  27 │ 35-44    80-119     30+            0          1
+  28 │ 35-44    120+       0-9g/day       2          1
+  29 │ 35-44    120+       10-19          0          3
+  30 │ 35-44    120+       20-29          2          2
+  31 │ 45-54    0-39g/day  0-9g/day       1         45
+  32 │ 45-54    0-39g/day  10-19          0         18
+  33 │ 45-54    0-39g/day  20-29          0         10
+  34 │ 45-54    0-39g/day  30+            0          4
+  35 │ 45-54    40-79      0-9g/day       6         32
+  36 │ 45-54    40-79      10-19          4         17
+  37 │ 45-54    40-79      20-29          5         10
+  38 │ 45-54    40-79      30+            5          2
+  39 │ 45-54    80-119     0-9g/day       3         13
+  40 │ 45-54    80-119     10-19          6          8
+  41 │ 45-54    80-119     20-29          1          4
+  42 │ 45-54    80-119     30+            2          2
+  43 │ 45-54    120+       0-9g/day       4          0
+  44 │ 45-54    120+       10-19          3          1
+  45 │ 45-54    120+       20-29          2          1
+  46 │ 45-54    120+       30+            4          0
+  47 │ 55-64    0-39g/day  0-9g/day       2         47
+  48 │ 55-64    0-39g/day  10-19          3         19
+  49 │ 55-64    0-39g/day  20-29          3          9
+  50 │ 55-64    0-39g/day  30+            4          2
+  51 │ 55-64    40-79      0-9g/day       9         31
+  52 │ 55-64    40-79      10-19          6         15
+  53 │ 55-64    40-79      20-29          4         13
+  54 │ 55-64    40-79      30+            3          3
+  55 │ 55-64    80-119     0-9g/day       9          9
+  56 │ 55-64    80-119     10-19          8          7
+  57 │ 55-64    80-119     20-29          3          3
+  58 │ 55-64    80-119     30+            4          0
+  59 │ 55-64    120+       0-9g/day       5          5
+  60 │ 55-64    120+       10-19          6          1
+  61 │ 55-64    120+       20-29          2          1
+  62 │ 55-64    120+       30+            5          1
+  63 │ 65-74    0-39g/day  0-9g/day       5         43
+  64 │ 65-74    0-39g/day  10-19          4         10
+  65 │ 65-74    0-39g/day  20-29          2          5
+  66 │ 65-74    0-39g/day  30+            0          2
+  67 │ 65-74    40-79      0-9g/day      17         17
+  68 │ 65-74    40-79      10-19          3          7
+  69 │ 65-74    40-79      20-29          5          4
+  70 │ 65-74    80-119     0-9g/day       6          7
+  71 │ 65-74    80-119     10-19          4          8
+  72 │ 65-74    80-119     20-29          2          1
+  73 │ 65-74    80-119     30+            1          0
+  74 │ 65-74    120+       0-9g/day       3          1
+  75 │ 65-74    120+       10-19          1          1
+  76 │ 65-74    120+       20-29          1          0
+  77 │ 65-74    120+       30+            1          0
+  78 │ 75+      0-39g/day  0-9g/day       1         17
+  79 │ 75+      0-39g/day  10-19          2          4
+  80 │ 75+      0-39g/day  30+            1          2
+  81 │ 75+      40-79      0-9g/day       2          3
+  82 │ 75+      40-79      10-19          1          2
+  83 │ 75+      40-79      20-29          0          3
+  84 │ 75+      40-79      30+            1          0
+  85 │ 75+      80-119     0-9g/day       1          0
+  86 │ 75+      80-119     10-19          1          0
+  87 │ 75+      120+       0-9g/day       2          0
+  88 │ 75+      120+       10-19          1          0

Now let's us instantiate our model with the data. But here I need to do some data wrangling to create the data matrix X. Specifically, I need to convert all of the categorical variables to integer values, representing the ordinal values of both our independent and also dependent variables:

using CategoricalArrays
+
+DataFrames.transform!(
+    esoph,
+    :agegp =>
+        x -> categorical(
+            x; levels=["25-34", "35-44", "45-54", "55-64", "65-74", "75+"], ordered=true
+        ),
+    :alcgp =>
+        x -> categorical(x; levels=["0-39g/day", "40-79", "80-119", "120+"], ordered=true),
+    :tobgp =>
+        x -> categorical(x; levels=["0-9g/day", "10-19", "20-29", "30+"], ordered=true);
+    renamecols=false,
+)
+DataFrames.transform!(
+    esoph, [:agegp, :alcgp, :tobgp] .=> ByRow(levelcode); renamecols=false
+)
+
+X = Matrix(select(esoph, [:agegp, :alcgp]))
+y = esoph[:, :tobgp]
+model = ordreg(X, y);

And, finally, we will sample from the Turing model. We will be using the default NUTS() sampler with 1_000 samples, with 4 Markov chains using multiple threads MCMCThreads():

chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4)
+summarystats(chain)
Summary Statistics
+    parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+        Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+  cutpoints[1]   -1.3806    0.6268    0.0119   2788.6718   2472.5534    1.0009       69.0744
+  cutpoints[2]   -0.2021    0.6072    0.0113   2897.1099   2438.5542    1.0013       71.7604
+  cutpoints[3]    0.8493    0.6203    0.0114   2962.0272   2770.1618    1.0021       73.3684
+          β[1]   -0.0696    0.1161    0.0021   3108.6746   2347.4018    1.0007       77.0008
+          β[2]   -0.0658    0.1691    0.0030   3128.0391   2909.9287    1.0016       77.4804
+

We had no problem with the Markov chains as all the rhat are well below 1.01 (or above 0.99). Note that the coefficients are in log-odds scale. They are the natural log of the odds[1], and odds is defined as:

odds=p1−p \text{odds} = \frac{p}{1-p}

where pp is a probability. So log-odds is defined as:

log⁡(odds)=log⁡(p1−x) \log(\text{odds}) = \log \left( \frac{p}{1-x} \right)

So in order to get odds from a log-odds we must undo the log operation with a exponentiation. This translates to:

odds=exp⁡(log⁡(odds)) \text{odds} = \exp ( \log ( \text{odds} ))

We can do this with a transformation in a DataFrame constructed from a Chains object:

using Chain
+
+@chain quantile(chain) begin
+    DataFrame
+    select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false)
+end
5×6 DataFrame
+ Row │ parameters    2.5%       25.0%     50.0%     75.0%     97.5%
+     │ Symbol        Float64    Float64   Float64   Float64   Float64
+─────┼─────────────────────────────────────────────────────────────────
+   1 │ cutpoints[1]  0.0738932  0.163178  0.252302  0.389008  0.845115
+   2 │ cutpoints[2]  0.254298   0.534801  0.818166  1.2405    2.71577
+   3 │ cutpoints[3]  0.704014   1.52676   2.32062   3.56916   8.04289
+   4 │ β[1]          0.745092   0.863281  0.933127  1.00918   1.17439
+   5 │ β[2]          0.677071   0.832166  0.935842  1.05535   1.29072

Our interpretation of odds is the same as in betting games. Anything below 1 signals a unlikely probability that yy will be 11. And anything above 1 increases the probability of yy being 11, while 1 itself is a neutral odds for yy being either 11 or 00. Since I am not a gambling man, let's talk about probabilities. So I will create a function called logodds2prob() that converts log-odds to probabilities:

function logodds2prob(logodds::Float64)
+    return exp(logodds) / (1 + exp(logodds))
+end
+
+@chain quantile(chain) begin
+    DataFrame
+    select(_, :parameters, names(_, r"%") .=> ByRow(logodds2prob); renamecols=false)
+end
5×6 DataFrame
+ Row │ parameters    2.5%       25.0%     50.0%     75.0%     97.5%
+     │ Symbol        Float64    Float64   Float64   Float64   Float64
+─────┼─────────────────────────────────────────────────────────────────
+   1 │ cutpoints[1]  0.0688087  0.140286  0.201471  0.280062  0.458028
+   2 │ cutpoints[2]  0.202741   0.34845   0.449995  0.553671  0.730877
+   3 │ cutpoints[3]  0.41315    0.604236  0.698851  0.781141  0.889416
+   4 │ β[1]          0.426964   0.463312  0.482703  0.502285  0.540101
+   5 │ β[2]          0.403722   0.454198  0.483429  0.513466  0.563457

There you go, much better now. Let's analyze our results. The cutpoints is the basal rate of the probability of our dependent variable having values less than a certain value. For example the cutpoint for having values less than 2 which its code represents the tobacco consumption of 10-19 g/day has a median value of 20%.

Now let's take a look at our coefficients All coefficients whose 95% credible intervals captures the value 12=0.5\frac{1}{2} = 0.5 tells that the effect on the propensity of tobacco consumption is inconclusive. It is pretty much similar to a 95% credible interval that captures the 0 in the linear regression coefficients.

That's how you interpret 95% credible intervals from a quantile() output of a ordinal regression Chains object converted from log-odds to probability.

Footnotes

[1] actually the logit function or the log-odds is the logarithm of the odds p1−p\frac{p}{1-p} where pp is a probability.

References

Breslow, N. E. & Day, N. E. (1980). Statistical Methods in Cancer Research. Volume 1: The Analysis of Case-Control Studies. IARC Lyon / Oxford University Press.

Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language.
\ No newline at end of file diff --git a/pages/09_count_reg/index.html b/pages/09_count_reg/index.html new file mode 100644 index 00000000..5456d0d8 --- /dev/null +++ b/pages/09_count_reg/index.html @@ -0,0 +1,168 @@ + Bayesian Regression with Count Data
Bayesian Stats
Julia Logo
Code on GitHub
  1. Comparison with Linear Regression
  2. Bayesian Regression with Count Data
    1. Using Poisson Likelihood
    2. Using Negative Binomial Likelihood
  3. Example - Roaches Extermination
    1. Poisson Regression
    2. Negative Binomial Regression
  4. References

Bayesian Regression with Count Data

Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The third of these is regression with count data (also called Poisson regression).

A regression with count data behaves exactly like a linear model: it makes a prediction simply by computing a weighted sum of the independent variables X\mathbf{X} by the estimated coefficients β\boldsymbol{\beta}, plus an intercept α\alpha. However, instead of returning a continuous value yy, such as linear regression, it returns the natural log of yy.

We use regression with count data when our dependent variable is restricted to positive integers, i.e. y∈Z+y \in \mathbb{Z}^+. See the figure below for a graphical intuition of the exponential function:

using CairoMakie
+
+f, ax, l = lines(-6 .. 6, exp; axis=(xlabel=L"x", ylabel=L"e^x"))

Exponential Function

As we can see, the exponential function is basically a mapping of any real number to a positive real number in the range between 0 and +∞+\infty (non-inclusive):

Exponential(x)={R∈[−∞,+∞]}→{R∈[0,+∞]} \text{Exponential}(x) = \{ \mathbb{R} \in [- \infty , + \infty] \} \to \{ \mathbb{R} \in [0, + \infty] \}

That is, the exponential function is the ideal candidate for when we need to convert something continuous without restrictions to something continuous restricted to taking positive values only. That is why it is used when we need a model to have a positive-only dependent variable. This is the case of a dependent variable for count data.

Comparison with Linear Regression

Linear regression follows the following mathematical formulation:

Linear=θ0+θ1x1+θ2x2+…+θnxn \text{Linear} = \theta_0 + \theta_1 x_1 + \theta_2 x_2 + \ldots + \theta_n x_n

  • θ\theta - model parameters

    • θ0\theta_0 - intercept

    • θ1,θ2,…\theta_1, \theta_2, \dots - independent variables x1,x2,…x_1, x_2, \dots coefficients

  • nn - total number of independent variables

Regression with count data would add the exponential function to the linear term:

log⁡(y)=θ0⋅θ1x1⋅θ2x2⋅⋯⋅θnxn \log(y) = \theta_0 \cdot \theta_1 x_1 \cdot \theta_2 x_2 \cdot \dots \cdot \theta_n x_n

which is the same as:

y=e(θ0+θ1x1+θ2x2+…+θnxn) y = e^{(\theta_0 + \theta_1 x_1 + \theta_2 x_2 + \ldots + \theta_n x_n)}

Bayesian Regression with Count Data

We can model regression with count data in two ways. The first option with a Poisson likelihood function and the second option with a negative binomial likelihood function.

With the Poisson likelihood we model a discrete and positive dependent variable yy by assuming that a given number of independent yy events will occur with a known constant average rate.

In a negative binomial likelihood, model a discrete and positive dependent variable yy by assuming that a given number nn of independent yy events will occur by asking a yes-no question for each nn with probability pp until kk success(es) is obtained. Note that it becomes identical to the Poisson likelihood when at the limit of k→∞k \to \infty. This makes the negative binomial a robust option to replace a Poisson likelihood to model phenomena with a overdispersion (excess expected variation in data). This occurs due to the Poisson likelihood making an assumption that the dependent variable yy has the same mean and variance, while in the negative binomial likelihood the mean and the variance do not need to be equal.

Using Poisson Likelihood

y∼Poisson(e(α+X⋅β))α∼Normal(μα,σα)β∼Normal(μβ,σβ) \begin{aligned} \mathbf{y} &\sim \text{Poisson}\left( e^{(\alpha + \mathbf{X} \cdot \boldsymbol{\beta})} \right) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \end{aligned}

where:

  • y\mathbf{y} – discrete and positive dependent variable.

  • ee – exponential.

  • α\alpha – intercept.

  • β\boldsymbol{\beta} – coefficient vector.

  • X\mathbf{X} – data matrix.

As we can see, the linear predictor α+X⋅β\alpha + \mathbf{X} \cdot \boldsymbol{\beta} is the logarithm of the value of yy. So we need to apply the exponential function the values of the linear predictor:

log⁡(y)=α+X⋅βy=eαX⋅βy=eα⋅e(X⋅β) \begin{aligned} \log(\mathbf{y}) &= \alpha + \mathbf{X} \cdot \boldsymbol{\beta} \\ \mathbf{y} &= e^{\alpha \mathbf{X} \cdot \boldsymbol{\beta}} \\ \mathbf{y} &= e^{\alpha} \cdot e^{\left( \mathbf{X} \cdot \boldsymbol{\beta} \right) } \end{aligned}

The intercept α\alpha and coefficients β\boldsymbol{\beta} are only interpretable after exponentiation.

What remains is to specify the model parameters' prior distributions:

  • Prior Distribution of α\alpha – Knowledge we possess regarding the model's intercept.

  • Prior Distribution of β\boldsymbol{\beta} – Knowledge we possess regarding the model's independent variables' coefficients.

Our goal is to instantiate a regression with count data using the observed data (y\mathbf{y} and X\mathbf{X}) and find the posterior distribution of our model's parameters of interest (α\alpha and β\boldsymbol{\beta}). This means to find the full posterior distribution of:

P(θ∣y)=P(α,β∣y) P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta} \mid \mathbf{y})

Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error parameter σ\sigma in our regression with count data. This is due to the Poisson not having a "scale" parameter such as the σ\sigma parameter in the Gaussian/normal distribution.

This is easily accomplished with Turing:

using Turing
+using LazyArrays
+using Random: seed!
+seed!(123)
+
+@model function poissonreg(X, y; predictors=size(X, 2))
+    #priors
+    α ~ Normal(0, 2.5)
+    β ~ filldist(TDist(3), predictors)
+
+    #likelihood
+    return y ~ arraydist(LazyArray(@~ LogPoisson.(α .+ X * β)))
+end;

Here I am specifying very weakly informative priors:

  • α∼Normal(0,2.5)\alpha \sim \text{Normal}(0, 2.5) – This means a normal distribution centered on 0 with a standard deviation of 2.5. That prior should with ease cover all possible values of α\alpha. Remember that the normal distribution has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty).

  • β∼Student-t(0,1,3)\boldsymbol{\beta} \sim \text{Student-}t(0,1,3) – The predictors all have a prior distribution of a Student-tt distribution centered on 0 with variance 1 and degrees of freedom ν=3\nu = 3. That wide-tailed tt distribution will cover all possible values for our coefficients. Remember the Student-tt also has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). Also the filldist() is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution.

Turing's arraydist() function wraps an array of distributions returning a new distribution sampling from the individual distributions. And the LazyArrays' LazyArray() constructor wrap a lazy object that wraps a computation producing an array to an array. Last, but not least, the macro @~ creates a broadcast and is a nice short hand for the familiar dot . broadcasting operator in Julia. This is an efficient way to tell Turing that our y vector is distributed lazily as a LogPoisson broadcasted to α added to the product of the data matrix X and β coefficient vector. LogPoisson is Turing's efficient distribution that already apply exponentiation to all the linear predictors.

Using Negative Binomial Likelihood

y∼Negative Binomial(e(α+X⋅β),ϕ)ϕ∼Gamma(0.01,0.01)α∼Normal(μα,σα)β∼Normal(μβ,σβ) \begin{aligned} \mathbf{y} &\sim \text{Negative Binomial}\left( e^{(\alpha + \mathbf{X} \cdot \boldsymbol{\beta})}, \phi \right) \\ \phi &\sim \text{Gamma}(0.01, 0.01) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \end{aligned}

where:

  • y\mathbf{y} – discrete and positive dependent variable.

  • ee – exponential.

  • ϕ\phi – dispersion.

  • ϕ−\phi^- – reciprocal dispersion.

  • α\alpha – intercept.

  • β\boldsymbol{\beta} – coefficient vector.

  • X\mathbf{X} – data matrix.

Note that when we compare with the Poisson model, we have a new parameter ϕ\phi that parameterizes the negative binomial likelihood. This parameter is the probability of successes pp of the negative binomial distribution and we generally use a Gamma distribution as prior so that the inverse of ϕ\phi which is ϕ−\phi^- fulfills the function of a "reciprocal dispersion" parameter. Most of the time we use a weakly informative prior of the parameters shape α=0.01\alpha = 0.01 and scale θ=0.01\theta = 0.01 (Gelman et al., 2013; 2020). But you can also use ϕ−∼Exponential(1)\phi^- \sim \text{Exponential}(1) as prior (McElreath, 2020).

Here is what a Gamma(0.01,0.01)\text{Gamma}(0.01, 0.01) looks like:

using Distributions
+f, ax, l = lines(
+    Gamma(0.01, 0.01);
+    linewidth=2,
+    axis=(xlabel=L"\phi", ylabel="Density", limits=(0, 0.03, nothing, nothing)),
+)

Gamma Distribution with α=0.01\alpha = 0.01 and θ=0.01\theta = 0.01

In both likelihood options, what remains is to specify the model parameters' prior distributions:

  • Prior Distribution of α\alpha – Knowledge we possess regarding the model's intercept.

  • Prior Distribution of β\boldsymbol{\beta} – Knowledge we possess regarding the model's independent variables' coefficients.

Our goal is to instantiate a regression with count data using the observed data (y\mathbf{y} and X\mathbf{X}) and find the posterior distribution of our model's parameters of interest (α\alpha and β\boldsymbol{\beta}). This means to find the full posterior distribution of:

P(θ∣y)=P(α,β∣y) P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta} \mid \mathbf{y})

Note that contrary to the linear regression, which used a Gaussian/normal likelihood function, we don't have an error parameter σ\sigma in our regression with count data. This is due to neither the Poisson nor negative binomial distributions having a "scale" parameter such as the σ\sigma parameter in the Gaussian/normal distribution.

Alternative Negative Binomial Parameterization

One last thing before we get into the details of the negative binomial distribution is to consider an alternative parameterization. Julia's Distributions.jl and, consequently, Turing's parameterization of the negative binomial distribution follows the following the Wolfram reference:

Negative-Binomial(k∣r,p)∼Γ(k+r)k!Γ(r)pr(1−p)k,for k=0,1,2,… \text{Negative-Binomial}(k \mid r, p) \sim \frac{\Gamma(k+r)}{k! \Gamma(r)} p^r (1 - p)^k, \quad \text{for } k = 0, 1, 2, \ldots

where:

  • kk – number of failures before the rrth success in a sequence of independent Bernoulli trials

  • rr – number of successes

  • pp – probability of success in an individual Bernoulli trial

This is not ideal for most of the modeling situations that we would employ the negative binomial distribution. In particular, we want to have a parameterization that is more appropriate for count data. What we need is the familiar mean (or location) and variance (or scale) parameterization. If we look in Stan's documentation for the neg_binomial_2 function, we have the following two equations:

μ=r(1−p)pμ+μ2ϕ=r(1−p)p2 \begin{aligned} \mu &= \frac{r (1 - p)}{p} \\ \mu + \frac{\mu^2}{\phi} &= \frac{r (1 - p)}{p^2} \end{aligned}

With a little bit of algebra, we can substitute the first equation of (11) into the right hand side of the second equation and get the following:

μ+μ2ϕ=μp1+μϕ=1pp=11+μϕ \begin{aligned} \mu + \frac{\mu^2}{\phi} &= \frac{μ}{p} \\ 1 + \frac{\mu}{\phi} &= \frac{1}{p} \\ p &= \frac{1}{\frac{1 + \mu}{\phi}} \end{aligned}

Then in (11) we have:

μ=r(1−(11+μϕ))⋅(1+μϕ)μ=r((1+μϕ)−1)r=ϕ \begin{aligned} \mu &= r \left(1 - \left( \frac{1}{\frac{1 + \mu}{\phi}} \right) \right) \cdot \left(1 + \frac{\mu}{\phi} \right) \\ \mu &= r \left( \left(1 + \frac{\mu}{\phi} \right) - 1 \right) \\ r &= \phi \end{aligned}

Hence, the resulting map is Negative-Binomial(μ,ϕ)≡Negative-Binomial(r=ϕ,p=11+μϕ)\text{Negative-Binomial}(\mu, \phi) \equiv \text{Negative-Binomial} \left( r = \phi, p = \frac{1}{\frac{1 + \mu}{\phi}} \right). I would like to point out that this implementation was done by Tor Fjelde in a COVID-19 model with the code available in GitHub. So we can use this parameterization in our negative binomial regression model. But first, we need to define an alternative negative binomial distribution function:

function NegativeBinomial2(μ, ϕ)
+    p = 1 / (1 + μ / ϕ)
+    p = p > 0 ? p : 1e-4 # numerical stability
+    r = ϕ
+
+    return NegativeBinomial(r, p)
+end
NegativeBinomial2 (generic function with 1 method)
+

Now we create our Turing model with the alternative NegBinomial2 parameterization:

+
@model function negbinreg(X, y; predictors=size(X, 2))
+    #priors
+    α ~ Normal(0, 2.5)
+    β ~ filldist(TDist(3), predictors)
+    ϕ⁻ ~ Gamma(0.01, 0.01)
+    ϕ = 1 / ϕ⁻
+
+    #likelihood
+    return y ~ arraydist(LazyArray(@~ NegativeBinomial2.(exp.(α .+ X * β), ϕ)))
+end;
+

Here I am also specifying very weakly informative priors:

+
    +

  • α∼Normal(0,2.5)\alpha \sim \text{Normal}(0, 2.5) – This means a normal distribution centered on 0 with a standard deviation of 2.5. That prior should with ease cover all possible values of α\alpha. Remember that the normal distribution has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty).

    + +

  • β∼Student-t(0,1,3)\boldsymbol{\beta} \sim \text{Student-}t(0,1,3) – The predictors all have a prior distribution of a Student-tt distribution centered on 0 with variance 1 and degrees of freedom ν=3\nu = 3. That wide-tailed tt distribution will cover all possible values for our coefficients. Remember the Student-tt also has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). Also the filldist() is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution.

    + +

  • ϕ∼Exponential(1)\phi \sim \text{Exponential}(1) – overdispersion parameter of the negative binomial distribution.

    + +
+

Turing's arraydist() function wraps an array of distributions returning a new distribution sampling from the individual distributions. And the LazyArrays' LazyArray() constructor wrap a lazy object that wraps a computation producing an array to an array. Last, but not least, the macro @~ creates a broadcast and is a nice short hand for the familiar dot . broadcasting operator in Julia. This is an efficient way to tell Turing that our y vector is distributed lazily as a NegativeBinomial2 broadcasted to α added to the product of the data matrix X and β coefficient vector. Note that NegativeBinomial2 does not apply exponentiation so we had to include the exp.() broadcasted function to all the linear predictors.

+

Example - Roaches Extermination

+

For our example, I will use a famous dataset called roaches (Gelman & Hill, 2007), which is data on the efficacy of a pest management system at reducing the number of roaches in urban apartments. It has 262 observations and the following variables:

+
    +

  • y – number of roaches caught.

    + +

  • roach1 – pretreatment number of roaches.

    + +

  • treatment – binary/dummy (0 or 1) for treatment indicator.

    + +

  • senior – binary/dummy (0 or 1) for only elderly residents in building.

    + +

  • exposure2 – number of days for which the roach traps were used

    + +
+

Ok let's read our data with CSV.jl and output into a DataFrame from DataFrames.jl:

+
using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/roaches.csv"
+roaches = CSV.read(HTTP.get(url).body, DataFrame)
+describe(roaches)
5×7 DataFrame
+ Row │ variable   mean       min   median   max        nmissing  eltype
+     │ Symbol     Float64    Real  Float64  Real       Int64     DataType
+─────┼────────────────────────────────────────────────────────────────────
+   1 │ y          25.6489     0        3.0  357               0  Int64
+   2 │ roach1     42.1935     0.0      7.0  450.0             0  Float64
+   3 │ treatment   0.603053   0        1.0    1               0  Int64
+   4 │ senior      0.305344   0        0.0    1               0  Int64
+   5 │ exposure2   1.02105    0.2      1.0    4.28571         0  Float64
+

As you can see from the describe() output the average number of roaches caught by the pest management system is around 26 roaches. The average number of roaches pretreatment is around 42 roaches (oh boy...). 30% of the buildings has only elderly residents and 60% of the buildings received a treatment by the pest management system. Also note that the traps were set in general for only 1 day and it ranges from 0.2 days (almost 5 hours) to 4.3 days (which is approximate 4 days and 7 hours).

+

Poisson Regression

+

Let's first run the Poisson regression. First, we instantiate our model with the data:

+
X = Matrix(select(roaches, Not(:y)))
+y = roaches[:, :y]
+model_poisson = poissonreg(X, y);
+

And, finally, we will sample from the Turing model. We will be using the default NUTS() sampler with 1_000 samples, with 4 Markov chains using multiple threads MCMCThreads():

+
chain_poisson = sample(model_poisson, NUTS(), MCMCThreads(), 1_000, 4)
+summarystats(chain_poisson)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α    2.9867    0.1526    0.0257    307.5895     81.9752    1.0121        9.3558
+        β[1]    0.0065    0.0002    0.0000   3084.6402   2620.6876    1.0023       93.8237
+        β[2]   -0.5007    0.1066    0.0179    277.4972     68.4987    1.0220        8.4405
+        β[3]   -0.3620    0.1247    0.0215    319.4964     79.5812    1.0136        9.7179
+        β[4]    0.1265    0.2666    0.0452    310.0447     75.1405    1.0124        9.4304
+
+

We had no problem with the Markov chains as all the rhat are well below 1.01 (or above 0.99). Note that the coefficients are in log scale. So we need to apply the exponential function to them. We can do this with a transformation in a DataFrame constructed from a Chains object:

+
using Chain
+
+@chain quantile(chain_poisson) begin
+    DataFrame
+    select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false)
+end
5×6 DataFrame
+ Row │ parameters  2.5%       25.0%      50.0%      75.0%      97.5%
+     │ Symbol      Float64    Float64    Float64    Float64    Float64
+─────┼───────────────────────────────────────────────────────────────────
+   1 │ α           17.8944    18.8715    19.4401    20.0227    21.6618
+   2 │ β[1]         1.00636    1.00648    1.00655    1.00661    1.00672
+   3 │ β[2]         0.569218   0.587878   0.598309   0.608326   0.642261
+   4 │ β[3]         0.640725   0.669002   0.685442   0.701448   0.74567
+   5 │ β[4]         1.07174    1.14706    1.17526    1.20443    1.26015
+

Let's analyze our results. The intercept α is the basal number of roaches caught y and has a median value of 19.4 roaches caught. The remaining 95% credible intervals for the βs can be interpreted as follows:

+
    +

  • β[1] – first column of X, roach1, has 95% credible interval 1.01 to 1.01. This means that each increase in one unit of roach1 is related to an increase of 1% more roaches caught.

    + +

  • β[2] – second column of X, treatment, has 95% credible interval 0.57 to 0.63. This means that if an apartment was treated with the pest management system then we expect an decrease of around 40% roaches caught.

    + +

  • β[3] – third column of X, senior, has a 95% credible interval from 0.64 to 0.73. This means that if an apartment building has only elderly residents then we expect an decrease of around 30% roaches caught.

    + +

  • β[4] – fourth column of X, exposure2, has a 95% credible interval from 1.09 to 1.26. Each increase in one day for the exposure of traps in an apartment we expect an increase of between 9% to 26% roaches caught.

    + +
+

That's how you interpret 95% credible intervals from a quantile() output of a regression with count data Chains object converted from a log scale.

+

Negative Binomial Regression

+

Let's now run the negative binomial regression.

+
model_negbin = negbinreg(X, y);
+

We will also default NUTS() sampler with 1_000 samples, with 4 Markov chains using multiple threads MCMCThreads():

+
chain_negbin = sample(model_negbin, NUTS(), MCMCThreads(), 1_000, 4)
+summarystats(chain_negbin)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α    2.3935    0.3639    0.0077   2289.8720   1937.3708    1.0026       50.6474
+        β[1]    0.0127    0.0015    0.0000   4640.3636   2667.0438    1.0014      102.6357
+        β[2]   -0.7327    0.1499    0.0027   3027.2982   2637.8716    1.0028       66.9578
+        β[3]   -0.3199    0.1553    0.0027   3278.7372   2381.6903    1.0018       72.5192
+        β[4]    0.4266    0.3365    0.0073   2198.4749   1778.9698    1.0027       48.6259
+          ϕ⁻    1.4058    0.0800    0.0014   3056.0376   2581.1948    1.0003       67.5935
+
+

We had no problem with the Markov chains as all the rhat are well below 1.01 (or above 0.99). Note that the coefficients are in log scale. So we need to also apply the exponential function as we did before.

+
@chain quantile(chain_negbin) begin
+    DataFrame
+    select(_, :parameters, names(_, r"%") .=> ByRow(exp); renamecols=false)
+end
6×6 DataFrame
+ Row │ parameters  2.5%      25.0%     50.0%      75.0%      97.5%
+     │ Symbol      Float64   Float64   Float64    Float64    Float64
+─────┼─────────────────────────────────────────────────────────────────
+   1 │ α           5.28389   8.5925    11.0566    13.9834    21.9849
+   2 │ β[1]        1.00987   1.01173    1.01277    1.01381    1.016
+   3 │ β[2]        0.358343  0.433873   0.481535   0.53245    0.645215
+   4 │ β[3]        0.540157  0.650629   0.724575   0.806835   0.984437
+   5 │ β[4]        0.829193  1.21638    1.50724    1.92373    3.05504
+   6 │ ϕ⁻          3.52309   3.85985    4.07161    4.30516    4.80385
+

Our results show much more uncertainty in the coefficients than in the Poisson regression. So it might be best to use the Poisson regression in the roaches dataset.

+

References

+

Gelman, A., & Hill, J. (2007). Data analysis using regression and multilevel/hierarchical models. Cambridge university press.

+

Gelman, A., Carlin, J. B., Stern, H. S., Dunson, D. B., Vehtari, A., & Rubin, D. B. (2013). Bayesian Data Analysis. Chapman and Hall/CRC.

+

Gelman, A., Hill, J., & Vehtari, A. (2020). Regression and other stories. Cambridge University Press.

+

McElreath, R. (2020). Statistical rethinking: A Bayesian course with examples in R and Stan. CRC press.

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/10_robust_reg/index.html b/pages/10_robust_reg/index.html new file mode 100644 index 00000000..b656b12a --- /dev/null +++ b/pages/10_robust_reg/index.html @@ -0,0 +1,178 @@ + Robust Bayesian Regression
Bayesian Stats
Julia Logo
Code on GitHub
  1. Comparison Between Normal vs Student-tt
  2. Bayesian Robust Regression
  3. Example - Duncan's Prestige
  4. References

Robust Bayesian Regression

Leaving the universe of linear models, we start to venture into generalized linear models (GLM). The fourth of these is robust regression.

A regression with count data behaves exactly like a linear model: it makes a prediction simply by computing a weighted sum of the independent variables X\mathbf{X} by the estimated coefficients β\boldsymbol{\beta}, plus an intercept α\alpha. However, instead of using a Gaussian/normal likelihood function, it uses a Student-tt likelihood function.

We use robust regression in the same context as linear regression: our dependent variable is continuous. But robust regression allows us to better handle outliers in our data.

Before we dive in the nuts and bolts of robust regression let's remember the Gaussian/normal curve that has a bell shape (figure below). It does not have a "fat tail" (or sometimes known as "long tail"). In other words, the observations are not far from the mean. When we use this distribution as a likelihood function in the Bayesian models, we force that all estimates must be conditioned into a normal distribution of the dependent variable. If there are many outliers in the data (observations quite far from the mean), this causes the estimates of the independent variables' coefficients to be unstable. This is because the normal distribution cannot contemplate observations that are very spread away from the mean without having to change the mean's position (or location). In other words, the bell curve needs to "shift" to be able to contemplate outliers, thus making the inference unstable.

using CairoMakie
+using Distributions
+
+f, ax, l = lines(-4 .. 4, Normal(0, 1); linewidth=5, axis=(; xlabel=L"x", ylabel="Density"))

Normal with μ=0\mu=0 and σ=1\sigma = 1

So we need a more "malleable" distribution as a likelihood function. A distribution that is more robust to outliers. A distribution similar to Normal but that has "fatter" (or "longer") tails to precisely contemplate observations very far from the average without having to "shift" the mean's position (or location). For that we have the Student-tt distribution. See the figure below to remember its shape.

f, ax, l = lines(-4 .. 4, TDist(2); linewidth=5, axis=(xlabel=L"x", ylabel="Density"))
+

Student-tt with ν=2\nu = 2

+

Comparison Between Normal vs Student-tt

+

Take a look at the tails in the comparison below:

+
f, ax, l = lines(
+    -4 .. 4,
+    Normal(0, 1);
+    linewidth=5,
+    label="Normal",
+    axis=(; xlabel=L"x", ylabel="Density"),
+)
+lines!(ax, -4 .. 4, TDist(2); linewidth=5, label="Student")
+axislegend(ax)
+

Comparison between Normal and Student-tt Distributions

+

Bayesian Robust Regression

+

The standard approach for modeling a continuous dependent variable is with a Gaussian/normal likelihood function. This implies that the model error, σ\sigma of the Gaussian/normal likelihood function is distributed as a normal distribution:

+y∼Normal(α+X⋅β,σ)α∼Normal(μα,σα)β∼Normal(μβ,σβ)σ∼Exponential(λσ) +\begin{aligned} +\mathbf{y} &\sim \text{Normal}\left( \alpha + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} + +

From a Bayesian point of view, there is nothing special about Gaussian/normal likelihood function It is just a probabilistic distribution specified in a model. We can make the model more robust by using a Student-tt distribution as a likelihood function. This implies that the model error, σ\sigma does not follow a normal distribution, instead it follows a Student-tt distribution:

+y∼Student(ν,α+X⋅β,σ)α∼Normal(μα,σα)β∼Normal(μβ,σβ)ν∼Log-Normal(2,1)σ∼Exponential(λσ) +\begin{aligned} +\mathbf{y} &\sim \text{Student}\left( \nu, \alpha + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ +\alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ +\nu &\sim \text{Log-Normal}(2, 1) \\ +\sigma &\sim \text{Exponential}(\lambda_\sigma) +\end{aligned} + +

Here are some differences:

+
    +

  1. Student-tt likelihood function requires one additional parameter: ν\nu, degrees of freedom. These control how "fat" (or "long") the tails will be. Values of ν>20\nu> 20 forces the Student-tt distribution to practically become a normal distribution.

    + +

  2. There is nothing special about ν\nu. It is just another parameter for the model to estimate. So just specify a prior on it. In this case, I am using a Log-Normal distribution with mean 2 and standard deviation 1.

    + +
+

Note that there is also nothing special about the priors of the β\boldsymbol{\beta} coefficients or the intercept α\alpha. We could very well also specify other distributions as priors or even make the model even more robust to outliers by specifying priors as Student-tt distributions with degrees of freedom ν=3\nu = 3:

+α∼Student(να=3,μα,σα)β∼Student(νβ=3,μβ,σβ) +\begin{aligned} +\alpha &\sim \text{Student}(\nu_\alpha = 3, \mu_\alpha, \sigma_\alpha) \\ +\boldsymbol{\beta} &\sim \text{Student}(\nu_{\boldsymbol{\beta}} = 3, \mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) +\end{aligned} + +

Our goal is to instantiate a regression with count data using the observed data (y\mathbf{y} and X\mathbf{X}) and find the posterior distribution of our model's parameters of interest (α\alpha and β\boldsymbol{\beta}). This means to find the full posterior distribution of:

+P(θ∣y)=P(α,β,σ∣y) P(\boldsymbol{\theta} \mid \mathbf{y}) = P(\alpha, \boldsymbol{\beta}, \sigma \mid \mathbf{y}) +

This is easily accomplished with Turing:

+
using Turing
+using Statistics: mean, std
+using StatsBase: mad
+using Random: seed!
+seed!(123)
+
+@model function robustreg(X, y; predictors=size(X, 2))
+    #priors
+    α ~ LocationScale(median(y), 2.5 * mad(y), TDist(3))
+    β ~ filldist(TDist(3), predictors)
+    σ ~ Exponential(1)
+    ν ~ LogNormal(2, 1)
+
+    #likelihood
+    return y ~ arraydist(LocationScale.(α .+ X * β, σ, TDist.(ν)))
+end;
+

Here I am specifying very weakly informative priors:

+
    +

  • α∼Student-t(median⁡(y),2.5⋅MAD⁡(y),να=3)\alpha \sim \text{Student-}t(\operatorname{median}(\mathbf{y}), 2.5 \cdot \operatorname{MAD}(\mathbf{y}), \nu_{\alpha} = 3) – This means a Student-tt distribution with degrees of freedom ν = 3 centered on y's median with variance 2.5 times the mean absolute deviation (MAD) of y. That prior should with ease cover all possible values of α\alpha. Remember that the Student-tt distribution has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). The LocationScale() Turing's function adds location and scale parameters to distributions that doesn't have it. This is the case with the TDist() distribution which only takes the ν degrees of of freedom as parameter.

    + +

  • β∼Student-t(0,1,νβ)\boldsymbol{\beta} \sim \text{Student-}t(0,1,\nu_{\boldsymbol{\beta}}) – The predictors all have a prior distribution of a Student-tt distribution with degrees of freedom ν = 3 centered on 0 with variance 1 and degrees of freedom νβ\nu_{\boldsymbol{\beta}}. That wide-tailed tt distribution will cover all possible values for our coefficients. Remember the Student-tt also has support over all the real number line ∈(−∞,+∞)\in (-\infty, +\infty). Also the filldist() is a nice Turing's function which takes any univariate or multivariate distribution and returns another distribution that repeats the input distribution.

    + +

  • σ∼Exponential(1)\sigma \sim \text{Exponential}(1) – A wide-tailed-positive-only distribution perfectly suited for our model's error.

    + +
+

Turing's arraydist() function wraps an array of distributions returning a new distribution sampling from the individual distributions. It creates a broadcast and is a nice short hand for the familiar dot . broadcasting operator in Julia. By specifying that y vector is "broadcasted distributed" as a LocationScale broadcasted to mean (location parameter) α added to the product of the data matrix X and β coefficient vector along with a variance (scale parameter) σ. To conclude, we place inside the LocationScale a broadcasted TDist with ν degrees of freedom parameter.

+

Example - Duncan's Prestige

+

For our example, I will use a famous dataset called duncan (Duncan, 1961), which is data from occupation's prestige filled with outliers. It has 45 observations and the following variables:

+
    +

  • profession: name of the profession.

    + +

  • type: type of occupation. A qualitative variable:

    +
      +

    • prof - professional or management.

      + +

    • wc - white-collar.

      + +

    • bc - blue-collar.

      + +
    + +

  • income: percentage of people in the occupation earning over U$ 3,500 per year in 1950 (more or less U$ 36,000 in 2017).

    + +

  • education: percentage of people in the occupation who had a high school diploma in 1949 (which, being cynical, we can say is somewhat equivalent to a PhD degree in 2017).

    + +

  • prestige: percentage of respondents in the survey who classified their occupation as at least "good" with respect to prestige.

    + +
+

Ok let's read our data with CSV.jl and output into a DataFrame from DataFrames.jl:

+
using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/duncan.csv"
+duncan = CSV.read(HTTP.get(url).body, DataFrame)
+describe(duncan)
5×7 DataFrame
+ Row │ variable    mean     min          median  max             nmissing  eltype
+     │ Symbol      Union…   Any          Union…  Any             Int64     DataType
+─────┼──────────────────────────────────────────────────────────────────────────────
+   1 │ profession           RR.engineer          welfare.worker         0  String31
+   2 │ type                 bc                   wc                     0  String7
+   3 │ income      41.8667  7            42.0    81                     0  Int64
+   4 │ education   52.5556  7            45.0    100                    0  Int64
+   5 │ prestige    47.6889  3            41.0    97                     0  Int64
+

As you can see from the describe() output the average occupation's percentage of respondents who classified their occupation as at least "good" with respect to prestige is around 41%. But prestige variable is very dispersed and actually has a bimodal distribution:

+
f = Figure()
+plt = data(duncan) * mapping(:prestige) * AlgebraOfGraphics.density()
+draw!(f[1, 1], plt)
UndefVarError: `data` not defined
+
+

// Image matching '/assets/pages/10_robust_reg/code/prestige_density' not found. //

Density Plot of prestige

+

Besides that, the mean prestige per type shows us where the source of variation might come from:

+
gdf = groupby(duncan, :type)
+f = Figure()
+plt =
+    data(combine(gdf, :prestige => mean)) * mapping(:type, :prestige_mean) * visual(BarPlot)
+draw!(f[1, 1], plt)
UndefVarError: `data` not defined
+
+

// Image matching '/assets/pages/10_robust_reg/code/prestige_per_type' not found. //

Mean prestige per type

+

Now let's us instantiate our model with the data:

+
X = Matrix(select(duncan, [:income, :education]))
+y = duncan[:, :prestige]
+model = robustreg(X, y);
+

And, finally, we will sample from the Turing model. We will be using the default NUTS() sampler with 1_000 samples, with 4 Markov chains using multiple threads MCMCThreads():

+
chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4)
+summarystats(chain)
Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α   -7.5790    3.0116    0.0628   2309.7654   1978.4710    1.0042       70.5574
+        β[1]    0.7719    0.1024    0.0025   1748.2750   1843.0804    1.0028       53.4053
+        β[2]    0.4464    0.0836    0.0020   1702.5880   1820.3078    1.0052       52.0097
+           σ    7.1578    1.7728    0.0477   1225.3095    826.2654    1.0027       37.4300
+           ν    2.8148    2.4661    0.0635   1452.4245   1333.5578    1.0009       44.3678
+
+

We had no problem with the Markov chains as all the rhat are well below 1.01 (or above 0.99). Also note that all degrees of freedom parameters, the ν stuff, have been estimated with mean around 3 to 5, which indeed signals that our model needed fat tails to make a robust inference. Our model has an error σ of around 7. So it estimates occupation's prestige ±7. The intercept α is the basal occupation's prestige value. So each occupation has -7±7 prestige before we add the coefficients multiplied by the occupations' independent variables. And from our coefficients β\boldsymbol{\beta}, we can see that the quantile() tells us the uncertainty around their estimates:

+
quantile(chain)
Quantiles
+  parameters       2.5%     25.0%     50.0%     75.0%     97.5%
+      Symbol    Float64   Float64   Float64   Float64   Float64
+
+           α   -13.3670   -9.6056   -7.5776   -5.5600   -1.7272
+        β[1]     0.5567    0.7114    0.7743    0.8380    0.9659
+        β[2]     0.2796    0.3936    0.4460    0.4993    0.6170
+           σ     4.1120    5.8269    7.0177    8.3067   10.9849
+           ν     1.0269    1.6074    2.1733    3.1841    8.1579
+
+
    +

  • β[1] – first column of X, income, has 95% credible interval from 0.55 to 0.96. This means that an increase of U$ 1,000 in occupations' annual income is associated with an increase in roughly 0.5 to 1.0 in occupation's prestige.

    + +

  • β[2] – second column of X, education, has a 95% credible interval from 0.29 to 0.61. So we expect that an increase of 1% in occupations' percentage of respondents who had a high school diploma increases occupations' prestige roughly 0.3 to 0.6.

    + +
+

That's how you interpret 95% credible intervals from a quantile() output of a robust regression Chains object.

+

References

+

Duncan, O. D. (1961). A socioeconomic index for all occupations. Class: Critical Concepts, 1, 388–426.

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/11_multilevel_models/index.html b/pages/11_multilevel_models/index.html new file mode 100644 index 00000000..f232dc1d --- /dev/null +++ b/pages/11_multilevel_models/index.html @@ -0,0 +1,201 @@ + Multilevel Models (a.k.a. Hierarchical Models)
Bayesian Stats
Julia Logo
Code on GitHub
  1. When to use Multilevel Models?
  2. Hyperpriors
  3. Three Approaches to Multilevel Models
    1. Random-Intercept Model
    2. Random-Slope Model
    3. Random-Intercept-Slope Model
    4. Correlated-Random-Intercept-Slope Model
  4. Example - Cheese Ratings
  5. References

Multilevel Models (a.k.a. Hierarchical Models)

Bayesian hierarchical models (also called multilevel models) are a statistical model written at multiple levels (hierarchical form) that estimates the parameters of the posterior distribution using the Bayesian approach. The sub-models combine to form the hierarchical model, and Bayes' theorem is used to integrate them with the observed data and to account for all the uncertainty that is present. The result of this integration is the posterior distribution, also known as an updated probability estimate, as additional evidence of the likelihood function is integrated together with the prior distribution of the parameters.

Hierarchical modeling is used when information is available at several different levels of observation units. The hierarchical form of analysis and organization helps to understand multiparameter problems and also plays an important role in the development of computational strategies.

Hierarchical models are mathematical statements that involve several parameters, so that the estimates of some parameters depend significantly on the values of other parameters. The figure below shows a hierarchical model in which there is a ϕ\phi hyperparameter that parameterizes the parameters θ1,θ2,…,θN\theta_1, \theta_2, \dots, \theta_N that are finally used to infer the posterior density of some variable of interest y=y1,y2,…,yN\mathbf{y} = y_1, y_2, \dots, y_N.

Bayesian Workflow

Hierarchical Model

When to use Multilevel Models?

Multilevel models are particularly suitable for research projects where participant data is organized at more than one level, i.e. nested data. Units of analysis are usually individuals (at a lower level) that are nested in contextual/aggregate units (at a higher level). An example is when we are measuring the performance of individuals and we have additional information about belonging to different groups such as sex, age group, hierarchical level, educational level or housing status.

There is a main assumption that cannot be violated in multilevel models which is exchangeability (de Finetti, 1974; Nau, 2001). Yes, this is the same assumption that we discussed in 2. What is Bayesian Statistics?. This assumption assumes that groups are exchangeable. The figure below shows a graphical representation of the exchangeability. The groups shown as "cups" that contain observations shown as "balls". If in the model's inferences, this assumption is violated, then multilevel models are not appropriate. This means that, since there is no theoretical justification to support exchangeability, the inferences of the multilevel model are not robust and the model can suffer from several pathologies and should not be used for any scientific or applied analysis.

Bayesian Workflow Bayesian Workflow

Exchangeability – Images from Michael Betancourt

Hyperpriors

As the priors of the parameters are sampled from another prior of the hyperparameter (upper-level's parameter), which are called hyperpriors. This makes one group's estimates help the model to better estimate the other groups by providing more robust and stable estimates.

We call the global parameters as population effects (or population-level effects, also sometimes called fixed effects) and the parameters of each group as group effects (or group-level effects, also sometimes called random effects). That is why multilevel models are also known as mixed models in which we have both fixed effects and random effects.

Three Approaches to Multilevel Models

Multilevel models generally fall into three approaches:

  1. Random-intercept model: each group receives a different intercept in addition to the global intercept.

  2. Random-slope model: each group receives different coefficients for each (or a subset of) independent variable(s) in addition to a global intercept.

  3. Random-intercept-slope model: each group receives both a different intercept and different coefficients for each independent variable in addition to a global intercept.

  4. Correlated-Random-intercept-slope model: each group receives both a different intercept and different coefficients for each independent variable in addition to a global intercept; while also taking into account the correlation/covariance amongst intercept/coefficients.

Random-Intercept Model

The first approach is the random-intercept model in which we specify a different intercept for each group, in addition to the global intercept. These group-level intercepts are sampled from a hyperprior.

To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. Mathematically a Bayesian multilevel random-slope linear regression model is:

y∼Normal(α+αj+X⋅β,σ)α∼Normal(μα,σα)αj∼Normal(0,τ)β∼Normal(μβ,σβ)τ∼Cauchy+(0,ψα)σ∼Exponential(λσ) \begin{aligned} \mathbf{y} &\sim \text{Normal}\left( \alpha + \alpha_j + \mathbf{X} \cdot \boldsymbol{\beta}, \sigma \right) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \alpha_j &\sim \text{Normal}(0, \tau) \\ \boldsymbol{\beta} &\sim \text{Normal}(\mu_{\boldsymbol{\beta}}, \sigma_{\boldsymbol{\beta}}) \\ \tau &\sim \text{Cauchy}^+(0, \psi_{\alpha})\\ \sigma &\sim \text{Exponential}(\lambda_\sigma) \end{aligned}

The priors on the global intercept α\alpha, global coefficients β\boldsymbol{\beta} and error σ\sigma, along with the Gaussian/normal likelihood on y\mathbf{y} are the same as in the linear regression model. But now we have new parameters. The first are the group intercepts prior αj\alpha_j that denotes that every group 1,2,…,J1, 2, \dots, J has its own intercept sampled from a normal distribution centered on 0 with a standard deviation ψα\psi_\alpha. This group intercept is added to the linear predictor inside the Gaussian/normal likelihood function. The group intercepts' standard deviation τ\tau have a hyperprior (being a prior of a prior) which is sampled from a positive-constrained Cauchy distribution (a special case of the Student-tt distribution with degrees of freedom ν=1\nu = 1) with mean 0 and standard deviation σα\sigma_\alpha. This makes the group-level intercept's dispersions being sampled from the same parameter τ\tau which allows the model to use information from one group intercept to infer robust information regarding another group's intercept dispersion and so on.

This is easily accomplished with Turing:

using Turing
+using LinearAlgebra
+using Statistics: mean, std
+using Random: seed!
+seed!(123)
+
+@model function varying_intercept(
+    X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)
+)
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))       # population-level intercept
+    β ~ filldist(Normal(0, 2), predictors)  # population-level coefficients
+    σ ~ Exponential(std(y))                 # residual SD
+    #prior for variance of random intercepts
+    #usually requires thoughtful specification
+    τ ~ truncated(Cauchy(0, 2); lower=0)    # group-level SDs intercepts
+    αⱼ ~ filldist(Normal(0, τ), n_gr)       # group-level intercepts
+
+    #likelihood
+    ŷ = α .+ X * β .+ αⱼ[idx]
+    return y ~ MvNormal(ŷ, σ^2 * I)
+end;

Random-Slope Model

The second approach is the random-slope model in which we specify a different slope for each group, in addition to the global intercept. These group-level slopes are sampled from a hyperprior.

To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. Mathematically a Bayesian multilevel random-slope linear regression model is:

y∼Normal(α+X⋅βj⋅τ,σ)α∼Normal(μα,σα)βj∼Normal(0,1)τ∼Cauchy+(0,ψβ)σ∼Exponential(λσ) \begin{aligned} \mathbf{y} &\sim \text{Normal}\left( \alpha + \mathbf{X} \cdot \boldsymbol{\beta}_j \cdot \boldsymbol{\tau}, \sigma \right) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \boldsymbol{\beta}_j &\sim \text{Normal}(0, 1) \\ \boldsymbol{\tau} &\sim \text{Cauchy}^+(0, \psi_{\boldsymbol{\beta}})\\ \sigma &\sim \text{Exponential}(\lambda_\sigma) \end{aligned}

Here we have a similar situation from before with the same hyperprior, but now it is a hyperprior for the the group coefficients' standard deviation prior: βj\boldsymbol{\beta}_j. This makes the group-level coefficients's dispersions being sampled from the same parameter τ\tau which allows the model to use information from one group coefficients to infer robust information regarding another group's coefficients dispersion and so on.

In Turing we can accomplish this as:

@model function varying_slope(X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2))
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))                    # population-level intercept
+    σ ~ Exponential(std(y))                              # residual SD
+    #prior for variance of random slopes
+    #usually requires thoughtful specification
+    τ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs
+    βⱼ ~ filldist(Normal(0, 1), predictors, n_gr)        # group-level standard normal slopes
+
+    #likelihood
+    ŷ = α .+ X * βⱼ * τ
+    return y ~ MvNormal(ŷ, σ^2 * I)
+end;

Random-Intercept-Slope Model

The third approach is the random-intercept-slope model in which we specify a different intercept and slope for each group, in addition to the global intercept. These group-level intercepts and slopes are sampled from hyperpriors.

To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. Mathematically a Bayesian multilevel random-intercept-slope linear regression model is:

y∼Normal(α+αj+X⋅βj⋅τβ,σ)α∼Normal(μα,σα)αj∼Normal(0,τα)βj∼Normal(0,1)τα∼Cauchy+(0,ψα)τβ∼Cauchy+(0,ψβ)σ∼Exponential(λσ) \begin{aligned} \mathbf{y} &\sim \text{Normal}\left( \alpha + \alpha_j + \mathbf{X} \cdot \boldsymbol{\beta}_j \cdot \boldsymbol{\tau}_{\boldsymbol{\beta}}, \sigma \right) \\ \alpha &\sim \text{Normal}(\mu_\alpha, \sigma_\alpha) \\ \alpha_j &\sim \text{Normal}(0, \tau_{\alpha}) \\ \boldsymbol{\beta}_j &\sim \text{Normal}(0, 1) \\ \tau_{\alpha} &\sim \text{Cauchy}^+(0, \psi_{\alpha})\\ \boldsymbol{\tau}_{\boldsymbol{\beta}} &\sim \text{Cauchy}^+(0, \psi_{\boldsymbol{\beta}})\\ \sigma &\sim \text{Exponential}(\lambda_\sigma) \end{aligned}

Here we have a similar situation from before with the same hyperpriors, but now we fused both random-intercept and random-slope together.

In Turing we can accomplish this as:

@model function varying_intercept_slope(
+    X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)
+)
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))                     # population-level intercept
+    σ ~ Exponential(std(y))                               # residual SD
+    #prior for variance of random intercepts and slopes
+    #usually requires thoughtful specification
+    τₐ ~ truncated(Cauchy(0, 2); lower=0)                 # group-level SDs intercepts
+    τᵦ ~ filldist(truncated(Cauchy(0, 2); lower=0), n_gr) # group-level slopes SDs
+    αⱼ ~ filldist(Normal(0, τₐ), n_gr)                    # group-level intercepts
+    βⱼ ~ filldist(Normal(0, 1), predictors, n_gr)         # group-level standard normal slopes
+
+    #likelihood
+    ŷ = α .+ αⱼ[idx] .+ X * βⱼ * τᵦ
+    return y ~ MvNormal(ŷ, σ^2 * I)
+end;

In all of the models so far, we are using the MvNormal construction where we specify both a vector of means (first positional argument) and a covariance matrix (second positional argument). Regarding the covariance matrix σ^2 * I, it uses the model's errors σ, here parameterized as a standard deviation, squares it to produce a variance parameterization, and multiplies by I, which is Julia's LinearAlgebra standard module implementation to represent an identity matrix of any size.

Correlated-Random-Intercept-Slope Model

The third approach is the correlated-random-intercept-slope model in which we specify a different intercept and slope for each group, in addition to the global intercept. These group-level intercepts and slopes are sampled from hyperpriors. Finally, we also model the correlation/covariance amongst the intercepts and slopes. We assume that they are not independent anymore, rather they are correlated.

In order to model the correlation between parameters, we need a prior on correlation/covariance matrices. There are several ways to define priors on covariance matrices. In some ancient time, "Bayesian elders" used distributions such as the Wishart distribution and the Inverse Wishart distribution. However, priors on covariance matrices are not that intuitive, and can be hard to translate into real-world scenarios. That's why I much prefer to construct my covariances matrices from a correlation matrix and a vector of standard deviations. Remember that every covariance matrix Σ\boldsymbol{\Sigma} can be decomposed into:

Σ=diagmatrix(τ)⋅Ω⋅diagmatrix(τ)\boldsymbol{\Sigma}=\text{diag}_\text{matrix}(\boldsymbol{\tau}) \cdot \boldsymbol{\Omega} \cdot \text{diag}_\text{matrix}(\boldsymbol{\tau})

where Ω\boldsymbol{\Omega} is a correlation matrix with 11s in the diagonal and the off-diagonal elements between -1 e 1 ρ∈(−1,1)\rho \in (-1, 1). τ\boldsymbol{\tau} is a vector composed of the variables' standard deviation from Σ\boldsymbol{\Sigma} (is is the Σ\boldsymbol{\Sigma}'s diagonal).

These are much more intuitive and easy to work with, specially if you are not working together with some "Bayesian elders". This approach leaves us with the need to specify a prior on correlation matrices. Luckily, we have the LKJ distribution: distribution for positive definite matrices with unit diagonals (exactly what a correlation matrix is).

To illustrate a multilevel model, I will use the linear regression example with a Gaussian/normal likelihood function. Mathematically a Bayesian multilevel correlated-random-intercept-slope linear regression model is:

y∼Normal(Xβj,σ)βj∼Multivariate Normal(μj,Σ)forj∈{1,…,J}Σ∼LKJ(η)σ∼Exponential(λσ) \begin{aligned} \mathbf{y} & \sim \text{Normal}(\mathbf{X} \boldsymbol{\beta}_{j}, \sigma) \\ \boldsymbol{\beta}_j & \sim \text{Multivariate Normal}(\boldsymbol{\mu}_j, \boldsymbol{\Sigma}) \quad \text{for}\quad j \in \{ 1, \dots, J \} \\ \boldsymbol{\Sigma} & \sim \text{LKJ}(\eta) \\ \sigma & \sim \text{Exponential}(\lambda_\sigma) \end{aligned}

We don't have any α\alphas. If we want a varying intercept, we just insert a column filled with 11s in the data matrix X\mathbf{X}. Mathematically, this makes the column behave like an "identity" variable (because the number 11 in the multiplication operation 1⋅β1 \cdot \beta is the identity element. It maps x→xx \to x keeping the value of xx intact) and, consequently, we can interpret the column's coefficient as the model's intercept.

In Turing we can accomplish this as:

using PDMats
+
+@model function correlated_varying_intercept_slope(
+    X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)
+)
+    #priors
+    Ω ~ LKJCholesky(predictors, 2.0) # Cholesky decomposition correlation matrix
+    σ ~ Exponential(std(y))
+
+    #prior for variance of random correlated intercepts and slopes
+    #usually requires thoughtful specification
+    τ ~ filldist(truncated(Cauchy(0, 2); lower=0), predictors) # group-level SDs
+    γ ~ filldist(Normal(0, 5), predictors, n_gr)               # matrix of group coefficients
+
+    #reconstruct Σ from Ω and τ
+    Σ_L = Diagonal(τ) * Ω.L
+    Σ = PDMat(Cholesky(Σ_L + 1e-6 * I))                        # numerical instability
+    #reconstruct β from Σ and γ
+    β = Σ * γ
+
+    #likelihood
+    return y ~ arraydist([Normal(X[i, :] ⋅ β[:, idx[i]], σ) for i in 1:length(y)])
+end;

In the correlated_varying_intercept_slope model, we are using the efficient Cholesky decomposition version of the LKJ prior with LKJCholesky. We put priors on all group coefficients γ and reconstruct both the covariance matrix Σ and the coefficient matrix β.

Example - Cheese Ratings

For our example, I will use a famous dataset called cheese (Boatwright, McCulloch & Rossi, 1999), which is data from cheese ratings. A group of 10 rural and 10 urban raters rated 4 types of different cheeses (A, B, C and D) in two samples. So we have 4⋅20⋅2=1604 \cdot 20 \cdot 2 = 160 observations and 4 variables:

  • cheese: type of cheese from A to D

  • rater: id of the rater from 1 to 10

  • background: type of rater, either rural or urban

  • y: rating of the cheese

OK, let's read our data with CSV.jl and output into a DataFrame from DataFrames.jl:

using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv"
+cheese = CSV.read(HTTP.get(url).body, DataFrame)
+describe(cheese)
4×7 DataFrame
+ Row │ variable    mean     min    median  max    nmissing  eltype
+     │ Symbol      Union…   Any    Union…  Any    Int64     DataType
+─────┼───────────────────────────────────────────────────────────────
+   1 │ cheese               A              D             0  String1
+   2 │ rater       5.5      1      5.5     10            0  Int64
+   3 │ background           rural          urban         0  String7
+   4 │ y           70.8438  33     71.5    91            0  Int64

As you can see from the describe() output, the mean cheese ratings is around 70 ranging from 33 to 91.

In order to prepare the data for Turing, I will convert the Strings in variables cheese and background to Ints. Regarding cheese, I will create 4 dummy variables one for each cheese type; and background will be converted to integer data taking two values: one for each background type. My intent is to model background as a group both for intercept and coefficients. Take a look at how the data will look like for the first 5 observations:

for c in unique(cheese[:, :cheese])
+    cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0)
+end
+
+cheese[:, :background_int] = map(cheese[:, :background]) do b
+    if b == "rural"
+        1
+    elseif b == "urban"
+        2
+    else
+        missing
+    end
+end
+
+first(cheese, 5)
5×9 DataFrame
+ Row │ cheese   rater  background  y      cheese_A  cheese_B  cheese_C  cheese_D  background_int
+     │ String1  Int64  String7     Int64  Int64     Int64     Int64     Int64     Int64
+─────┼───────────────────────────────────────────────────────────────────────────────────────────
+   1 │ A            1  rural          67         1         0         0         0               1
+   2 │ A            1  rural          66         1         0         0         0               1
+   3 │ B            1  rural          51         0         1         0         0               1
+   4 │ B            1  rural          53         0         1         0         0               1
+   5 │ C            1  rural          75         0         0         1         0               1

Now let's us instantiate our model with the data. Here, I will specify a vector of Ints named idx to represent the different observations' group memberships. This will be used by Turing when we index a parameter with the idx, e.g. αⱼ[idx].

X = Matrix(select(cheese, Between(:cheese_A, :cheese_D)));
+y = cheese[:, :y];
+idx = cheese[:, :background_int];

The first model is the varying_intercept:

model_intercept = varying_intercept(X, idx, y)
+chain_intercept = sample(model_intercept, NUTS(), MCMCThreads(), 1_000, 4)
+println(DataFrame(summarystats(chain_intercept)))
9×8 DataFrame
+ Row │ parameters  mean       std       mcse        ess_bulk  ess_tail  rhat     ess_per_sec
+     │ Symbol      Float64    Float64   Float64     Float64   Float64   Float64  Float64
+─────┼───────────────────────────────────────────────────────────────────────────────────────
+   1 │ α            70.7489   5.02573   0.176587     1049.21   944.855  1.00255      19.8948
+   2 │ β[1]          2.91501  1.30963   0.0257346    2596.92  2763.9    1.00097      49.2419
+   3 │ β[2]        -10.632    1.34056   0.028425     2224.99  2724.4    1.00189      42.1895
+   4 │ β[3]          6.52144  1.3159    0.0265587    2450.36  2667.92   1.00052      46.463
+   5 │ β[4]          1.09178  1.31719   0.0264376    2475.97  2627.54   1.00117      46.9485
+   6 │ σ             7.3913   0.449209  0.00807096   3127.77  2530.83   1.00102      59.3078
+   7 │ τ             6.11784  5.54993   0.152948     1863.81  1849.7    1.00172      35.341
+   8 │ αⱼ[1]        -3.39662  4.92084   0.174051     1057.29   929.725  1.00312      20.0479
+   9 │ αⱼ[2]         3.63474  4.95267   0.175295     1054.17   920.198  1.00331      19.9889
+

Here we can see that the model has a population-level intercept α along with population-level coefficients βs for each cheese dummy variable. But notice that we have also group-level intercepts for each of the groups αⱼs. Specifically, αⱼ[1] are the rural raters and αⱼ[2] are the urban raters.

Now let's go to the second model, varying_slope:

model_slope = varying_slope(X, idx, y)
+chain_slope = sample(model_slope, NUTS(), MCMCThreads(), 1_000, 4)
+println(DataFrame(summarystats(chain_slope)))
12×8 DataFrame
+ Row │ parameters  mean        std       mcse       ess_bulk  ess_tail   rhat     ess_per_sec
+     │ Symbol      Float64     Float64   Float64    Float64   Float64    Float64  Float64
+─────┼────────────────────────────────────────────────────────────────────────────────────────
+   1 │ α           71.6225     6.12929   0.80038     70.2758    24.8834  1.0542       1.3079
+   2 │ σ            7.98871    0.435528  0.0142789  879.095   1907.61    1.00481     16.3607
+   3 │ τ[1]         5.95148    5.05219   0.283585   297.117   1630.98    1.01456      5.52961
+   4 │ τ[2]         7.13182    7.54656   1.22127    120.798     26.97    1.03962      2.24817
+   5 │ βⱼ[1,1]      0.199936   0.839602  0.0512732  282.448    223.989   1.01652      5.25661
+   6 │ βⱼ[2,1]     -0.933251   1.06721   0.0676919  231.737    252.835   1.01661      4.31283
+   7 │ βⱼ[3,1]      0.522532   0.883634  0.035598   636.854   1246.34    1.02562     11.8524
+   8 │ βⱼ[4,1]      0.0664401  0.797414  0.0514516  278.881    154.864   1.02163      5.19022
+   9 │ βⱼ[1,2]      0.290895   0.843615  0.0732187  135.896     84.0176  1.03096      2.52915
+  10 │ βⱼ[2,2]     -0.853134   1.05917   0.0568238  415.139    467.237   1.01995      7.7261
+  11 │ βⱼ[3,2]      0.563184   0.876935  0.0375404  474.159    818.21    1.01366      8.82451
+  12 │ βⱼ[4,2]      0.124051   0.787212  0.0525178  224.833    143.946   1.018        4.18434
+

Here we can see that the model has still a population-level intercept α. But now our population-level coefficients βs are replaced by group-level coefficients βⱼs along with their standard deviation τᵦs. Specifically βⱼ's first index denotes the 4 dummy cheese variables' and the second index are the group membership. So, for example βⱼ[1,1] is the coefficient for cheese_A and rural raters (group 1).

Now let's go to the third model, varying_intercept_slope:

model_intercept_slope = varying_intercept_slope(X, idx, y)
+chain_intercept_slope = sample(model_intercept_slope, NUTS(), MCMCThreads(), 1_000, 4)
+println(DataFrame(summarystats(chain_intercept_slope)))
15×8 DataFrame
+ Row │ parameters  mean       std       mcse        ess_bulk  ess_tail  rhat     ess_per_sec
+     │ Symbol      Float64    Float64   Float64     Float64   Float64   Float64  Float64
+─────┼───────────────────────────────────────────────────────────────────────────────────────
+   1 │ α           70.8973    7.06973   0.226273    1080.56    988.602  1.00354     13.1307
+   2 │ σ            7.07716   0.403315  0.00763609  2791.06   2389.42   1.00236     33.9161
+   3 │ τₐ           6.19068   5.86481   0.187349    1286.35   1007.7    1.00342     15.6313
+   4 │ τᵦ[1]        6.19739   5.39243   0.202217     646.032  1159.5    1.00601      7.85039
+   5 │ τᵦ[2]        6.24244   5.30258   0.215136     598.407  1527.72   1.00463      7.27167
+   6 │ αⱼ[1]       -3.62952   5.05254   0.176822    1253.95    761.597  1.00336     15.2377
+   7 │ αⱼ[2]        3.42934   5.05582   0.176181    1271.82    839.894  1.00348     15.4548
+   8 │ βⱼ[1,1]      0.273658  0.779529  0.0188846   1727.94   1703.37   1.00193     20.9975
+   9 │ βⱼ[2,1]     -0.88519   1.07495   0.0402493    794.926   375.533  1.00615      9.6597
+  10 │ βⱼ[3,1]      0.540418  0.897065  0.0325307    849.719   418.537  1.00343     10.3255
+  11 │ βⱼ[4,1]      0.119311  0.761924  0.0183527   1730.92   1000.11   1.00092     21.0336
+  12 │ βⱼ[1,2]      0.230796  0.819673  0.0210441   1571.83   1681.27   1.00307     19.1004
+  13 │ βⱼ[2,2]     -0.916136  1.02804   0.0319528   1027.79   2087.51   1.00208     12.4894
+  14 │ βⱼ[3,2]      0.577911  0.873303  0.0269948   1094.9    1738.12   1.00304     13.3049
+  15 │ βⱼ[4,2]      0.115502  0.784112  0.0186258   1758.58   1981.55   1.0031      21.3698
+

Now we have fused the previous model in one. We still have a population-level intercept α. But now we have in the same model group-level intercepts for each of the groups αⱼs and group-level along with their standard deviation τₐ. We also have the coefficients βⱼs with their standard deviation τᵦs. The parameters are interpreted exactly as the previous cases.

Now let's go to the fourth model, varying_intercept_slope. Here we are going to add a columns of into the data matrix X:

X_correlated = hcat(fill(1, size(X, 1)), X)
+model_correlated = correlated_varying_intercept_slope(X_correlated, idx, y)
+chain_correlated = sample(model_correlated, NUTS(), MCMCThreads(), 1_000, 4)
+println(DataFrame(summarystats(chain_correlated)))
31×8 DataFrame
+ Row │ parameters  mean         std        mcse          ess_bulk  ess_tail  rhat        ess_per_sec
+     │ Symbol      Float64      Float64    Float64       Float64   Float64   Float64     Float64
+─────┼───────────────────────────────────────────────────────────────────────────────────────────────
+   1 │ Ω.L[1,1]     1.0         0.0        NaN             NaN      NaN      NaN          NaN
+   2 │ Ω.L[2,1]     0.0499316   0.31051      0.00698543   1975.54  2314.64     1.00168      1.52283
+   3 │ Ω.L[3,1]    -0.454537    0.249772     0.00571052   1977.49  2262.51     1.00096      1.52433
+   4 │ Ω.L[4,1]     0.308835    0.25287      0.00599969   1843.28  2199.73     1.00014      1.42088
+   5 │ Ω.L[5,1]    -0.125632    0.307712     0.00654629   2239.37  2376.57     1.00314      1.72619
+   6 │ Ω.L[2,2]     0.946451    0.0731193    0.00168809   2176.21  2182.87     1.00018      1.67751
+   7 │ Ω.L[3,2]     0.00316296  0.326756     0.00512707   4010.69  2567.33     1.0003       3.0916
+   8 │ Ω.L[4,2]     0.0351896   0.354115     0.00527131   4546.36  3111.1      1.00178      3.50452
+   9 │ Ω.L[5,2]     0.0261588   0.366052     0.00538327   4345.52  2064.09     1.00291      3.3497
+  10 │ Ω.L[3,3]     0.775999    0.148672     0.00348151   1845.72  2482.95     1.00082      1.42276
+  11 │ Ω.L[4,3]    -0.0330662   0.349329     0.00604209   3242.42  2090.97     1.00099      2.49939
+  12 │ Ω.L[5,3]     0.0374594   0.360744     0.00516219   4901.98  2431.25     1.00176      3.77865
+  13 │ Ω.L[4,4]     0.753896    0.150472     0.00357772   1850.93  2236.89     1.00165      1.42677
+  14 │ Ω.L[5,4]     0.0116934   0.349883     0.00506856   4664.43  2772.27     1.00128      3.59553
+  15 │ Ω.L[5,5]     0.684545    0.179677     0.00469272   1541.5   1667.6      1.00377      1.18825
+  16 │ σ            7.08991     0.392342     0.00609954   4281.99  2994.66     1.00037      3.30073
+  17 │ τ[1]         3.82687     0.792461     0.0243249    1071.64  1274.54     1.00157      0.82606
+  18 │ τ[2]         0.515217    0.457244     0.0118942    1162.98  1701.28     1.00367      0.896471
+  19 │ τ[3]         1.57042     0.675595     0.0194601    1075.05  1066.81     1.00327      0.82869
+  20 │ τ[4]         0.79098     0.502001     0.0137349    1042.75   683.387    1.0037       0.803794
+  21 │ τ[5]         0.536124    0.435909     0.0118638    1106.15  1058.94     1.00122      0.852668
+  22 │ γ[1,1]       5.1869      2.19555      0.0618344    1249.75  1539.01     1.00063      0.963357
+  23 │ γ[2,1]      -0.104914    4.6473       0.0843589    3038.11  2765.66     0.999955     2.3419
+  24 │ γ[3,1]      -2.26726     3.24333      0.0677281    2307.05  2102.85     1.00399      1.77836
+  25 │ γ[4,1]       0.786036    4.18682      0.0682086    3757.86  2648.24     1.00005      2.89671
+  26 │ γ[5,1]       0.317867    4.53047      0.067744     4456.83  2826.46     1.0041       3.43551
+  27 │ γ[1,2]       5.68517     2.38212      0.0679177    1209.5   1530.76     1.00015      0.932333
+  28 │ γ[2,2]       0.406946    4.69823      0.0761723    3785.93  2816.48     1.00131      2.91835
+  29 │ γ[3,2]      -2.43749     3.41662      0.0695933    2451.7   2481.02     1.00083      1.88987
+  30 │ γ[4,2]       1.40997     4.31934      0.0742324    3389.91  2550.09     1.00076      2.61308
+  31 │ γ[5,2]      -0.565495    4.69425      0.0822518    3281.19  2340.3      1.00369      2.52928
+

We can see that we also get all of the Cholesky factors of the correlation matrix Ω which we can transform back into a correlation matrix by doing Ω = Ω.L * Ω.L'.

References

Boatwright, P., McCulloch, R., & Rossi, P. (1999). Account-level modeling for trade promotion: An application of a constrained parameter hierarchical model. Journal of the American Statistical Association, 94(448), 1063–1073.

de Finetti, B. (1974). Theory of Probability (Volume 1). New York: John Wiley & Sons.

Nau, R. F. (2001). De Finetti was Right: Probability Does Not Exist. Theory and Decision, 51(2), 89–124. https://doi.org/10.1023/A:1015525808214

Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language.
\ No newline at end of file diff --git a/pages/12_Turing_tricks/index.html b/pages/12_Turing_tricks/index.html new file mode 100644 index 00000000..508506d3 --- /dev/null +++ b/pages/12_Turing_tricks/index.html @@ -0,0 +1,494 @@ + Computational Tricks with Turing <br/> (Non-Centered Parametrization <br/> and QR Decomposition)
Bayesian Stats
Julia Logo
Code on GitHub
  1. QR Decomposition
  2. Non-Centered Parametrization
  3. References

Computational Tricks with Turing
(Non-Centered Parametrization
and QR Decomposition)

There are some computational tricks that we can employ with Turing. I will cover here two computational tricks:

  1. QR Decomposition

  2. Non-Centered Parametrization

QR Decomposition

Back in "Linear Algebra 101" we've learned that any matrix (even rectangular ones) can be factored into the product of two matrices:

  • Q\mathbf{Q}: an orthogonal matrix (its columns are orthogonal unit vectors meaning QT=Q−1)\mathbf{Q}^T = \mathbf{Q}^{-1}).

  • R\mathbf{R}: an upper triangular matrix.

This is commonly known as the QR Decomposition:

A=Q⋅R \mathbf{A} = \mathbf{Q} \cdot \mathbf{R}

Let me show you an example with a random matrix A∈R3×2\mathbf{A} \in \mathbb{R}^{3 \times 2}:

A = rand(3, 2)
3×2 Matrix{Float64}:
+ 0.720103  0.295367
+ 0.573619  0.276597
+ 0.664468  0.983436

Now let's factor A using LinearAlgebra's qr() function:

using LinearAlgebra: qr, I
+Q, R = qr(A)
LinearAlgebra.QRCompactWY{Float64, Matrix{Float64}, Matrix{Float64}}
+Q factor: 3×3 LinearAlgebra.QRCompactWYQ{Float64, Matrix{Float64}, Matrix{Float64}}
+R factor:
+2×2 Matrix{Float64}:
+ -1.13539  -0.902615
+  0.0       0.562299

Notice that qr() produced a tuple containing two matrices Q and R. Q is a 3x3 orthogonal matrix. And R is a 2x2 upper triangular matrix. So that QT=Q−1\mathbf{Q}^T = \mathbf{Q}^{-1} (the transpose is equal the inverse):

Matrix(Q') ≈ Matrix(Q^-1)
MethodError: no method matching ^(::LinearAlgebra.AdjointQ{Float64, LinearAlgebra.QRCompactWYQ{Float64, Matrix{Float64}, Matrix{Float64}}}, ::Int64)
+
+Closest candidates are:
+  ^(!Matched::Float16, ::Integer)
+   @ Base math.jl:1283
+  ^(!Matched::Regex, ::Integer)
+   @ Base regex.jl:863
+  ^(!Matched::Float32, ::Integer)
+   @ Base math.jl:1277
+  ...
+
+

Also note that QT⋅Q−1=I\mathbf{Q}^T \cdot \mathbf{Q}^{-1} = \mathbf{I} (identity matrix):

Q' * Q ≈ I(3)
true
+

This is nice. But what can we do with QR decomposition? It can speed up Turing's sampling by a huge factor while also decorrelating the columns of X\mathbf{X}, i.e. the independent variables. The orthogonal nature of QR decomposition alters the posterior's topology and makes it easier for HMC or other MCMC samplers to explore it. Let's see how fast we can get with QR decomposition. First, let's go back to the kidiq example in 6. Bayesian Linear Regression:

+
using Turing
+using LinearAlgebra: I
+using Statistics: mean, std
+using Random: seed!
+seed!(123)
+
+@model function linreg(X, y; predictors=size(X, 2))
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))
+    β ~ filldist(TDist(3), predictors)
+    σ ~ Exponential(1)
+
+    #likelihood
+    return y ~ MvNormal(α .+ X * β, σ^2 * I)
+end;
+
+using DataFrames
+using CSV
+using HTTP
+
+url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/kidiq.csv"
+kidiq = CSV.read(HTTP.get(url).body, DataFrame)
+X = Matrix(select(kidiq, Not(:kid_score)))
+y = kidiq[:, :kid_score]
+model = linreg(X, y)
+chain = sample(model, NUTS(), MCMCThreads(), 1_000, 4)
Chains MCMC chain (1000×17×4 Array{Float64, 3}):
+
+Iterations        = 501:1:1500
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 6.24 seconds
+Compute duration  = 23.56 seconds
+parameters        = α, β[1], β[2], β[3], σ
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α   21.5126    8.6720    0.2217   1528.8886   1868.2612    1.0009       64.8962
+        β[1]    2.0014    1.7943    0.0419   2281.1940   1734.6512    1.0016       96.8290
+        β[2]    0.5788    0.0584    0.0013   2163.9754   2292.8814    1.0006       91.8534
+        β[3]    0.2566    0.3092    0.0074   1762.0214   2135.6795    1.0010       74.7919
+           σ   17.8859    0.6033    0.0106   3271.1669   2347.2435    1.0008      138.8500
+
+Quantiles
+  parameters      2.5%     25.0%     50.0%     75.0%     97.5%
+      Symbol   Float64   Float64   Float64   Float64   Float64
+
+           α    4.7278   15.7633   21.2942   27.4322   38.4426
+        β[1]   -0.5876    0.7324    1.6761    2.9919    6.3388
+        β[2]    0.4662    0.5392    0.5793    0.6184    0.6924
+        β[3]   -0.3477    0.0440    0.2588    0.4733    0.8490
+           σ   16.7525   17.4685   17.8796   18.2703   19.1238
+
+

See the wall duration in Turing's chain: for me it took around 24 seconds.

+

Now let's us incorporate QR decomposition in the linear regression model. Here, I will use the "thin" instead of the "fat" QR, which scales the Q\mathbf{Q} and R\mathbf{R} matrices by a factor of n−1\sqrt{n-1} where nn is the number of rows of X\mathbf{X}. In practice it is better to implement the thin QR decomposition, which is to be preferred to the fat QR decomposition. It is numerically more stable. Mathematically, the thin QR decomposition is:

+x=Q∗R∗Q∗=Q⋅n−1R∗=1n−1⋅Rμ=α+X⋅β+σ=α+Q∗⋅R∗⋅β+σ=α+Q∗⋅(R∗⋅β)+σ=α+Q∗⋅βundefined+σ +\begin{aligned} +x &= \mathbf{Q}^* \mathbf{R}^* \\ +\mathbf{Q}^* &= \mathbf{Q} \cdot \sqrt{n - 1} \\ +\mathbf{R}^* &= \frac{1}{\sqrt{n - 1}} \cdot \mathbf{R}\\ +\boldsymbol{\mu} +&= \alpha + \mathbf{X} \cdot \boldsymbol{\beta} + \sigma +\\ +&= \alpha + \mathbf{Q}^* \cdot \mathbf{R}^* \cdot \boldsymbol{\beta} + \sigma +\\ +&= \alpha + \mathbf{Q}^* \cdot (\mathbf{R}^* \cdot \boldsymbol{\beta}) + \sigma +\\ +&= \alpha + \mathbf{Q}^* \cdot \widetilde{\boldsymbol{\beta}} + \sigma +\\ +\end{aligned} + +

Then we can recover the original β\boldsymbol{\beta} with:

+β=R∗−1⋅βundefined \boldsymbol{\beta} = \mathbf{R}^{*-1} \cdot \widetilde{\boldsymbol{\beta}} +

In Turing, a model with QR decomposition would be the same linreg but with a different X matrix supplied, since it is a data transformation. First, we decompose your model data X into Q and R:

+
Q, R = qr(X)
+Q_ast = Matrix(Q) * sqrt(size(X, 1) - 1)
+R_ast = R / sqrt(size(X, 1) - 1);
+

Then, we instantiate a model with Q instead of X and sample as you would:

+
model_qr = linreg(Q_ast, y)
+chain_qr = sample(model_qr, NUTS(1_000, 0.65), MCMCThreads(), 1_000, 4)
Chains MCMC chain (1000×17×4 Array{Float64, 3}):
+
+Iterations        = 1001:1:2000
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 2.65 seconds
+Compute duration  = 7.82 seconds
+parameters        = α, β[1], β[2], β[3], σ
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters       mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol    Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α    32.9057    7.8274    0.2470   1006.3265   1242.3293    1.0026      128.7521
+        β[1]   -49.9922    7.0245    0.2210   1009.4898   1401.5358    1.0022      129.1568
+        β[2]    22.0858    3.5735    0.1101   1054.2580   1418.8568    1.0028      134.8846
+        β[3]     0.2869    0.8775    0.0238   1370.8734   1800.6496    1.0010      175.3932
+           σ    17.8703    0.5859    0.0113   2699.9100   2464.9195    1.0019      345.4337
+
+Quantiles
+  parameters       2.5%      25.0%      50.0%      75.0%      97.5%
+      Symbol    Float64    Float64    Float64    Float64    Float64
+
+           α    17.6798    27.6922    32.7076    38.1740    47.9793
+        β[1]   -63.6746   -54.6970   -50.1683   -45.2700   -36.2948
+        β[2]    15.1066    19.7019    22.1804    24.4485    29.1569
+        β[3]    -1.3554    -0.2981     0.2697     0.8374     2.1505
+           σ    16.7293    17.4690    17.8683    18.2608    19.0084
+
+

See the wall duration in Turing's chain_qr: for me it took around 5 seconds. Much faster than the regular linreg. Now we have to reconstruct our β\boldsymbol{\beta}s:

+
betas = mapslices(
+    x -> R_ast^-1 * x, chain_qr[:, namesingroup(chain_qr, :β), :].value.data; dims=[2]
+)
+chain_beta = setrange(
+    Chains(betas, ["real_β[$i]" for i in 1:size(Q_ast, 2)]), 1_001:1:2_000
+)
+chain_qr_reconstructed = hcat(chain_beta, chain_qr)
Chains MCMC chain (1000×20×4 Array{Float64, 3}):
+
+Iterations        = 1001:1:2000
+Number of chains  = 4
+Samples per chain = 1000
+parameters        = real_β[1], real_β[2], real_β[3], α, β[1], β[2], β[3], σ
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters       mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol    Float64   Float64   Float64     Float64     Float64   Float64       Missing
+
+   real_β[1]     6.2103    2.1904    0.0339   4190.5377   2075.4888    1.0015       missing
+   real_β[2]     0.5062    0.0616    0.0015   1657.4420   2135.3528    1.0021       missing
+   real_β[3]    -0.0701    0.2143    0.0058   1370.8734   1800.6496    1.0010       missing
+           α    32.9057    7.8274    0.2470   1006.3265   1242.3293    1.0026       missing
+        β[1]   -49.9922    7.0245    0.2210   1009.4898   1401.5358    1.0022       missing
+        β[2]    22.0858    3.5735    0.1101   1054.2580   1418.8568    1.0028       missing
+        β[3]     0.2869    0.8775    0.0238   1370.8734   1800.6496    1.0010       missing
+           σ    17.8703    0.5859    0.0113   2699.9100   2464.9195    1.0019       missing
+
+Quantiles
+  parameters       2.5%      25.0%      50.0%      75.0%      97.5%
+      Symbol    Float64    Float64    Float64    Float64    Float64
+
+   real_β[1]     1.8485     4.7441     6.2339     7.7050    10.3766
+   real_β[2]     0.3815     0.4656     0.5071     0.5480     0.6252
+   real_β[3]    -0.5252    -0.2045    -0.0659     0.0728     0.3310
+           α    17.6798    27.6922    32.7076    38.1740    47.9793
+        β[1]   -63.6746   -54.6970   -50.1683   -45.2700   -36.2948
+        β[2]    15.1066    19.7019    22.1804    24.4485    29.1569
+        β[3]    -1.3554    -0.2981     0.2697     0.8374     2.1505
+           σ    16.7293    17.4690    17.8683    18.2608    19.0084
+
+

Non-Centered Parametrization

+

Now let's us explore Non-Centered Parametrization (NCP). This is useful when the posterior's topology is very difficult to explore as has regions where HMC sampler has to change the step size LL and the ϵ\epsilon factor. This is I've showed one of the most infamous case in 5. Markov Chain Monte Carlo (MCMC): Neal's Funnel (Neal, 2003):

+
using CairoMakie
+using Distributions
+funnel_y = rand(Normal(0, 3), 10_000)
+funnel_x = rand(Normal(), 10_000) .* exp.(funnel_y / 2)
+
+f, ax, s = scatter(
+    funnel_x,
+    funnel_y;
+    color=(:steelblue, 0.3),
+    axis=(; xlabel=L"X", ylabel=L"Y", limits=(-100, 100, nothing, nothing)),
+)
+

Neal's Funnel

+

Here we see that in upper part of the funnel HMC has to take few steps LL and be more liberal with the ϵ\epsilon factor. But, the opposite is in the lower part of the funnel: way more steps LL and be more conservative with the ϵ\epsilon factor.

+

To see the devil's funnel (how it is known in some Bayesian circles) in action, let's code it in Turing and then sample:

+
@model function funnel()
+    y ~ Normal(0, 3)
+    return x ~ Normal(0, exp(y / 2))
+end
+
+chain_funnel = sample(funnel(), NUTS(), MCMCThreads(), 1_000, 4)
Chains MCMC chain (1000×14×4 Array{Float64, 3}):
+
+Iterations        = 501:1:1500
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 5.91 seconds
+Compute duration  = 23.42 seconds
+parameters        = y, x
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters      mean       std      mcse   ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64    Float64    Float64   Float64       Float64
+
+           y    0.3304    2.5005    0.4266    34.1757    64.0933    1.0981        1.4589
+           x   -0.7113    8.8403    0.6150   467.3926   188.6939    1.0944       19.9527
+
+Quantiles
+  parameters       2.5%     25.0%     50.0%     75.0%     97.5%
+      Symbol    Float64   Float64   Float64   Float64   Float64
+
+           y    -3.2368   -2.1271    0.0819    1.9243    5.8793
+           x   -13.3185   -0.6603   -0.2764    0.4686    6.4055
+
+

Wow, take a look at those rhat values... That sucks: all are above 1.01 even with 4 parallel chains with 1,000 iterations!

+

How do we deal with that? We reparametrize! Note that we can add two normal distributions in the following manner:

+Normal(μ,σ)=Standard Normal⋅σ+μ \text{Normal}(\mu, \sigma) = \text{Standard Normal} \cdot \sigma + \mu +

where the standard normal is the normal with mean μ=0\mu = 0 and standard deviation σ=1\sigma = 1. This is why is called Non-Centered Parametrization because we "decouple" the parameters and reconstruct them before.

+
@model function ncp_funnel()
+    x̃ ~ Normal()
+    ỹ ~ Normal()
+    y = 3.0 * ỹ         # implies y ~ Normal(0, 3)
+    return x = exp(y / 2) * x̃  # implies x ~ Normal(0, exp(y / 2))
+end
+
+chain_ncp_funnel = sample(ncp_funnel(), NUTS(), MCMCThreads(), 1_000, 4)
Chains MCMC chain (1000×14×4 Array{Float64, 3}):
+
+Iterations        = 501:1:1500
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 5.38 seconds
+Compute duration  = 21.32 seconds
+parameters        = x̃, ỹ
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters      mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           x̃   -0.0006    1.0002    0.0161   3829.5014   2972.2855    0.9999      179.6454
+           ỹ   -0.0444    1.0054    0.0160   3917.0636   2987.0567    0.9999      183.7530
+
+Quantiles
+  parameters      2.5%     25.0%     50.0%     75.0%     97.5%
+      Symbol   Float64   Float64   Float64   Float64   Float64
+
+           x̃   -1.8916   -0.6865   -0.0231    0.6704    2.0065
+           ỹ   -2.0393   -0.7107   -0.0524    0.6362    1.9541
+
+

Much better now: all rhat are well below 1.01 (or below 0.99).

+

How we would implement this a real-world model in Turing? Let's go back to the cheese random-intercept model in 11. Multilevel Models (a.k.a. Hierarchical Models). Here was the approach that we took, also known as Centered Parametrization (CP):

+
@model function varying_intercept(
+    X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)
+)
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))       # population-level intercept
+    β ~ filldist(Normal(0, 2), predictors)  # population-level coefficients
+    σ ~ Exponential(std(y))                 # residual SD
+    #prior for variance of random intercepts
+    #usually requires thoughtful specification
+    τ ~ truncated(Cauchy(0, 2); lower=0)    # group-level SDs intercepts
+    αⱼ ~ filldist(Normal(0, τ), n_gr)       # CP group-level intercepts
+
+    #likelihood
+    ŷ = α .+ X * β .+ αⱼ[idx]
+    return y ~ MvNormal(ŷ, σ^2 * I)
+end;
+

To perform a Non-Centered Parametrization (NCP) in this model we do as following:

+
@model function varying_intercept_ncp(
+    X, idx, y; n_gr=length(unique(idx)), predictors=size(X, 2)
+)
+    #priors
+    α ~ Normal(mean(y), 2.5 * std(y))       # population-level intercept
+    β ~ filldist(Normal(0, 2), predictors)  # population-level coefficients
+    σ ~ Exponential(std(y))                 # residual SD
+
+    #prior for variance of random intercepts
+    #usually requires thoughtful specification
+    τ ~ truncated(Cauchy(0, 2); lower=0)   # group-level SDs intercepts
+    zⱼ ~ filldist(Normal(0, 1), n_gr)      # NCP group-level intercepts
+
+    #likelihood
+    ŷ = α .+ X * β .+ zⱼ[idx] .* τ
+    return y ~ MvNormal(ŷ, σ^2 * I)
+end;
+

Here we are using a NCP with the zⱼs following a standard normal and we reconstruct the group-level intercepts by multiplying the zⱼs by τ. Since the original αⱼs had a prior centered on 0 with standard deviation τ, we only have to use the multiplication by τ to get back the αⱼs.

+

Now let's see how NCP compares to the CP. First, let's redo our CP hierarchical model:

+
url = "https://raw.githubusercontent.com/storopoli/Bayesian-Julia/master/datasets/cheese.csv"
+cheese = CSV.read(HTTP.get(url).body, DataFrame)
+
+for c in unique(cheese[:, :cheese])
+    cheese[:, "cheese_$c"] = ifelse.(cheese[:, :cheese] .== c, 1, 0)
+end
+
+cheese[:, :background_int] = map(cheese[:, :background]) do b
+    if b == "rural"
+        1
+    elseif b == "urban"
+        2
+    else
+        missing
+    end
+end
+
+X = Matrix(select(cheese, Between(:cheese_A, :cheese_D)));
+y = cheese[:, :y];
+idx = cheese[:, :background_int];
+
+model_cp = varying_intercept(X, idx, y)
+chain_cp = sample(model_cp, NUTS(), MCMCThreads(), 1_000, 4)
Chains MCMC chain (1000×21×4 Array{Float64, 3}):
+
+Iterations        = 501:1:1500
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 7.72 seconds
+Compute duration  = 26.82 seconds
+parameters        = α, β[1], β[2], β[3], β[4], σ, τ, αⱼ[1], αⱼ[2]
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters       mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol    Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α    70.6388    5.4554    0.2162    915.6164    759.2940    1.0028       34.1457
+        β[1]     2.9639    1.3435    0.0265   2565.3867   2773.4856    1.0012       95.6698
+        β[2]   -10.5915    1.3904    0.0277   2522.8976   2504.4545    1.0016       94.0853
+        β[3]     6.5485    1.3619    0.0264   2668.6532   2413.9865    0.9997       99.5209
+        β[4]     1.0905    1.3750    0.0285   2338.0486   2471.5094    1.0010       87.1918
+           σ     7.4087    0.4507    0.0088   2643.0429   2581.2624    1.0001       98.5658
+           τ     6.2641    5.7017    0.1952   1222.7006   1203.5556    1.0038       45.5976
+       αⱼ[1]    -3.3228    5.3516    0.2167    865.3478    780.3551    1.0027       32.2710
+       αⱼ[2]     3.7386    5.3657    0.2173    894.2647    746.3891    1.0034       33.3494
+
+Quantiles
+  parameters       2.5%      25.0%      50.0%     75.0%     97.5%
+      Symbol    Float64    Float64    Float64   Float64   Float64
+
+           α    59.8857    68.3215    70.7533   73.2141   81.3633
+        β[1]     0.3781     2.0754     2.9757    3.8536    5.5841
+        β[2]   -13.2884   -11.5392   -10.6060   -9.6587   -7.8122
+        β[3]     3.9443     5.6304     6.5351    7.4435    9.2425
+        β[4]    -1.6999     0.1515     1.1072    2.0425    3.7349
+           σ     6.5891     7.0826     7.3872    7.7026    8.3437
+           τ     1.8181     3.2859     4.6436    7.1806   20.4141
+       αⱼ[1]   -14.0825    -5.6883    -3.3988   -1.1424    7.5331
+       αⱼ[2]    -6.5136     1.3252     3.5179    5.9215   14.7231
+
+

Now let's do the NCP hierarchical model:

+
model_ncp = varying_intercept_ncp(X, idx, y)
+chain_ncp = sample(model_ncp, NUTS(), MCMCThreads(), 1_000, 4)
Chains MCMC chain (1000×21×4 Array{Float64, 3}):
+
+Iterations        = 501:1:1500
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 11.84 seconds
+Compute duration  = 45.18 seconds
+parameters        = α, β[1], β[2], β[3], β[4], σ, τ, zⱼ[1], zⱼ[2]
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters       mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol    Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α    71.1250    3.5397    0.1218    878.2014    784.3499    1.0054       19.4365
+        β[1]     2.9093    1.2858    0.0297   1876.0233   2481.1601    1.0026       41.5206
+        β[2]   -10.6841    1.3789    0.0608    571.3700    192.4652    1.0074       12.6457
+        β[3]     6.5318    1.3379    0.0326   1713.4983   1455.9135    1.0030       37.9235
+        β[4]     1.0746    1.3279    0.0316   1767.8948   2009.5112    1.0071       39.1274
+           σ     7.3765    0.4561    0.0158    777.6440    188.6146    1.0139       17.2110
+           τ     5.0157    2.6654    0.1354    604.8463    193.9385    1.0075       13.3866
+       zⱼ[1]    -0.9156    0.7846    0.0225   1184.6917   1334.6682    1.0041       26.2199
+       zⱼ[2]     0.8326    0.8046    0.0223   1253.2053    947.4690    1.0023       27.7362
+
+Quantiles
+  parameters       2.5%      25.0%      50.0%     75.0%     97.5%
+      Symbol    Float64    Float64    Float64   Float64   Float64
+
+           α    63.6608    68.9974    71.0956   73.1433   78.5094
+        β[1]     0.3584     2.0549     2.9066    3.7865    5.4293
+        β[2]   -13.5656   -11.5643   -10.6594   -9.7752   -7.9711
+        β[3]     3.9994     5.5582     6.4857    7.4393    9.0957
+        β[4]    -1.4433     0.2116     1.0321    1.9658    3.6415
+           σ     6.5332     7.0626     7.3592    7.6827    8.3100
+           τ     1.8047     3.1023     4.2758    6.3272   11.7515
+       zⱼ[1]    -2.5756    -1.4160    -0.8862   -0.3486    0.5239
+       zⱼ[2]    -0.6131     0.2661     0.7795    1.3822    2.4805
+
+

Notice that some models are better off with a standard Centered Parametrization (as is our cheese case here). While others are better off with a Non-Centered Parametrization. But now you know how to apply both parametrizations in Turing. Before we conclude, we need to recover our original αⱼs. We can do this by multiplying zⱼ[idx] .* τ:

+
τ = summarystats(chain_ncp)[:τ, :mean]
+αⱼ = mapslices(
+    x -> x * τ, chain_ncp[:, namesingroup(chain_ncp, :zⱼ), :].value.data; dims=[2]
+)
+chain_ncp_reconstructed = hcat(
+    MCMCChains.resetrange(chain_ncp), Chains(αⱼ, ["αⱼ[$i]" for i in 1:length(unique(idx))])
+)
Chains MCMC chain (1000×23×4 Array{Float64, 3}):
+
+Iterations        = 1:1000
+Number of chains  = 4
+Samples per chain = 1000
+Wall duration     = 11.84 seconds
+Compute duration  = 45.18 seconds
+parameters        = α, β[1], β[2], β[3], β[4], σ, τ, zⱼ[1], zⱼ[2], αⱼ[1], αⱼ[2]
+internals         = lp, n_steps, is_accept, acceptance_rate, log_density, hamiltonian_energy, hamiltonian_energy_error, max_hamiltonian_energy_error, tree_depth, numerical_error, step_size, nom_step_size
+
+Summary Statistics
+  parameters       mean       std      mcse    ess_bulk    ess_tail      rhat   ess_per_sec
+      Symbol    Float64   Float64   Float64     Float64     Float64   Float64       Float64
+
+           α    71.1250    3.5397    0.1218    878.2014    784.3499    1.0054       19.4365
+        β[1]     2.9093    1.2858    0.0297   1876.0233   2481.1601    1.0026       41.5206
+        β[2]   -10.6841    1.3789    0.0608    571.3700    192.4652    1.0074       12.6457
+        β[3]     6.5318    1.3379    0.0326   1713.4983   1455.9135    1.0030       37.9235
+        β[4]     1.0746    1.3279    0.0316   1767.8948   2009.5112    1.0071       39.1274
+           σ     7.3765    0.4561    0.0158    777.6440    188.6146    1.0139       17.2110
+           τ     5.0157    2.6654    0.1354    604.8463    193.9385    1.0075       13.3866
+       zⱼ[1]    -0.9156    0.7846    0.0225   1184.6917   1334.6682    1.0041       26.2199
+       zⱼ[2]     0.8326    0.8046    0.0223   1253.2053    947.4690    1.0023       27.7362
+       αⱼ[1]    -4.5922    3.9351    0.1128   1184.6917   1334.6682    1.0041       26.2199
+       αⱼ[2]     4.1762    4.0358    0.1118   1253.2053    947.4690    1.0023       27.7362
+
+Quantiles
+  parameters       2.5%      25.0%      50.0%     75.0%     97.5%
+      Symbol    Float64    Float64    Float64   Float64   Float64
+
+           α    63.6608    68.9974    71.0956   73.1433   78.5094
+        β[1]     0.3584     2.0549     2.9066    3.7865    5.4293
+        β[2]   -13.5656   -11.5643   -10.6594   -9.7752   -7.9711
+        β[3]     3.9994     5.5582     6.4857    7.4393    9.0957
+        β[4]    -1.4433     0.2116     1.0321    1.9658    3.6415
+           σ     6.5332     7.0626     7.3592    7.6827    8.3100
+           τ     1.8047     3.1023     4.2758    6.3272   11.7515
+       zⱼ[1]    -2.5756    -1.4160    -0.8862   -0.3486    0.5239
+       zⱼ[2]    -0.6131     0.2661     0.7795    1.3822    2.4805
+       αⱼ[1]   -12.9183    -7.1023    -4.4447   -1.7483    2.6277
+       αⱼ[2]    -3.0749     1.3348     3.9098    6.9328   12.4414
+
+

References

+

Neal, Radford M. (2003). Slice Sampling. The Annals of Statistics, 31(3), 705–741. Retrieved from https://www.jstor.org/stable/3448413

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/13_epi_models/index.html b/pages/13_epi_models/index.html new file mode 100644 index 00000000..7d21e152 --- /dev/null +++ b/pages/13_epi_models/index.html @@ -0,0 +1,149 @@ + Bonus: Epidemiological Models using ODE Solvers in Turing
Bayesian Stats
Julia Logo
Code on GitHub
  1. How to code an ODE in Julia?
  2. How to use a ODE solver in a Turing Model
  3. References

Bonus: Epidemiological Models using ODE Solvers in Turing

Ok, now this is something that really makes me very excited with Julia's ecosystem. If you want to use an Ordinary Differential Equation solver in your Turing model, you don't need to code it from scratch. You've just borrow a pre-made one from DifferentialEquations.jl. This is what makes Julia so great. We can use functions and types defined in other packages into another package and it will probably work either straight out of the bat or without much effort!

For this tutorial I'll be using Brazil's COVID data from the Media Consortium. For reproducibility, we'll restrict the data to the year of 2020:

using Downloads
+using DataFrames
+using CSV
+using Chain
+using Dates
+
+url = "https://data.brasil.io/dataset/covid19/caso_full.csv.gz"
+file = Downloads.download(url)
+df = DataFrame(CSV.File(file))
+br = @chain df begin
+    filter(
+        [:date, :city] =>
+            (date, city) ->
+                date < Dates.Date("2021-01-01") &&
+                    date > Dates.Date("2020-04-01") &&
+                    ismissing(city),
+        _,
+    )
+    groupby(:date)
+    combine(
+        [
+            :estimated_population_2019,
+            :last_available_confirmed_per_100k_inhabitants,
+            :last_available_deaths,
+            :new_confirmed,
+            :new_deaths,
+        ] .=>
+            sum .=> [
+                :estimated_population_2019,
+                :last_available_confirmed_per_100k_inhabitants,
+                :last_available_deaths,
+                :new_confirmed,
+                :new_deaths,
+            ],
+    )
+end;

Let's take a look in the first observations

first(br, 5)
5×6 DataFrame
+ Row │ date        estimated_population_2019  last_available_confirmed_per_100k_inhabitants  last_available_deaths  new_confirmed  new_deaths
+     │ Date        Int64                      Float64                                        Int64                  Int64          Int64
+─────┼────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+   1 │ 2020-04-02                  210147125                                        79.6116                    305           1167          61
+   2 │ 2020-04-03                  210147125                                        90.9596                    365           1114          60
+   3 │ 2020-04-04                  210147125                                       103.622                     445           1169          80
+   4 │ 2020-04-05                  210147125                                       115.594                     496           1040          51
+   5 │ 2020-04-06                  210147125                                       125.766                     569            840          73

Also the bottom rows

last(br, 5)
5×6 DataFrame
+ Row │ date        estimated_population_2019  last_available_confirmed_per_100k_inhabitants  last_available_deaths  new_confirmed  new_deaths
+     │ Date        Int64                      Float64                                        Int64                  Int64          Int64
+─────┼────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────────
+   1 │ 2020-12-27                  210147125                                      1.22953e5                 191250          17614         337
+   2 │ 2020-12-28                  210147125                                      1.23554e5                 191788          27437         538
+   3 │ 2020-12-29                  210147125                                      1.24291e5                 192839          56371        1051
+   4 │ 2020-12-30                  210147125                                      1.25047e5                 194056          55600        1217
+   5 │ 2020-12-31                  210147125                                      1.25724e5                 195072          54469        1016

Here is a plot of the data:

using AlgebraOfGraphics
+using CairoMakie
+f = Figure()
+plt = data(br) * mapping(:date => L"t", :new_confirmed => "infected daily") * visual(Lines)
+draw!(f[1, 1], plt)

Infected in Brazil during COVID in 2020

The Susceptible-Infected-Recovered (SIR) (Grinsztajn, Semenova, Margossian & Riou, 2021) model splits the population in three time-dependent compartments: the susceptible, the infected (and infectious), and the recovered (and not infectious) compartments. When a susceptible individual comes into contact with an infectious individual, the former can become infected for some time, and then recover and become immune. The dynamics can be summarized in a system ODEs:

SIR Model

Susceptible-Infected-Recovered (SIR) model

dSdt=−βSINdIdt=βSIN−γIdRdt=γI \begin{aligned} \frac{dS}{dt} &= -\beta S \frac{I}{N}\\ \frac{dI}{dt} &= \beta S \frac{I}{N} - \gamma I \\ \frac{dR}{dt} &= \gamma I \end{aligned}

where:

  • S(t)S(t) – the number of people susceptible to becoming infected (no immunity)

  • I(t)I(t) – the number of people currently infected (and infectious)

  • R(t)R(t) – the number of recovered people (we assume they remain immune indefinitely)

  • β\beta – the constant rate of infectious contact between people

  • γ\gamma – constant recovery rate of infected individuals

How to code an ODE in Julia?

It's very easy:

  1. Use DifferentialEquations.jl

  2. Create a ODE function

  3. Choose:

    • Initial Conditions – u0u_0

    • Parameters – pp

    • Time Span – tt

    • Optional – Solver or leave blank for auto

PS: If you like SIR models checkout epirecipes/sir-julia

The following function provides the derivatives of the model, which it changes in-place. State variables and parameters are unpacked from u and p; this incurs a slight performance hit, but makes the equations much easier to read.

using DifferentialEquations
+
+function sir_ode!(du, u, p, t)
+    (S, I, R) = u
+    (β, γ) = p
+    N = S + I + R
+    infection = β * I * S / N
+    recovery = γ * I
+    @inbounds begin
+        du[1] = -infection # Susceptible
+        du[2] = infection - recovery # Infected
+        du[3] = recovery # Recovered
+    end
+    return nothing
+end;

This is what the infection would look with some fixed β and γ in a timespan of 100 days starting from day one with 1,167 infected (Brazil in April 2020):

i₀ = first(br[:, :new_confirmed])
+N = maximum(br[:, :estimated_population_2019])
+
+u = [N - i₀, i₀, 0.0]
+p = [0.5, 0.05]
+prob = ODEProblem(sir_ode!, u, (1.0, 100.0), p)
+sol_ode = solve(prob)
+f = Figure()
+plt =
+    data(stack(DataFrame(sol_ode), Not(:timestamp))) *
+    mapping(
+        :timestamp => L"t",
+        :value => L"N";
+        color=:variable => renamer(["value1" => "S", "value2" => "I", "value3" => "R"]),
+    ) *
+    visual(Lines; linewidth=3)
+draw!(f[1, 1], plt; axis=(; title="SIR Model for 100 days, β = $(p[1]), γ = $(p[2])"))

SIR ODE Solution for Brazil's 100 days of COVID in early 2020

How to use a ODE solver in a Turing Model

Please note that we are using the alternative negative binomial parameterization as specified in 8. Bayesian Regression with Count Data:

function NegativeBinomial2(μ, ϕ)
+    p = 1 / (1 + μ / ϕ)
+    r = ϕ
+
+    return NegativeBinomial(r, p)
+end
NegativeBinomial2 (generic function with 1 method)
+

Now this is the fun part. It's easy: just stick it inside!

+
using Turing
+using LazyArrays
+using Random: seed!
+seed!(123)
+
+@model function bayes_sir(infected, i₀, r₀, N)
+    #calculate number of timepoints
+    l = length(infected)
+
+    #priors
+    β ~ TruncatedNormal(2, 1, 1e-4, 10)     # using 10 because numerical issues arose
+    γ ~ TruncatedNormal(0.4, 0.5, 1e-4, 10) # using 10 because numerical issues arose
+    ϕ⁻ ~ truncated(Exponential(5); lower=0, upper=1e5)
+    ϕ = 1.0 / ϕ⁻
+
+    #ODE Stuff
+    I = i₀
+    u0 = [N - I, I, r₀] # S,I,R
+    p = [β, γ]
+    tspan = (1.0, float(l))
+    prob = ODEProblem(sir_ode!, u0, tspan, p)
+    sol = solve(
+        prob,
+        Tsit5(); # similar to Dormand-Prince RK45 in Stan but 20% faster
+        saveat=1.0,
+    )
+    solᵢ = Array(sol)[2, :] # New Infected
+    solᵢ = max.(1e-4, solᵢ) # numerical issues arose
+
+    #likelihood
+    return infected ~ arraydist(LazyArray(@~ NegativeBinomial2.(solᵢ, ϕ)))
+end;
+

Now run the model and inspect our parameters estimates. We will be using the default NUTS() sampler with 1_000 samples on only one Markov chain:

+
infected = br[:, :new_confirmed]
+r₀ = first(br[:, :new_deaths])
+model_sir = bayes_sir(infected, i₀, r₀, N)
+chain_sir = sample(model_sir, NUTS(), 1_000)
+summarystats(chain_sir[[:β, :γ]])
Summary Statistics
+  parameters      mean       std      mcse   ess_bulk   ess_tail      rhat   ess_per_sec
+      Symbol   Float64   Float64   Float64    Float64    Float64   Float64       Float64
+
+           β    1.1199    0.0292    0.0013   477.2173   505.0551    0.9996       11.2615
+           γ    1.0869    0.0296    0.0014   477.1981   493.8323    0.9996       11.2610
+
+

Hope you had learned some new bayesian computational skills and also took notice of the amazing potential of Julia's ecosystem of packages.

+

References

+

Grinsztajn, L., Semenova, E., Margossian, C. C., & Riou, J. (2021). Bayesian workflow for disease transmission modeling in Stan. ArXiv:2006.02985 [q-Bio, Stat]. http://arxiv.org/abs/2006.02985

+ +
+
+ Last modified: January 29, 2024. Website built with Franklin.jl and the Julia programming language. +
+
+
+
+
+
\ No newline at end of file diff --git a/pages/images/ML_code_breakdown.svg b/pages/images/ML_code_breakdown.svg new file mode 100644 index 00000000..e828f5ef --- /dev/null +++ b/pages/images/ML_code_breakdown.svg @@ -0,0 +1,3 @@ + + + \ No newline at end of file diff --git a/pages/images/SIR.png b/pages/images/SIR.png new file mode 100644 index 00000000..8e2c73bc Binary files /dev/null and b/pages/images/SIR.png differ diff --git a/pages/images/SIR.tex b/pages/images/SIR.tex new file mode 100644 index 00000000..ad003f00 --- /dev/null +++ b/pages/images/SIR.tex @@ -0,0 +1,20 @@ +% Taken from: +% https://github.com/codatmo/Liverpool +\documentclass{standalone} + +\usepackage{tikz} +\usetikzlibrary{shapes} +\usetikzlibrary{arrows} +\usetikzlibrary{positioning} +\usetikzlibrary{calc} +\usetikzlibrary{fit} +\begin{document} + \begin{tikzpicture}[node distance=0.75cm,auto,>=latex',every node/.append style={align=center},int/.style={draw, circle, minimum size=0.9cm}] + \node [int] (S) {$S$}; + \node [int, right=of S] (I) {$I$}; + \node [int, right=of I] (R) {$R$}; + + \path[->, auto=false] (S) edge node {$\beta S \frac{I}{N}$ \\[2.5em]} (I); + \path[->, auto=false] (I) edge node {$\gamma I$ \\[2.5em]} (R); + \end{tikzpicture} +\end{document} diff --git a/pages/images/andrey_markov.jpg b/pages/images/andrey_markov.jpg new file mode 100644 index 00000000..e3553a36 Binary files /dev/null and b/pages/images/andrey_markov.jpg differ diff --git a/pages/images/assumptions-vs-reality.jpeg b/pages/images/assumptions-vs-reality.jpeg new file mode 100644 index 00000000..ffa5266f Binary files /dev/null and b/pages/images/assumptions-vs-reality.jpeg differ diff --git a/pages/images/bayesian_workflow.dot b/pages/images/bayesian_workflow.dot new file mode 100644 index 00000000..0bb7e910 --- /dev/null +++ b/pages/images/bayesian_workflow.dot @@ -0,0 +1,16 @@ +digraph bayesian_workflow { + forcelabels = true; + graph [overlap = false, + fontsize = 10, + rankdir = LR, + dpi = 300] + node [shape = oval, + fontname = Helvetica] + A [label = "Model\nSpecification"] + B [label = "Prior\nSpecification"] + C [label = "Posterior\nInference"] + A -> B + B -> A [label = "Prior\nPredictive\nCheck"] + B -> C + C -> B [label = "Posterior\nPredictive\nCheck"] +} diff --git a/pages/images/bayesian_workflow.png b/pages/images/bayesian_workflow.png new file mode 100644 index 00000000..87110e1b Binary files /dev/null and b/pages/images/bayesian_workflow.png differ diff --git a/pages/images/benchmarks.svg b/pages/images/benchmarks.svg new file mode 100644 index 00000000..8805979f --- /dev/null +++ b/pages/images/benchmarks.svg @@ -0,0 +1 @@ +CJuliaLuaJITRustGoFortranJavaJavaScriptMatlabMathematicaPythonROctaveiteration_pi_summatrix_multiplymatrix_statisticsparse_integersprint_to_filerecursion_fibonaccirecursion_quicksortuserfunc_mandelbrotbenchmark100101102103104 \ No newline at end of file diff --git a/pages/images/exchangeability-1.png b/pages/images/exchangeability-1.png new file mode 100644 index 00000000..79ded147 Binary files /dev/null and b/pages/images/exchangeability-1.png differ diff --git a/pages/images/exchangeability-2.png b/pages/images/exchangeability-2.png new file mode 100644 index 00000000..dceca37a Binary files /dev/null and b/pages/images/exchangeability-2.png differ diff --git a/pages/images/finetti.jpg b/pages/images/finetti.jpg new file mode 100644 index 00000000..f1ed056a Binary files /dev/null and b/pages/images/finetti.jpg differ diff --git a/pages/images/fisher.jpg b/pages/images/fisher.jpg new file mode 100644 index 00000000..1dbfa912 Binary files /dev/null and b/pages/images/fisher.jpg differ diff --git a/pages/images/george_box.jpg b/pages/images/george_box.jpg new file mode 100644 index 00000000..81e1758c Binary files /dev/null and b/pages/images/george_box.jpg differ diff --git a/pages/images/hamilton.png b/pages/images/hamilton.png new file mode 100644 index 00000000..0590a533 Binary files /dev/null and b/pages/images/hamilton.png differ diff --git a/pages/images/hastings.jpg b/pages/images/hastings.jpg new file mode 100644 index 00000000..564c471a Binary files /dev/null and b/pages/images/hastings.jpg differ diff --git a/pages/images/hierarchical.png b/pages/images/hierarchical.png new file mode 100644 index 00000000..378095e0 Binary files /dev/null and b/pages/images/hierarchical.png differ diff --git a/pages/images/josiah_gibbs.jpg b/pages/images/josiah_gibbs.jpg new file mode 100644 index 00000000..b8966c43 Binary files /dev/null and b/pages/images/josiah_gibbs.jpg differ diff --git a/pages/images/kolmogorov.jpg b/pages/images/kolmogorov.jpg new file mode 100644 index 00000000..e44b9d02 Binary files /dev/null and b/pages/images/kolmogorov.jpg differ diff --git a/pages/images/language_comparisons.svg b/pages/images/language_comparisons.svg new file mode 100644 index 00000000..cbbff8aa --- /dev/null +++ b/pages/images/language_comparisons.svg @@ -0,0 +1,3 @@ + + +
Easy
Easy
Hard
Hard
Fast
Fast
Slow
SlowViewer does not support full SVG 1.1 \ No newline at end of file diff --git a/pages/images/meme-pvalue.jpg b/pages/images/meme-pvalue.jpg new file mode 100644 index 00000000..dac18ec6 Binary files /dev/null and b/pages/images/meme-pvalue.jpg differ diff --git a/pages/images/meme-pvalue2.jpg b/pages/images/meme-pvalue2.jpg new file mode 100644 index 00000000..cb813fe0 Binary files /dev/null and b/pages/images/meme-pvalue2.jpg differ diff --git a/pages/images/neyman.jpeg b/pages/images/neyman.jpeg new file mode 100644 index 00000000..a3c82dab Binary files /dev/null and b/pages/images/neyman.jpeg differ diff --git a/pages/images/nicholas_metropolis.png b/pages/images/nicholas_metropolis.png new file mode 100644 index 00000000..e4e7923a Binary files /dev/null and b/pages/images/nicholas_metropolis.png differ diff --git a/pages/images/stanislaw.jpg b/pages/images/stanislaw.jpg new file mode 100644 index 00000000..417caa44 Binary files /dev/null and b/pages/images/stanislaw.jpg differ diff --git a/pages/images/thomas_bayes.gif b/pages/images/thomas_bayes.gif new file mode 100644 index 00000000..97ad996e Binary files /dev/null and b/pages/images/thomas_bayes.gif differ diff --git a/robots.txt b/robots.txt new file mode 100644 index 00000000..6be3683d --- /dev/null +++ b/robots.txt @@ -0,0 +1,4 @@ +Sitemap: https://storopoli.io/Bayesian-Julia/sitemap.xml + +User-agent: * +Disallow: diff --git a/sitemap.xml b/sitemap.xml new file mode 100644 index 00000000..2a840520 --- /dev/null +++ b/sitemap.xml @@ -0,0 +1,88 @@ + + + + + https://storopoli.io/Bayesian-Julia/pages/03_prob_dist/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/06_linear_reg/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/07_logistic_reg/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/10_robust_reg/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/11_multilevel_models/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/12_Turing_tricks/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/05_MCMC/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/08_ordinal_reg/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/04_Turing/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/01_why_Julia/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/02_bayes_stats/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/09_count_reg/index.html + 2024-01-29 + monthly + 0.5 + + + https://storopoli.io/Bayesian-Julia/pages/13_epi_models/index.html + 2024-01-29 + monthly + 0.5 + + diff --git a/tag/code/index.html b/tag/code/index.html new file mode 100644 index 00000000..1d5e6d8d --- /dev/null +++ b/tag/code/index.html @@ -0,0 +1 @@ + Tag: code
Bayesian Stats
Julia Logo
Code on GitHub
\ No newline at end of file diff --git a/tag/syntax/index.html b/tag/syntax/index.html new file mode 100644 index 00000000..a7bed781 --- /dev/null +++ b/tag/syntax/index.html @@ -0,0 +1 @@ + Tag: syntax
Bayesian Stats
Julia Logo
Code on GitHub
\ No newline at end of file