From d45c9234d684f63b064a9054b20a661b2686729a Mon Sep 17 00:00:00 2001 From: DecoderLiu <105264284+DecoderLiu@users.noreply.github.com> Date: Wed, 10 Jul 2024 15:28:54 -0400 Subject: [PATCH 01/28] Update research.rst (#1795) --- docs/user/research.rst | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/docs/user/research.rst b/docs/user/research.rst index e7469091c..3eb887919 100644 --- a/docs/user/research.rst +++ b/docs/user/research.rst @@ -3,7 +3,7 @@ Research DeepXDE has been used in -- > 150 universities, e.g., +- > 190 universities, e.g., `Harvard University `_, `Massachusetts Institute of Technology `_, `Stanford University `_, @@ -14,6 +14,7 @@ DeepXDE has been used in `Princeton University `_, `Imperial College London `_, `University of California, Los Angeles `_, + `National University of Singapore `_, `Cornell University `_, `Nanyang Technological University `_, `New York University Abu Dhabi `_, @@ -33,6 +34,7 @@ DeepXDE has been used in `University of Illinois at Urbana-Champaign `_, `Technical University of Munich `_, `University of Southern California `_, + `University of Copenhagen `_, `Pennsylvania State University `_, `University of California Irvine `_, `University of Birmingham `_, @@ -52,8 +54,10 @@ DeepXDE has been used in `Kyoto University `_, `University of Liverpool `_, `Texas A&M University `_, + `Sun Yat-Sen University `_, `University of Basel `_, `Arizona State University `_, + `George Mason University `_, `University of Massachusetts Amherst `_, `Delft University of Technology `_, `Tongji University `_, @@ -148,13 +152,18 @@ DeepXDE has been used in `Bundeswehr University Munich `_, `Adolfo Ibáñez University `_, `Bauhaus-Universität Weimar `_, - `Henan Institute of Economics and Trade `_ + `Henan Institute of Economics and Trade `_, `Swansea University `_, `National University of Defence Technology `_, `University of Applied Sciences and Arts Northwestern Switzerland `_, `University of Los Andes `_, `Kuwait University `_, - `Boise State University `_ + `University of Engineering and Management `_, + `Boise State University `_, + `International School for Advanced Studies `_, + `University of Bayreuth `_, + `ITMO University `_, + `University of Calcutta `_ - > 30 national labs and research institutes, e.g., `Pacific Northwest National Laboratory `_, `Sandia National Laboratories `_, @@ -187,11 +196,13 @@ DeepXDE has been used in `Rutherford Appleton Laboratory `_, `Mathematical Sciences Research Laboratory `_, `German Aerospace Center `_, - `Materials Center Leoben Forschung GmbH `_ + `Materials Center Leoben Forschung GmbH `_, + `Yanqi Lake Beijing Institute of Mathematical Sciences and Applications `_ - > 10 industry, e.g., `Anailytica `_, `Ansys `_, `BirenTech Research `_, + `Bosch `_, `ExxonMobil `_, `General Motors `_, `RocketML `_, @@ -208,24 +219,35 @@ PINN #. J\. Seo. `Solving real-world optimization tasks using physics-informed neural computing `_. *Scientific Reports*, 14(1), 202, 2024. #. J\. Wu, Y. Wu, G. Zhang, & Y. Zhang. `Variable linear transformation improved physics-informed neural networks to solve thin-layer flow problems `_. *Journal of Computational Physics*, 112761, 2024. +#. B\. Bhaumik, S. De, & S. Changdar. `Deep learning based solution of nonlinear partial differential equations arising in the process of arterial blood flow `_. *Mathematics and Computers in Simulation*, 217, 21-36, 2024. +#. D\. Coscia, N. Demo, & G. Rozza. `PINA: a PyTorch framework for solving differential equations by deep learning for research and production environments `_. *ICLR 2024 Workshop on AI4DifferentialEquations In Science*, 2024. +#. S\. Liu, C. Su, J. Yao, Z. Hao, H. Su, Y. Wu, & J. Zhu. `Preconditioning for physics-informed neural networks ` *arXiv preprint arXiv:2402.00531*, 2024. #. N\. Patel, A. Aykutalp, & P. Laguna. `Calculating quasi-normal modes of Schwarzschild black holes with physics informed neural networks `_. *arXiv preprint arXiv:2401.01440*, 2024. #. J\. Li, Y. Lin, & K. Zhang. `Dynamic mode decomposition of the core surface flow inverted from geomagnetic field models `_. *Geophysical Research Letters*, 51(1), e2023GL106362, 2024. +#. G\. Lau, A. Hemachandra, S. Ng, & B. Low. `PINNACLE: PINN adaptive colLocation and experimental points selection `_. *The Twelfth International Conference on Learning Representations*, 2024. #. J\. M. Tucny, M. Durve, A. Montessori, and S. Succi. `Learning of viscosity functions in rarefied gas flows with physics-informed neural networks `_. *Computers Fluids*, 269:106114, 2024. #. P\. Brendel, V. Medvedev, & A. Rosskopf. `Physics-informed neural networks for magnetostatic problems on axisymmetric transformer geometries `_. *IEEE Journal of Emerging and Selected Topics in Industrial Electronics*, 2023. #. T\. Zhang, D. Wang, & Y. Lu. `RheologyNet: A physics-informed neural network solution to evaluate the thixotropic properties of cementitious materials `_. *Cement and Concrete Research*, 168, 107157, 2023. #. S\. C. Salas, A. O. Alvarado, F. Ortega-culaciati, & P. escapil-inchauspé. `Physics informed neural network for quasistatic fault slip forward and inverse problems `_. 2023. #. Z\. Wang, R. Keller, X. Deng, K. Hoshino, T. Tanaka, & Y. Nakahira. `Physics-informed representation and learning: Control and risk quantification `_. *arXiv preprint arXiv:2312.10594*, 2023. +#. C\. Li, & Z. Han. `Shallow water equations-fused dam-break wave propagation prediction model ensembled with a training process resampling method `_. *2023 International Conference on Intelligent Computing and Next Generation Networks（ICNGN)*, 1-6. 10.1109/ICNGN59831.2023.10396666. +#. X\. Yang, Y. Du, L. Li, Z. Zhou, & X. Zhang. `Physics-Informed Neural Network for Model Prediction and Dynamics Parameter Identification of Collaborative Robot Joints `_. *IEEE Robotics and Automation Letters*, vol. 8, no. 12, pp. 8462-8469, 2023. #. S\. H. Radbakhsh, K. Zandi, & M. Nik-bakht. `Physics-informed neural network for analyzing elastic beam behavior `_. *Structural Health Monitoring*, 2023. #. J\. Gong, Y. Han, J. Wu, & G. Hu. `Dynamical behavior of a particle-doped multi-segment dielectric elastomer minimal energy structure `_. *Smart Materials and Structures*, 33(1), 015016, 2023. #. S\. Burbulla. `Physics-informed neural networks for transformed geometries and manifolds `_. *arXiv preprint arXiv:2311.15940*, 2023. +#. J\. Shi, K. Manjunatha, & S. Reese. `Deep learning-based surrogate modeling of coronary in-stent restenosis `_. *Proceedings in Applied Mathematics and Mechanics*, 23, e202300090. +#. Y\. Jiang, W. Yang, Y. Zhu, & L. Hong. `Entropy structure informed learning for solving inverse problems of differential equations `_. *Chaos, Solitons & Fractals*, Volume 175, Part 2, 2023. +#. A\. Ogueda-Oliva, & P. Seshaiyer. `Literate programming for motivating and teaching neural network-based approaches to solve differential equations `_. *International Journal of Mathematical Education in Science and Technology*, 55(2), 509–542. #. B\. Jang, A. A. Kaptanoglu, R. Gaur, S. Pan, M. Landreman, & W. Dorland. `Grad-Shafranov equilibria via data-free physics informed neural networks `_. *arXiv preprint arXiv:2311.13491*, 2023. #. C\. Li. `Enhancing Navier-Stokes flow learning through the level set approach `_. *Available at SSRN 4641595*. #. X\. Zhu, X. Hu, & P. Sun. `Physics-informed neural networks for solving dynamic two-phase interface problems `_. *SIAM Journal on Scientific Computing*, 45(6), A2912-A2944, 2023. #. H\. Patel, A. Panda, T. Nikolaienko, S. Jaso, A. Lopez, & K. Kalyanaraman. `Accurate and fast Fischer-Tropsch reaction microkinetics using PINNs `_. *arXiv preprint arXiv:2311.10456*, 2023. #. J\. Plata Salas. `Física asistida por redes neuronales artificiales `_. *Repositorio Nacional CONACYT*, 2023. #. N\. Namaki, M. R. Eslahchi, & R. Salehi. `The use of physics-informed neural network approach to image restoration via nonlinear PDE tools `_. *Computers & Mathematics with Applications*, 152, 355-363, 2023. +#. A\. Hvatov, D. Aminev, & N. Demyanchuk. `Easy to learn hard to master - how to solve an arbitrary equation with PINN `_. *NeurIPS 2023 AI for Science Workshop*, 2023. #. H\. Son, H. Cho, & H. J. Hwang. `Physics-informed neural networks for microprocessor thermal management model `_. *IEEE Access*, 11, 122974-122979, 2023. #. S\. Savović, M. Ivanović, & R. Min. `A comparative study of the explicit finite difference method and physics-informed neural networks for solving the Burgers’ equation `_. *Axioms*, 12(10), 982, 2023. +#. M\. Marian, S. Tremmel. `Physics-Informed Machine Learning—An Emerging Trend in Tribology `_. *Lubricants*, 2023, 11, 463. #. L\. S. de Oliveira, L. Kunstmann, D. Pina, D. de Oliveira, & M. Mattoso. `PINNProv: Provenance for physics-informed neural networks `_. *In 2023 International Symposium on Computer Architecture and High Performance Computing Workshops (SBAC-PADW) (pp. 16-23). IEEE*, 2023. #. Z\. Wang, Z. Zhou, W. Xu, C. Sun, & R. Yan. `Physics informed neural networks for fault severity identification of axial piston pumps `_. *Journal of Manufacturing Systems*, 71, 421-437, 2023. #. K\. Prantikos, S. Chatzidakis, L. H. Tsoukalas, & A. Heifetz. `Physics-informed neural network with transfer learning (TL-PINN) based on domain similarity measure for prediction of nuclear reactor transients `_. *Scientific Reports*, 13(1), 16840, 2023. @@ -379,6 +401,7 @@ PINN DeepONet -------- +#. J\. He, S. Kushwaha, J. Park, S. Koric, D. Abueidda, & I. Jasiuk. `Predictions of transient vector solution fields with sequential deep operator network `_. *Acta Mech*, 2024 #. K\. Leng, M. Shankar, & J. Thiyagalingam. `Zero coordinate shift: Whetted automatic differentiation for physics-informed operator learning `_. *Journal of Computational Physics*, 505, 112904, 2024. #. M\. Lamarque, L. Bhan, R. Vazquez, & M. Krstic. `Gain Scheduling with a Neural Operator for a Transport PDE with Nonlinear Recirculation `_. *arXiv preprint arXiv:2401.02511*, 2024. #. A\. Xavier. `Solving Heat Conduction Problems with DeepONets `_. 2023. From 6e76854e330dff6f72080cb0b08298e011024edf Mon Sep 17 00:00:00 2001 From: Anran Jiao <38322669+anranjiao@users.noreply.github.com> Date: Thu, 11 Jul 2024 15:31:36 -0400 Subject: [PATCH 02/28] Updated research.rst (#1797) --- docs/user/research.rst | 281 ++++++++++++++++++++++++++--------------- 1 file changed, 176 insertions(+), 105 deletions(-) diff --git a/docs/user/research.rst b/docs/user/research.rst index 3eb887919..be649d091 100644 --- a/docs/user/research.rst +++ b/docs/user/research.rst @@ -3,167 +3,188 @@ Research DeepXDE has been used in -- > 190 universities, e.g., +- > 180 universities, e.g., `Harvard University `_, `Massachusetts Institute of Technology `_, `Stanford University `_, - `California Institute of Technology `_, `University of California, Berkeley `_, + `Yale University `_, + `University of California, Los Angeles `_, + `Imperial College London `_, `Johns Hopkins University `_, `University of Pennsylvania `_, + `Tsinghua University `_, + `California Institute of Technology `_, `Princeton University `_, - `Imperial College London `_, - `University of California, Los Angeles `_, - `National University of Singapore `_, `Cornell University `_, + `National University of Singapore `_, `Nanyang Technological University `_, + `University of California, San Diego `_, + `Peking University `_, `New York University Abu Dhabi `_, `University of British Columbia `_, - `Tsinghua University `_, - `Peking University `_, - `University of Texas at Austin `_, - `University of Pittsburgh `_, + `University of Copenhagen `_, `KU Leuven `_, - `Georgia Institute of Technology `_, + `University of Pittsburgh `_, + `Zhejiang University `_, + `University of Texas at Austin `_, + `Leiden University `_, `University of Minnesota `_, `University of Manchester `_, + `University of Chinese Academy of Sciences `_, + `Georgia Institute of Technology `_, + `Boston University `_, + `University of Southern California `_, `University of Wisconsin Madison `_, + `Technical University of Munich `_, `University of California, Santa Barbara `_, - `Boston University `_, + `University of Birmingham `_, + `Pennsylvania State University `_, `University of Colorado Boulder `_, `University of Illinois at Urbana-Champaign `_, - `Technical University of Munich `_, - `University of Southern California `_, - `University of Copenhagen `_, - `Pennsylvania State University `_, `University of California Irvine `_, - `University of Birmingham `_, + `King Abdullah University of Science and Technology `_, `University of Oslo `_, - `Zhejiang University `_, `University of Florida `_, + `Aarhus University `_, + `University of Exeter `_, `University of Southampton `_, `University of California, Santa Cruz `_, - `Aarhus University `_, - `University of Chinese Academy of Sciences `_, `Carnegie Mellon University `_, + `Seoul National University `_, `Sapienza University Rome `_, - `Brown University `_, `University of Alberta `_, + `University of Liverpool `_, + `Tongji University `_, + `University of Electronic Science and Technology of China `_, + `Brown University `_, `University of Bonn `_, + `Southern University of Science and Technology `_, + `Harbin Institute of Technology `_, `Purdue University `_, `Kyoto University `_, - `University of Liverpool `_, - `Texas A&M University `_, - `Sun Yat-Sen University `_, `University of Basel `_, - `Arizona State University `_, - `George Mason University `_, + `Central South University `_, `University of Massachusetts Amherst `_, - `Delft University of Technology `_, - `Tongji University `_, - `University of Sussex `_, + `Xi’an Jiaotong University `_, `Tel Aviv University `_, + `Texas A&M University `_, + `Arizona State University `_, + `Beijing Institute of Technology `_, + `Southeast University `_, + `Delft University of Technology `_, + `University of Naples Federico II `_, + `Tianjin University `_, + `Xiamen University `_, + `University of Calgary `_, + `Beijing Normal University `_, + `Kapodistrian University `_, + `RWTH Aachen University `_, + `China University of Geosciences `_, `Rice University `_, + `Beihang University `_, + `University of Sussex `_, `University of Bergen `_, - `RWTH Aachen University `_, - `University of Naples Federico II `_, - `Central South University `_, - `Southern University of Science and Technology `_, - `Xi’an Jiaotong University `_, - `University of Electronic Science and Technology of China `_, - `Southeast University `_, `KTH Royal Institute of Technology `_, - `Florida State University `_, - `Xiamen University `_, + `Northwestern Polytechnical University `_, `Tufts University `_, + `Wuhan University of Technology `_, `Universidade do Porto `_, - `Kapodistrian University `_, - `Beihang University `_, - `University of Strasbourg `_, - `Tianjing University `_, - `Beijing Normal University `_, - `China University of Geosciences `_, - `Beijing Institute of Technology `_, - `University of Western Ontario `_, + `Florida State University `_, `University Duisburg-Essen `_, - `University of Rome Tor Vergata `_, + `University of Western Ontario `_, + `University of Strasbourg `_, + `University of Surrey `_, + `Shanghai University `_, `Chalmers University of Technology `_, - `University of Victoria `_, - `Northwestern Polytechnical University `_, - `Wuhan University of Technology `_, - `Eindhoven University of Technology `_, `Kyushu University `_, - `University of Delaware `_, + `Nagoya University `_, + `University of Johannesburg `_, + `University of Rome Tor Vergata `_, `University of Kentucky `_, - `University of Surrey `_, + `Eindhoven University of Technology `_, + `Friedrich Schiller University of Jena `_, + `University of Victoria `_, + `University of Twente `_, + `University of Houston `_, + `Fuzhou University `_, + `University of Delaware `_, `University of Mississippi `_, + `Swansea University `_, + `University of the Basque Country `_, + `Hong Kong Baptist University `_, `University of Hawaii Manoa `_, - `University of Twente `_, `Federal University of Rio de Janeiro `_, - `Shanghai University `_, - `University of Johannesburg `_, - `University of Houston `_, - `Carleton University `_, `George Mason University `_, - `Universidade do Minho `_, - `University of Seville `_, - `Università degli Studi Roma Tre `_, + `University of Sevilla `_, + `International School for Advanced Studies `_, + `Beijing University of Technology `_, + `TU Wien `_, `Beijing Jiaotong University `_, - `Dortmund University of Technology `_, - `University of Stuttgart `_, + `Universidade do Minho `_, + `Nanchang University `_, + `Carleton University `_, + `South China Normal University `_, + `Roma Tre University `_, `AmirKabir University of Technology `_, + `Sabanci University `_, + `Concordia University `_, + `Tarbiat Modares University `_, + `Graz University of Technology `_, `National University of Colombia `_, - `University of Calabria `_, `Clemson University `_, - `Rensselaer Polytechnic Institute `_, - `Tarbiat Modares University `_, - `Isfahan University of Technology `_, - `Nanchang University `_, - `South China Normal University `_, + `Dortmund University of Technology `_, + `University of Los Andes `_, + `University of Stuttgart `_, `Universidad de salamanca `_, `Harbin Engineering University `_, - `Graz University of Technology `_, - `Missouri University of Science and Technology `_, - `California Polytechnic State University `_, + `Universiti Teknologi Petronas `_, + `ITMO University `_, `University of Nevada, Las Vegas `_, - `Universiti Teknologi PETRONAS `_, - `Sabanci University `_, + `University of Bayreuth `_, + `Macau University of Science & Technology `_, + `Isfahan University of Technology `_, + `Rensselaer Polytechnic Institute `_, + `Missouri University of Science and Technology `_, + `AGH University of Krakow `_, + `University of Calabria `_, + `Ulster University `_, `University of Thessaly `_, - `University of Kragujevac `_, + `Kuwait University `_, + `Brno University of Technology `_, `Old Dominion University `_, - `Macau University of Science & Technology `_, - `Universidade Federal Fluminense `_, + `University of Kragujevac `_, + `California Polytechnic State University `_, `Chung-Ang University `_, - `Brno University of Technology `_, + `Shanghai Normal University `_, `Cadi Ayyad University `_, + `Universidad Rey Juan Carlos `_, `Zhejiang A&F University `_, + `Universidade Federal Fluminense `_, + `Pontifical Catholic University of Valparaiso `_, + `Nazarbayev University `_, + `University of A Coruña `_, + `Worcester Polytechnic Institute `_, + `Xinjiang University `_, + `University of Las Palmas de Gran Canaria `_, `Hangzhou Dianzi University `_, + `Taras Shevchenko National University Kiev `_, + `University of Calcutta `_, `University of Kaiserslautern `_, - `Universidad Rey Juan Carlos `_, - `Shanghai Normal University `_, `San Francisco State University `_, - `Taras Shevchenko National University Kiev `_, - `Worcester Polytechnic Institute `_, - `Nazarbayev University `_, - `Xinjiang University `_, + `Boise State University `_, + `Necmettin Erbakan University `_, + `Shahrekord University `_, `Technical University of Cartagena `_, - `Dong A University `_, - `Universidad de Burgos `_, - `Bundeswehr University Munich `_, `Adolfo Ibáñez University `_, + `Bundeswehr University Munich `_, + `Universidad de Burgos `_, + `Dong A University `_, `Bauhaus-Universität Weimar `_, - `Henan Institute of Economics and Trade `_, - `Swansea University `_, + `Henan Institute of Economics and Trade `_ `National University of Defence Technology `_, `University of Applied Sciences and Arts Northwestern Switzerland `_, - `University of Los Andes `_, - `Kuwait University `_, `University of Engineering and Management `_, - `Boise State University `_, - `International School for Advanced Studies `_, - `University of Bayreuth `_, - `ITMO University `_, - `University of Calcutta `_ - > 30 national labs and research institutes, e.g., `Pacific Northwest National Laboratory `_, `Sandia National Laboratories `_, @@ -197,6 +218,11 @@ DeepXDE has been used in `Mathematical Sciences Research Laboratory `_, `German Aerospace Center `_, `Materials Center Leoben Forschung GmbH `_, + `Ecole Polytechnique `_, + `Scuola Superiore Meridionale `_, + `Mitsubishi Electric Research Laboratories `_, + `Forschungszentrum Jülich `_, + `China Ship Scientific Research Center `_, `Yanqi Lake Beijing Institute of Mathematical Sciences and Applications `_ - > 10 industry, e.g., `Anailytica `_, @@ -205,11 +231,11 @@ DeepXDE has been used in `Bosch `_, `ExxonMobil `_, `General Motors `_, + `Intel Corporation `_, `RocketML `_, `Saudi Aramco `_, `Shell `_, `SoftServe `_, - `Fraunhofer Institute for Integrated Systems and Device Technology IISB `_, `Quantiph `_ Here is a list of research papers that used DeepXDE. If you would like your paper to appear here, open an issue in the GitHub "Issues" section. @@ -217,8 +243,46 @@ Here is a list of research papers that used DeepXDE. If you would like your pape PINN ---- +#. S\. K. Vemuri, T. Büchner, & J. Denzler. `Estimating soil hydraulic parameters for unsaturated flow using physics-informed neural networks `_. In *International Conference on Computational Science*, 338-351, Cham: Springer Nature Switzerland, 2024, June. +#. N\. A. Niewiadomska, P. Maczuga, A. Oliver-Serra, L. Siwik, P. Sepulveda-Salaz, A. Paszyńska, M. Paszyński, & K. Pingali. `Modeling tsunami waves at the coastline of Valparaiso area of Chile with physics informed neural networks `_. In *International Conference on Computational Science*, 204-218, Cham: Springer Nature Switzerland, 2024, June. +#. N\. Alzhanov, E. Y. K. Ng, & Y. Zhao. `Three-dimensional physics-informed neural network simulation in coronary artery trees `_. *Fluids*, 9(7), 2024. +#. S\. Sripada, A. U. Gaitonde, J. A. Weibel, & A. M. Marconnet. `Robust inverse parameter fitting of thermal properties from the laser-based Ångstrom method in the presence of measurement noise using physics-informed neural networks (PINNs) `_. *Journal of Applied Physics*, 135(22):225106, June 2024. +#. T\. Zou, T. Yajima, & Y. Kawajiri. `A parameter estimation method for chromatographic separation process based on physics-informed neural network `_. *Journal of Chromatography A*, 1730:465077, 2024. +#. H\. Wang, G. Fang, B. Gao, B. Wang, & S. Meng. `Inversion of spatially distributed elastic moduli of 2.5d woven composites based on dic strain field using PINN method `_, 2024. +#. N\. Jha & E. Mallik. `GPINN with neural tangent kernel technique for nonlinear two point boundary value problems `_. *Neural Processing Letters*, 56(3):192, May 2024. +#. H\. Zhang, L. Jiang, X. Chu, Y. Wen, L. Li, Y. Xiao, & L. Wang. `Combining physics-informed graph neural network and finite difference for solving forward and inverse spatiotemporal PDEs `_, 2024. +#. N\. Jha & E. Mallik. `Gradient-based adaptive neural network technique for two-dimensional local fractional elliptic PDEs `_. *Physica Scripta*, 99(7):076005, June 2024. +#. H\. Wu, H. Luo, Y. Ma, J. Wang, & M. Long. `RoPINN: Region optimized physics-informed neural networks `_, 2024. +#. J\. H. Harmening, F. Pioch, L. Fuhrig, F.-J. Peitzmann, D. Schramm, & O. el Moctar. `Data-assisted training of a physics-informed neural network to predict the separated Reynolds-averaged turbulent flow field around an airfoil under variable angles of attack `_. *Neural Computing and Applications*, May 2024. +#. T\. Sahin, D. Wolff, M. von Danwitz, & A. Popp. `Towards a hybrid digital twin: Physics-informed neural networks as surrogate model of a reinforced concrete beam `_, 2024. +#. H\. Nganguia & D. Palaniappan. `Ciliary propulsion through non-uniform flows `_. *Journal of Fluid Mechanics*, 986:A14, 2024. +#. A\. T. Deresse & T. T. Dufera. `Exploring physics-informed neural networks for the generalized nonlinear Sine-Gordon equation `_. *Applied Computational Intelligence and Soft Computing*, 2024(1):3328977, 2024. +#. H\. Qiumei, M. Jiaxuan, & X. Zhen. `Mass-preserving spatio-temporal adaptive PINN for Cahn-Hilliard equations with strong nonlinearity and singularity `_, 2024. +#. Z.\ Zhang, J.-H. Lee, L. Sun, & G. X. Gu. `Weak-formulated physics-informed modeling and optimization for heterogeneous digital materials `_. *PNAS Nexus*, 3(5):pgae186, May 2024. +#. S\. Gao, Q. Li, M. A. Gosalvez, X. Lin, Y. Xing, & Z. Zhou. `Helium focused ion beam damage in silicon: Physics-informed neural network modeling of Helium bubble nucleation and early growth `_, 2024. +#. J\. Son, N. Park, H. Kwak, & J. Nam. `Optimizing a physics-informed machine learning model for pulsatile shear-thinning channel flow `_. *Journal of the Japanese Society of Rheology*, 52(2):113–122, 2024. +#. Raeisi, E., Yavuz, M., Khosravifarsani, M., & Fadaei, Y. `Mathematical modeling of interactions between colon cancer and immune system with a deep learning algorithm `_. *Eur. Phys. J. Plus*, 139(4):345, 2024. +#. K\. Yuan, C. Bauinger, X. Zhang, P. Baehr, M. Kirchhart, D. Dabert, A. Tousnakhoff, P. Boudier, & M. Paulitsch. `Fully-fused multi-layer perceptrons on Intel data center GPUs `_, 2024. +#. L\. Shang, Y. Zhao, S. Zheng, J. Wang, T. Zhang, & J. Wang. `Quantification of gradient energy coefficients using physics-informed neural networks `_. *International Journal of Mechanical Sciences*, 273:109210, 2024. +#. Z\. Zhang, C. Lin, & B. Wang. `Physics-informed shape optimization using coordinate projection `_. *Scientific Reports*, 14, 6537, 2024. +#. S\. Schoder & F. Kraxberger. `Feasibility study on solving the Helmholtz equation in 3D with PINNs `_. *arXiv preprint arXiv:2403.06623*, 2024. +#. V\. Trávníková, D. Wolff, N. Dirkes, S. Elgeti, E. von Lieres, & M. Behr. `A model hierarchy for predicting the flow in stirred tanks with physics-informed neural networks `_. *arXiv preprint arXiv:2403.04576*, 2024. +#. J\. S. Arnaud, T. Mark, & C.J. McDevitt. `A physics-constrained deep learning surrogate model of the runaway electron avalanche growth rate `_. *arXiv preprint arXiv:2403.04948*, 2024. +#. Y\. D. Hu, X.H. Wang, H. Zhou, & L. Wang. `A priori knowledge-based physics-informed neural networks for electromagnetic inverse scattering `_. *IEEE Transactions on Geoscience and Remote Sensing*, 2024. +#. R\. C. Sotero, J.M. Sanchez-Bornot, & I. Shaharabi-Farahani. `Parameter estimation in brain dynamics models from resting-state fMRI data using physics-informed neural networks `_. *bioRxiv*, 2024-02. +#. W\. Wu, M. Daneker, K.T. Turner, M.A. Jolley, & L. Lu. `Identifying heterogeneous micromechanical properties of biological tissues via physics-informed neural networks `_. *ArXiv*, 2024. +#. T\. Zhang, R. Yan, S. Zhang, D. Yang, & A. Chen. `Application of Fourier feature physics-information neural network in model of pipeline conveying fluid `_. *Thin-Walled Structures*, 198, 111693, 2024. +#. S\. Alkhadhr. `Modeling a clinical acoustic information system using physics-informed machine learning `_. 2024. +#. J\. Shi, K. Manjunatha, M. Behr, F. Vogt, & S. Reese. `A physics-informed deep learning framework for modeling of coronary in-stent restenosis `_. *Biomechanics and Modeling in Mechanobiology*, 23, 615-629, 2024. +#. C\. Kou, Y. Yin, Y. Zeng, S. Jia, Y. Luo, & X. Yuan. `Physics-informed neural network integrate with unclosed mechanism model for turbulent mass transfer `_. *Chemical Engineering Science*, 288, 119752, 2024. +#. B\. Jang, A. A. Kaptanoglu, R. Gaur, S. Pan, M. Landreman, & W. Dorland. `Grad–Shafranov equilibria via data-free physics informed neural networks `_. *Physics of Plasmas*, 31, 3, 2024. +#. Z\. Wang, R. Keller, X. Deng, K. Hoshino, T. Tanaka, & Y. Nakahira. `Physics-informed representation and learning: Control and risk quantification `_. In *Proceedings of the AAAI Conference on Artificial Intelligence*, 38, 19, 21699-21707, 2024, March. +#. M\. Mircea, D. Garlaschelli, & S. Semrau. `Inference of dynamical gene regulatory networks from single-cell data with physics informed neural networks `_. *arXiv preprint arXiv:2401.07379*, 2024. +#. R\. Casado-Vara, M. Severt, A. Díaz-Longueira, Á.M.D. Rey, & J.L. Calvo-Rolle. `Dynamic malware mitigation strategies for IoT networks: A mathematical epidemiology approach `_. *Mathematics*, 12, 250, 2024. +#. P\. Karnakov, S. Litvinov, & P. Koumoutsakos. `Solving inverse problems in physics by optimizing a discrete loss: Fast and accurate learning without neural networks `_. *PNAS Nexus*, 3, pgae005, 2024. #. J\. Seo. `Solving real-world optimization tasks using physics-informed neural computing `_. *Scientific Reports*, 14(1), 202, 2024. #. J\. Wu, Y. Wu, G. Zhang, & Y. Zhang. `Variable linear transformation improved physics-informed neural networks to solve thin-layer flow problems `_. *Journal of Computational Physics*, 112761, 2024. +#. Y\. Zhu, W. Kong, J. Deng, & X. Bian. `Physics-informed neural networks for incompressible flows with moving boundaries `_. *Physics of Fluids*, 36, 1, 2024. #. B\. Bhaumik, S. De, & S. Changdar. `Deep learning based solution of nonlinear partial differential equations arising in the process of arterial blood flow `_. *Mathematics and Computers in Simulation*, 217, 21-36, 2024. #. D\. Coscia, N. Demo, & G. Rozza. `PINA: a PyTorch framework for solving differential equations by deep learning for research and production environments `_. *ICLR 2024 Workshop on AI4DifferentialEquations In Science*, 2024. #. S\. Liu, C. Su, J. Yao, Z. Hao, H. Su, Y. Wu, & J. Zhu. `Preconditioning for physics-informed neural networks ` *arXiv preprint arXiv:2402.00531*, 2024. @@ -231,7 +295,7 @@ PINN #. S\. C. Salas, A. O. Alvarado, F. Ortega-culaciati, & P. escapil-inchauspé. `Physics informed neural network for quasistatic fault slip forward and inverse problems `_. 2023. #. Z\. Wang, R. Keller, X. Deng, K. Hoshino, T. Tanaka, & Y. Nakahira. `Physics-informed representation and learning: Control and risk quantification `_. *arXiv preprint arXiv:2312.10594*, 2023. #. C\. Li, & Z. Han. `Shallow water equations-fused dam-break wave propagation prediction model ensembled with a training process resampling method `_. *2023 International Conference on Intelligent Computing and Next Generation Networks（ICNGN)*, 1-6. 10.1109/ICNGN59831.2023.10396666. -#. X\. Yang, Y. Du, L. Li, Z. Zhou, & X. Zhang. `Physics-Informed Neural Network for Model Prediction and Dynamics Parameter Identification of Collaborative Robot Joints `_. *IEEE Robotics and Automation Letters*, vol. 8, no. 12, pp. 8462-8469, 2023. +#. X\. Yang, Y. Du, L. Li, Z. Zhou, & X. Zhang. `Physics-informed neural network for model prediction and dynamics parameter identification of collaborative robot joints `_. *IEEE Robotics and Automation Letters*, vol. 8, no. 12, pp. 8462-8469, 2023. #. S\. H. Radbakhsh, K. Zandi, & M. Nik-bakht. `Physics-informed neural network for analyzing elastic beam behavior `_. *Structural Health Monitoring*, 2023. #. J\. Gong, Y. Han, J. Wu, & G. Hu. `Dynamical behavior of a particle-doped multi-segment dielectric elastomer minimal energy structure `_. *Smart Materials and Structures*, 33(1), 015016, 2023. #. S\. Burbulla. `Physics-informed neural networks for transformed geometries and manifolds `_. *arXiv preprint arXiv:2311.15940*, 2023. @@ -258,7 +322,6 @@ PINN #. J\. Pan, X. Xiao, L. Guo, & X. Feng. `A high resolution physics-informed neural networks for high-dimensional convection-diffusion-reaction equations `_. *Applied Soft Computing*, 148, 110872, 2023. #. S\. Akins, & F. Zhu. `Comparing active learning performance driven by gaussian processes or bayesian neural networks for constrained trajectory exploration `_. *arXiv preprint arXiv:2309.16114*, 2023. #. I\. Bendaoud. `Approximation theory via deep neural networks and some applications `_. -#. J\. Shi, K. Manjunatha, & S. Reese. `Deep learning-based surrogate modeling of coronary in-stent restenosis `_. *PAMM*, e202300090, 2023. #. F\. Tangsijie, & L. Wei. `The buckling analysis of thin-walled structures based on physics-informed neural networks `_. *Chinese Journal of Theoretical and Applied Mechanics*, 55(11), 2539-2553, 2023. #. J\. Ran, X. Hu, X. Yuan, A. Li, & P. Wei. `Physics-Informed neural networks based low thrust orbit transfer design for spacecraft `_. *In 2023 CAA Symposium on Fault Detection, Supervision and Safety for Technical Processes (SAFEPROCESS) (pp. 1-7). IEEE*, 2023. #. L\. Mandl, A. Mielke, S. M. Seyedpour, & T. Ricken. `Affine transformations accelerate the training of physics-informed neural networks of a one-dimensional consolidation problem `_. *Scientific Reports*, 13(1), 15566, 2023. @@ -398,10 +461,23 @@ PINN #. G\. Pang, L. Lu, & G. Karniadakis. `fPINNs: Fractional physics-informed neural networks `_. *SIAM Journal on Scientific Computing*, 41(4), A2603--A2626, 2019. #. D\. Zhang, L. Lu, L. Guo, & G. Karniadakis. `Quantifying total uncertainty in physics-informed neural networks for solving forward and inverse stochastic problems `_. *Journal of Computational Physics*, 397, 108850, 2019. -DeepONet +Deep neural operators -------- -#. J\. He, S. Kushwaha, J. Park, S. Koric, D. Abueidda, & I. Jasiuk. `Predictions of transient vector solution fields with sequential deep operator network `_. *Acta Mech*, 2024 +#. A\. Jiao, Q. Yan, J. Harlim, & L. Lu. `Solving forward and inverse PDE problems on unknown manifolds via physics-informed neural operators `_. *arXiv preprint arXiv:2407.05477v1*, 2024. +#. G\. Fabiani, I. G. Kevrekidis, C. Siettos, & A. N. Yannacopoulos. `RandONet: Shallow-networks with random projections for learning linear and nonlinear operators `_. *Computer Methods in Applied Mechanics and Engineering*, 429:117130, 2024. +#. A\. Jiao, H. He, R. Ranade, J. Pathak, & L. Lu. `One-shot learning for solution operators of partial differential equations `_. *arXiv preprint arXiv:2104.05512*, 2024. +#. L\. Xiao, G. Mei, & N. Xu. `Knowledge-integrated deep learning for predicting stochastic thermal regime of embankment in permafrost region `_. *Journal of Rock Mechanics and Geotechnical Engineering*, 2024. +#. J\. He, S. Koric, D. Abueidda, A. Najafi, & I. Jasiuk. `Geom-DeepONet: A point-cloud-based deep operator network for field predictions on 3D parameterized geometries `_. *Computer Methods in Applied Mechanics and Engineering*, 429:117130, 2024. +#. S\. Kushwaha, J. Park, S. Koric, J. He, I. Jasiuk, & D. Abueidda. `Advanced deep operator networks to predict multiphysics solution fields in materials processing and additive manufacturing `_. *Additive Manufacturing*, 88:104266, 2024. +#. S\. Zampini, U. Zerbinati, G. Turkyyiah, & D. Keyes. `PETScML: Second-order solvers for training regression problems in Scientific Machine Learning `_. In *Proceedings of the Platform for Advanced Scientific Computing Conference*, 1-12, 2024, June. +#. L\. Branca & A. Pallottini. `Emulating the interstellar medium chemistry with neural operators `_. *Astronomy & Astrophysics*, 684, A203, 2024. +#. J\. Hayford, J. Goldman-Wetzler, E. Wang, & L. Lu. `Speeding up and reducing memory usage for scientific machine learning via mixed precision `_. *Computer Methods in Applied Mechanics and Engineering*, 428, 117093, 2024. +#. K\. Kobayashi, J. Daniell, & S.B. Alam. `Improved generalization with deep neural operators for engineering systems: Path towards digital twin `_. *Engineering Applications of Artificial Intelligence*, 131, 107844, 2024. +#. K\. Kobayashi & S.B. Alam. `Deep neural operator-driven real-time inference to enable digital twin solutions for nuclear energy systems `_. *Scientific Reports*, 14, 2101, 2024. +#. H\. Liu, B. Dahal, R. Lai, & W. Liao. `Generalization error guaranteed auto-encoder-based nonlinear model reduction for operator learning `_. *arXiv preprint arXiv:2401.10490*, 2024. +#. J\. He, D. Pal, A. Najafi, D. Abueidda, S. Koric, & I. Jasiuk. `Material-response-informed DeepONet and its application to polycrystal stress–strain prediction in crystal plasticity `_. *JOM*, 1-11, 2024. +#. M\. Lamarque, L. Bhan, Y. Shi, & M. Krstic. `Adaptive neural-operator backstepping control of a benchmark hyperbolic PDE `_. *arXiv preprint arXiv:2401.07862*, 2024. #. K\. Leng, M. Shankar, & J. Thiyagalingam. `Zero coordinate shift: Whetted automatic differentiation for physics-informed operator learning `_. *Journal of Computational Physics*, 505, 112904, 2024. #. M\. Lamarque, L. Bhan, R. Vazquez, & M. Krstic. `Gain Scheduling with a Neural Operator for a Transport PDE with Nonlinear Recirculation `_. *arXiv preprint arXiv:2401.02511*, 2024. #. A\. Xavier. `Solving Heat Conduction Problems with DeepONets `_. 2023. @@ -439,8 +515,3 @@ Multi-fidelity NN #. L\. Lu, M. Dao, P. Kumar, U. Ramamurty, G. Karniadakis, & S. Suresh. `Extraction of mechanical properties of materials through deep learning from instrumented indentation `_. *Proceedings of the National Academy of Sciences*, 117(13), 7052--7062, 2020. #. X\. Meng, & G. Karniadakis. `A composite neural network that learns from multi-fidelity data: Application to function approximation and inverse PDE problems `_. *Journal of Computational Physics*, 401, 109020, 2020. - -Others ------- - -#. A\. Jiao, H. He, R. Ranade, J. Pathak, & L. Lu. `One-shot learning for solution operators of partial differential equations `_. *arXiv preprint arXiv:2104.05512*, 2021. From 439039c38a4e59e6467479804c9c989ba25c1de5 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Wed, 24 Jul 2024 23:03:35 +0800 Subject: [PATCH 03/28] Backend paddle: DeepONetCartesianProd supports multi outputs (#1799) --- deepxde/nn/paddle/deeponet.py | 127 ++++++++++++++---- .../operator/stokes_aligned_zcs_pideeponet.py | 2 +- 2 files changed, 99 insertions(+), 30 deletions(-) diff --git a/deepxde/nn/paddle/deeponet.py b/deepxde/nn/paddle/deeponet.py index 212bed406..a3ac7d0ee 100755 --- a/deepxde/nn/paddle/deeponet.py +++ b/deepxde/nn/paddle/deeponet.py @@ -4,6 +4,13 @@ from .nn import NN from .. import activations from .. import initializers +from ..deeponet_strategy import ( + SingleOutputStrategy, + IndependentStrategy, + SplitBothStrategy, + SplitBranchStrategy, + SplitTrunkStrategy, +) class DeepONet(NN): @@ -89,14 +96,40 @@ class DeepONetCartesianProd(NN): Args: layer_sizes_branch: A list of integers as the width of a fully connected network, or `(dim, f)` where `dim` is the input dimension and `f` is a network - function. The width of the last layer in the branch and trunk net should be - equal. + function. The width of the last layer in the branch and trunk net + should be the same for all strategies except "split_branch" and "split_trunk". layer_sizes_trunk (list): A list of integers as the width of a fully connected network. activation: If `activation` is a ``string``, then the same activation is used in both trunk and branch nets. If `activation` is a ``dict``, then the trunk net uses the activation `activation["trunk"]`, and the branch net uses `activation["branch"]`. + num_outputs (integer): Number of outputs. In case of multiple outputs, i.e., `num_outputs` > 1, + `multi_output_strategy` below should be set. + multi_output_strategy (str or None): ``None``, "independent", "split_both", "split_branch" or + "split_trunk". It makes sense to set in case of multiple outputs. + + - None + Classical implementation of DeepONet with a single output. + Cannot be used with `num_outputs` > 1. + + - independent + Use `num_outputs` independent DeepONets, and each DeepONet outputs only + one function. + + - split_both + Split the outputs of both the branch net and the trunk net into `num_outputs` + groups, and then the kth group outputs the kth solution. + + - split_branch + Split the branch net and share the trunk net. The width of the last layer + in the branch net should be equal to the one in the trunk net multiplied + by the number of outputs. + + - split_trunk + Split the trunk net and share the branch net. The width of the last layer + in the trunk net should be equal to the one in the branch net multiplied + by the number of outputs. """ def __init__( @@ -105,45 +138,81 @@ def __init__( layer_sizes_trunk, activation, kernel_initializer, - regularization=None, + num_outputs=1, + multi_output_strategy=None, ): super().__init__() if isinstance(activation, dict): - activation_branch = activation["branch"] + self.activation_branch = activation["branch"] self.activation_trunk = activations.get(activation["trunk"]) else: - activation_branch = self.activation_trunk = activations.get(activation) - if callable(layer_sizes_branch[1]): - # User-defined network - self.branch = layer_sizes_branch[1] - else: - # Fully connected network - self.branch = FNN(layer_sizes_branch, activation_branch, kernel_initializer) - self.trunk = FNN(layer_sizes_trunk, self.activation_trunk, kernel_initializer) - # register bias to parameter for updating in optimizer and storage - self.b = self.create_parameter( - shape=(1,), default_initializer=initializers.get("zeros") + self.activation_branch = self.activation_trunk = activations.get(activation) + self.kernel_initializer = kernel_initializer + + self.num_outputs = num_outputs + if self.num_outputs == 1: + if multi_output_strategy is not None: + raise ValueError( + "num_outputs is set to 1, but multi_output_strategy is not None." + ) + elif multi_output_strategy is None: + multi_output_strategy = "independent" + print( + f"Warning: There are {num_outputs} outputs, but no multi_output_strategy selected. " + 'Use "independent" as the multi_output_strategy.' + ) + self.multi_output_strategy = { + None: SingleOutputStrategy, + "independent": IndependentStrategy, + "split_both": SplitBothStrategy, + "split_branch": SplitBranchStrategy, + "split_trunk": SplitTrunkStrategy, + }[multi_output_strategy](self) + + self.branch, self.trunk = self.multi_output_strategy.build( + layer_sizes_branch, layer_sizes_trunk + ) + if isinstance(self.branch, list): + self.branch = paddle.nn.LayerList(self.branch) + if isinstance(self.trunk, list): + self.trunk = paddle.nn.LayerList(self.trunk) + self.b = paddle.nn.ParameterList( + [ + paddle.create_parameter( + shape=[1,], + dtype=paddle.get_default_dtype(), + default_initializer=paddle.nn.initializer.Constant(value=0), + ) + for _ in range(self.num_outputs) + ] ) - self.regularizer = regularization + + def build_branch_net(self, layer_sizes_branch): + # User-defined network + if callable(layer_sizes_branch[1]): + return layer_sizes_branch[1] + # Fully connected network + return FNN(layer_sizes_branch, self.activation_branch, self.kernel_initializer) + + def build_trunk_net(self, layer_sizes_trunk): + return FNN(layer_sizes_trunk, self.activation_trunk, self.kernel_initializer) + + def merge_branch_trunk(self, x_func, x_loc, index): + y = x_func @ x_loc.T + y += self.b[index] + return y + + @staticmethod + def concatenate_outputs(ys): + return paddle.stack(ys, axis=2) def forward(self, inputs): x_func = inputs[0] x_loc = inputs[1] - # Branch net to encode the input function - x_func = self.branch(x_func) - # Trunk net to encode the domain of the output function + # Trunk net input transform if self._input_transform is not None: x_loc = self._input_transform(x_loc) - x_loc = self.activation_trunk(self.trunk(x_loc)) - # Dot product - if x_func.shape[-1] != x_loc.shape[-1]: - raise AssertionError( - "Output sizes of branch net and trunk net do not match." - ) - x = x_func @ x_loc.T - # Add bias - x += self.b - + x = self.multi_output_strategy.call(x_func, x_loc) if self._output_transform is not None: x = self._output_transform(inputs, x) return x diff --git a/examples/operator/stokes_aligned_zcs_pideeponet.py b/examples/operator/stokes_aligned_zcs_pideeponet.py index 6f224b090..a2223aa19 100644 --- a/examples/operator/stokes_aligned_zcs_pideeponet.py +++ b/examples/operator/stokes_aligned_zcs_pideeponet.py @@ -1,4 +1,4 @@ -"""Backend supported: tensorflow, pytorch""" +"""Backend supported: tensorflow, pytorch, paddle""" import deepxde as dde import matplotlib.pyplot as plt import numpy as np From 61d3d87530c5c7b7d75961a69ddf7e57b042fa4b Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Sat, 27 Jul 2024 22:30:45 +0000 Subject: [PATCH 04/28] Tensorflow 1.x backend: layer-by-layer dropout rate setting for DeepONet (#1792) --- deepxde/nn/tensorflow_compat_v1/deeponet.py | 94 ++++++++++++++++++--- 1 file changed, 80 insertions(+), 14 deletions(-) diff --git a/deepxde/nn/tensorflow_compat_v1/deeponet.py b/deepxde/nn/tensorflow_compat_v1/deeponet.py index d8db0ddc1..ba6319490 100644 --- a/deepxde/nn/tensorflow_compat_v1/deeponet.py +++ b/deepxde/nn/tensorflow_compat_v1/deeponet.py @@ -155,7 +155,13 @@ class DeepONet(NN): both trunk and branch nets. If `activation` is a ``dict``, then the trunk net uses the activation `activation["trunk"]`, and the branch net uses `activation["branch"]`. - dropout_rate (float): The dropout rate, between 0 and 1. + dropout_rate: If `dropout_rate` is a ``float`` between 0 and 1, then the + same rate is used in both trunk and branch nets. If `dropout_rate` + is a ``dict``, then the trunk net uses the rate `dropout_rate["trunk"]`, + and the branch net uses `dropout_rate["branch"]`. Both `dropout_rate["trunk"]` + and `dropout_rate["branch"]` should be ``float`` or lists of ``float``. + The list length should match the length of `layer_size_trunk` - 1 for the + trunk net and `layer_size_branch` - 2 for the branch net. trainable_branch: Boolean. trainable_trunk: Boolean or a list of booleans. num_outputs (integer): Number of outputs. In case of multiple outputs, i.e., `num_outputs` > 1, @@ -219,7 +225,31 @@ def __init__( "stacked " + kernel_initializer ) self.regularizer = regularizers.get(regularization) - self.dropout_rate = dropout_rate + if isinstance(dropout_rate, dict): + self.dropout_rate_branch = dropout_rate["branch"] + self.dropout_rate_trunk = dropout_rate["trunk"] + else: + self.dropout_rate_branch = self.dropout_rate_trunk = dropout_rate + if isinstance(self.dropout_rate_branch, list): + if not (len(layer_sizes_branch) - 2) == len(self.dropout_rate_branch): + raise ValueError( + "Number of dropout rates of branch net must be " + f"equal to {len(layer_sizes_branch) - 2}" + ) + else: + self.dropout_rate_branch = [self.dropout_rate_branch] * ( + len(layer_sizes_branch) - 2 + ) + if isinstance(self.dropout_rate_trunk, list): + if not (len(layer_sizes_trunk) - 1) == len(self.dropout_rate_trunk): + raise ValueError( + "Number of dropout rates of trunk net must be " + f"equal to {len(layer_sizes_trunk) - 1}" + ) + else: + self.dropout_rate_trunk = [self.dropout_rate_trunk] * ( + len(layer_sizes_trunk) - 1 + ) self.use_bias = use_bias self.stacked = stacked self.trainable_branch = trainable_branch @@ -303,9 +333,11 @@ def build_branch_net(self): activation=self.activation_branch, trainable=self.trainable_branch, ) - if self.dropout_rate > 0: + if self.dropout_rate_branch[i - 1] > 0: y_func = tf.layers.dropout( - y_func, rate=self.dropout_rate, training=self.training + y_func, + rate=self.dropout_rate_branch[i - 1], + training=self.training, ) y_func = self._stacked_dense( y_func, @@ -324,9 +356,11 @@ def build_branch_net(self): regularizer=self.regularizer, trainable=self.trainable_branch, ) - if self.dropout_rate > 0: + if self.dropout_rate_branch[i - 1] > 0: y_func = tf.layers.dropout( - y_func, rate=self.dropout_rate, training=self.training + y_func, + rate=self.dropout_rate_branch[i - 1], + training=self.training, ) y_func = self._dense( y_func, @@ -351,9 +385,9 @@ def build_trunk_net(self): if isinstance(self.trainable_trunk, (list, tuple)) else self.trainable_trunk, ) - if self.dropout_rate > 0: + if self.dropout_rate_trunk[i - 1] > 0: y_loc = tf.layers.dropout( - y_loc, rate=self.dropout_rate, training=self.training + y_loc, rate=self.dropout_rate_trunk[i - 1], training=self.training ) return y_loc @@ -454,7 +488,13 @@ class DeepONetCartesianProd(NN): both trunk and branch nets. If `activation` is a ``dict``, then the trunk net uses the activation `activation["trunk"]`, and the branch net uses `activation["branch"]`. - dropout_rate (float): The dropout rate, between 0 and 1. + dropout_rate: If `dropout_rate` is a ``float`` between 0 and 1, then the + same rate is used in both trunk and branch nets. If `dropout_rate` + is a ``dict``, then the trunk net uses the rate `dropout_rate["trunk"]`, + and the branch net uses `dropout_rate["branch"]`. Both `dropout_rate["trunk"]` + and `dropout_rate["branch"]` should be ``float`` or lists of ``float``. + The list length should match the length of `layer_size_trunk` - 1 for the + trunk net and `layer_size_branch` - 2 for the branch net. num_outputs (integer): Number of outputs. In case of multiple outputs, i.e., `num_outputs` > 1, `multi_output_strategy` below should be set. multi_output_strategy (str or None): ``None``, "independent", "split_both", "split_branch" or @@ -504,7 +544,31 @@ def __init__( self.activation_branch = self.activation_trunk = activations.get(activation) self.kernel_initializer = initializers.get(kernel_initializer) self.regularizer = regularizers.get(regularization) - self.dropout_rate = dropout_rate + if isinstance(dropout_rate, dict): + self.dropout_rate_branch = dropout_rate["branch"] + self.dropout_rate_trunk = dropout_rate["trunk"] + else: + self.dropout_rate_branch = self.dropout_rate_trunk = dropout_rate + if isinstance(self.dropout_rate_branch, list): + if not (len(layer_size_branch) - 2) == len(self.dropout_rate_branch): + raise ValueError( + "Number of dropout rates of branch net must be " + f"equal to {len(layer_size_branch) - 2}" + ) + else: + self.dropout_rate_branch = [self.dropout_rate_branch] * ( + len(layer_size_branch) - 2 + ) + if isinstance(self.dropout_rate_trunk, list): + if not (len(layer_size_trunk) - 1) == len(self.dropout_rate_trunk): + raise ValueError( + "Number of dropout rates of trunk net must be " + f"equal to {len(layer_size_trunk) - 1}" + ) + else: + self.dropout_rate_trunk = [self.dropout_rate_trunk] * ( + len(layer_size_trunk) - 1 + ) self._inputs = None self.num_outputs = num_outputs @@ -571,9 +635,11 @@ def build_branch_net(self): kernel_initializer=self.kernel_initializer, kernel_regularizer=self.regularizer, ) - if self.dropout_rate > 0: + if self.dropout_rate_branch[i - 1] > 0: y_func = tf.layers.dropout( - y_func, rate=self.dropout_rate, training=self.training + y_func, + rate=self.dropout_rate_branch[i - 1], + training=self.training, ) y_func = tf.layers.dense( y_func, @@ -596,9 +662,9 @@ def build_trunk_net(self): kernel_initializer=self.kernel_initializer, kernel_regularizer=self.regularizer, ) - if self.dropout_rate > 0: + if self.dropout_rate_trunk[i - 1] > 0: y_loc = tf.layers.dropout( - y_loc, rate=self.dropout_rate, training=self.training + y_loc, rate=self.dropout_rate_trunk[i - 1], training=self.training ) return y_loc From a462f5efabcb17db6176b4ad34366ca4fdb7693f Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Mon, 29 Jul 2024 16:18:13 +0000 Subject: [PATCH 05/28] Fix warning messages (#1806) --- deepxde/nn/tensorflow_compat_v1/deeponet.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepxde/nn/tensorflow_compat_v1/deeponet.py b/deepxde/nn/tensorflow_compat_v1/deeponet.py index ba6319490..2eeb4db4c 100644 --- a/deepxde/nn/tensorflow_compat_v1/deeponet.py +++ b/deepxde/nn/tensorflow_compat_v1/deeponet.py @@ -267,7 +267,7 @@ def __init__( elif multi_output_strategy is None: multi_output_strategy = "independent" print( - "Warning: There are {num_outputs} outputs, but no multi_output_strategy selected. " + f"Warning: There are {num_outputs} outputs, but no multi_output_strategy selected. " 'Use "independent" as the multi_output_strategy.' ) self.multi_output_strategy = { @@ -580,7 +580,7 @@ def __init__( elif multi_output_strategy is None: multi_output_strategy = "independent" print( - "Warning: There are {num_outputs} outputs, but no multi_output_strategy selected. " + f"Warning: There are {num_outputs} outputs, but no multi_output_strategy selected. " 'Use "independent" as the multi_output_strategy.' ) self.multi_output_strategy = { From 0cd82d48d7ddb67d90d26cdc29065b1182e5cb5f Mon Sep 17 00:00:00 2001 From: Yiii9 <103125169+Yiii9@users.noreply.github.com> Date: Tue, 30 Jul 2024 15:40:56 -0700 Subject: [PATCH 06/28] Update research.rst (#1808) Add BioME --- docs/user/research.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/user/research.rst b/docs/user/research.rst index be649d091..e578e3b7d 100644 --- a/docs/user/research.rst +++ b/docs/user/research.rst @@ -227,6 +227,7 @@ DeepXDE has been used in - > 10 industry, e.g., `Anailytica `_, `Ansys `_, + `BioME `_, `BirenTech Research `_, `Bosch `_, `ExxonMobil `_, From f95c04670bfbff3ab09a5f5fd05770374c5980ee Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Fri, 2 Aug 2024 00:07:35 +0000 Subject: [PATCH 07/28] Add uniform_boundary_points to dde.geometry.Hypercube (#1811) --- deepxde/geometry/geometry_nd.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/deepxde/geometry/geometry_nd.py b/deepxde/geometry/geometry_nd.py index a011cd417..78a805eba 100644 --- a/deepxde/geometry/geometry_nd.py +++ b/deepxde/geometry/geometry_nd.py @@ -81,6 +81,37 @@ def uniform_points(self, n, boundary=True): ) return x + def uniform_boundary_points(self, n): + points_per_face = max(1, n // (2 * self.dim)) + points = [] + for d in range(self.dim): + for boundary in [self.xmin[d], self.xmax[d]]: + xi = [] + for i in range(self.dim): + if i == d: + xi.append(np.array([boundary], dtype=config.real(np))) + else: + ni = int(np.ceil(points_per_face ** (1 / (self.dim - 1)))) + xi.append( + np.linspace( + self.xmin[i], + self.xmax[i], + num=ni + 1, + endpoint=False, + dtype=config.real(np), + )[1:] + ) + face_points = np.array(list(itertools.product(*xi))) + points.append(face_points) + points = np.vstack(points) + if n != len(points): + print( + "Warning: {} points required, but {} points sampled.".format( + n, len(points) + ) + ) + return points + def random_points(self, n, random="pseudo"): x = sample(n, self.dim, random) return (self.xmax - self.xmin) * x + self.xmin From 0a3babe16f0374f397436e7176f16ef5827638a8 Mon Sep 17 00:00:00 2001 From: Min Zhu <84722601+MinZhu123@users.noreply.github.com> Date: Sun, 4 Aug 2024 13:23:26 -0400 Subject: [PATCH 08/28] Add clear() for forward-mode autodiff (#1813) --- deepxde/gradients/gradients.py | 2 ++ deepxde/gradients/gradients_forward.py | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/deepxde/gradients/gradients.py b/deepxde/gradients/gradients.py index 6c3bf4b87..742796b50 100644 --- a/deepxde/gradients/gradients.py +++ b/deepxde/gradients/gradients.py @@ -67,3 +67,5 @@ def clear(): """Clear cached Jacobians and Hessians.""" if config.autodiff == "reverse": gradients_reverse.clear() + elif config.autodiff == "forward": + gradients_forward.clear() diff --git a/deepxde/gradients/gradients_forward.py b/deepxde/gradients/gradients_forward.py index c4cd5d6cf..b58aa7621 100644 --- a/deepxde/gradients/gradients_forward.py +++ b/deepxde/gradients/gradients_forward.py @@ -110,3 +110,8 @@ def jacobian(ys, xs, i=None, j=None): def hessian(ys, xs, component=0, i=0, j=0): dys_xj = jacobian(ys, xs, i=None, j=j) return jacobian(dys_xj, xs, i=component, j=i) + + +def clear(): + """Clear cached Jacobians""" + jacobian._Jacobians.clear() From d0c5aa25414ca469e07564d973e1ee499aff3c13 Mon Sep 17 00:00:00 2001 From: lululxvi Date: Fri, 16 Aug 2024 11:57:24 -0400 Subject: [PATCH 09/28] Bug fix: Error checking for DeepONet --- deepxde/nn/deeponet_strategy.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/deepxde/nn/deeponet_strategy.py b/deepxde/nn/deeponet_strategy.py index e8ac5f3dc..6d502b938 100644 --- a/deepxde/nn/deeponet_strategy.py +++ b/deepxde/nn/deeponet_strategy.py @@ -27,10 +27,6 @@ class SingleOutputStrategy(DeepONetStrategy): """Single output build strategy is the standard build method.""" def build(self, layer_sizes_branch, layer_sizes_trunk): - if layer_sizes_branch[-1] != layer_sizes_trunk[-1]: - raise AssertionError( - "Output sizes of branch net and trunk net do not match." - ) branch = self.net.build_branch_net(layer_sizes_branch) trunk = self.net.build_trunk_net(layer_sizes_trunk) return branch, trunk From 7f1b87f6b7104a13d03d1c147ad060e8c6332e96 Mon Sep 17 00:00:00 2001 From: HydrogenSulfate <490868991@qq.com> Date: Sun, 1 Sep 2024 02:35:23 +0800 Subject: [PATCH 10/28] Add Paddle backend support to poisson_1d_pideeponet and heat_sample (#1828) --- examples/operator/poisson_1d_pideeponet.py | 2 +- examples/pinn_forward/heat_resample.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/operator/poisson_1d_pideeponet.py b/examples/operator/poisson_1d_pideeponet.py index 4310b503e..1b14fa035 100755 --- a/examples/operator/poisson_1d_pideeponet.py +++ b/examples/operator/poisson_1d_pideeponet.py @@ -1,4 +1,4 @@ -"""Backend supported: tensorflow.compat.v1, tensorflow, pytorch""" +"""Backend supported: tensorflow.compat.v1, tensorflow, pytorch, paddle""" import deepxde as dde import matplotlib.pyplot as plt import numpy as np diff --git a/examples/pinn_forward/heat_resample.py b/examples/pinn_forward/heat_resample.py index 464a97f28..6dfc22e37 100755 --- a/examples/pinn_forward/heat_resample.py +++ b/examples/pinn_forward/heat_resample.py @@ -1,4 +1,4 @@ -"""Backend supported: tensorflow.compat.v1, tensorflow, pytorch""" +"""Backend supported: tensorflow.compat.v1, tensorflow, pytorch, paddle""" import deepxde as dde import numpy as np From 22e613d3ab595396fc159d1cc5636f9205a9c0af Mon Sep 17 00:00:00 2001 From: Anran Jiao <38322669+anranjiao@users.noreply.github.com> Date: Sun, 1 Sep 2024 12:15:05 -0400 Subject: [PATCH 11/28] Fix a bug in quadruple.py (#1830) --- deepxde/data/quadruple.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deepxde/data/quadruple.py b/deepxde/data/quadruple.py index 2e9fc0ee4..efbe91767 100644 --- a/deepxde/data/quadruple.py +++ b/deepxde/data/quadruple.py @@ -94,7 +94,7 @@ def train_next_batch(self, batch_size=None): self.train_x[0][indices_branch], self.train_x[1][indices_branch], self.train_x[2][indices_trunk], - ), self.train_y[indices_branch, indices_trunk] + ), self.train_y[indices_branch][:, indices_trunk] def test(self): return self.test_x, self.test_y From d5a903dd70e8180ba14f165313e3b19efad76c85 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 3 Sep 2024 19:26:15 -0400 Subject: [PATCH 12/28] Bump actions/download-artifact from 3 to 4.1.7 in /.github/workflows (#1832) Bumps [actions/download-artifact](https://github.com/actions/download-artifact) from 3 to 4.1.7. - [Release notes](https://github.com/actions/download-artifact/releases) - [Commits](https://github.com/actions/download-artifact/compare/v3...v4.1.7) --- updated-dependencies: - dependency-name: actions/download-artifact dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index df0db6451..186726af9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -33,7 +33,7 @@ jobs: steps: - name: Download SDist and wheel - uses: actions/download-artifact@v3 + uses: actions/download-artifact@v4.1.7 with: name: artifact path: dist From f7aa5637cf81d943a0504cd43f7fa90fc70b505c Mon Sep 17 00:00:00 2001 From: Christopher Laurens Woolford <70346478+cwoolfo1@users.noreply.github.com> Date: Fri, 6 Sep 2024 20:47:26 -0500 Subject: [PATCH 13/28] Backend pytorch: FNN supports regularization (#1833) --- deepxde/nn/pytorch/fnn.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/deepxde/nn/pytorch/fnn.py b/deepxde/nn/pytorch/fnn.py index 873013acc..94f09e5da 100644 --- a/deepxde/nn/pytorch/fnn.py +++ b/deepxde/nn/pytorch/fnn.py @@ -9,7 +9,9 @@ class FNN(NN): """Fully-connected neural network.""" - def __init__(self, layer_sizes, activation, kernel_initializer): + def __init__( + self, layer_sizes, activation, kernel_initializer, regularization=None + ): super().__init__() if isinstance(activation, list): if not (len(layer_sizes) - 1) == len(activation): @@ -21,6 +23,7 @@ def __init__(self, layer_sizes, activation, kernel_initializer): self.activation = activations.get(activation) initializer = initializers.get(kernel_initializer) initializer_zero = initializers.get("zeros") + self.regularizer = regularization self.linears = torch.nn.ModuleList() for i in range(1, len(layer_sizes)): From ba8e8240ed7ab4ed9d30316409a9f20713027bbc Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Sun, 20 Oct 2024 22:44:15 +0000 Subject: [PATCH 14/28] Backend Tensorflow 1.x: branch subnet refactoring for DeepONet (#1849) --- deepxde/nn/tensorflow_compat_v1/deeponet.py | 85 ++++++++++++--------- 1 file changed, 47 insertions(+), 38 deletions(-) diff --git a/deepxde/nn/tensorflow_compat_v1/deeponet.py b/deepxde/nn/tensorflow_compat_v1/deeponet.py index 2eeb4db4c..f7fbe992e 100644 --- a/deepxde/nn/tensorflow_compat_v1/deeponet.py +++ b/deepxde/nn/tensorflow_compat_v1/deeponet.py @@ -318,58 +318,67 @@ def build(self): self.built = True def build_branch_net(self): - y_func = self.X_func if callable(self.layer_size_func[1]): # User-defined network - y_func = self.layer_size_func[1](y_func) - elif self.stacked: + return self.layer_size_func[1](self.X_func) + + if self.stacked: # Stacked fully connected network - stack_size = self.layer_size_func[-1] - for i in range(1, len(self.layer_size_func) - 1): - y_func = self._stacked_dense( - y_func, - self.layer_size_func[i], - stack_size, - activation=self.activation_branch, - trainable=self.trainable_branch, - ) - if self.dropout_rate_branch[i - 1] > 0: - y_func = tf.layers.dropout( - y_func, - rate=self.dropout_rate_branch[i - 1], - training=self.training, - ) + return self._build_stacked_branch_net() + + # Unstacked fully connected network + return self._build_unstacked_branch_net() + + def _build_stacked_branch_net(self): + y_func = self.X_func + stack_size = self.layer_size_func[-1] + + for i in range(1, len(self.layer_size_func) - 1): y_func = self._stacked_dense( y_func, - 1, - stack_size, - use_bias=self.use_bias, + self.layer_size_func[i], + stack_size=stack_size, + activation=self.activation_branch, trainable=self.trainable_branch, ) - else: - # Unstacked fully connected network - for i in range(1, len(self.layer_size_func) - 1): - y_func = self._dense( + if self.dropout_rate_branch[i - 1] > 0: + y_func = tf.layers.dropout( y_func, - self.layer_size_func[i], - activation=self.activation_branch, - regularizer=self.regularizer, - trainable=self.trainable_branch, + rate=self.dropout_rate_branch[i - 1], + training=self.training, ) - if self.dropout_rate_branch[i - 1] > 0: - y_func = tf.layers.dropout( - y_func, - rate=self.dropout_rate_branch[i - 1], - training=self.training, - ) + return self._stacked_dense( + y_func, + 1, + stack_size=stack_size, + use_bias=self.use_bias, + trainable=self.trainable_branch, + ) + + def _build_unstacked_branch_net(self): + y_func = self.X_func + + for i in range(1, len(self.layer_size_func) - 1): y_func = self._dense( y_func, - self.layer_size_func[-1], - use_bias=self.use_bias, + self.layer_size_func[i], + activation=self.activation_branch, regularizer=self.regularizer, trainable=self.trainable_branch, ) - return y_func + if self.dropout_rate_branch[i - 1] > 0: + y_func = tf.layers.dropout( + y_func, + rate=self.dropout_rate_branch[i - 1], + training=self.training, + ) + return self._dense( + y_func, + self.layer_size_func[-1], + use_bias=self.use_bias, + regularizer=self.regularizer, + trainable=self.trainable_branch, + ) def build_trunk_net(self): y_loc = self.X_loc From 04bbe7fde40b1b48e5d41e3a0c0982ac508ef5db Mon Sep 17 00:00:00 2001 From: Cheng Gong Date: Fri, 25 Oct 2024 20:36:30 -0400 Subject: [PATCH 15/28] change np.Inf to np.inf for numpy>2.0 (#1863) --- deepxde/callbacks.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/deepxde/callbacks.py b/deepxde/callbacks.py index 65710354c..63c5961b5 100644 --- a/deepxde/callbacks.py +++ b/deepxde/callbacks.py @@ -137,7 +137,7 @@ def __init__( self.monitor = monitor self.monitor_op = np.less self.epochs_since_last_save = 0 - self.best = np.Inf + self.best = np.inf def on_epoch_end(self): self.epochs_since_last_save += 1 @@ -221,7 +221,7 @@ def on_train_begin(self): if self.baseline is not None: self.best = self.baseline else: - self.best = np.Inf if self.monitor_op == np.less else -np.Inf + self.best = np.inf if self.monitor_op == np.less else -np.inf def on_epoch_end(self): if self.model.train_state.epoch < self.start_from_epoch: From 3ec0bc78945cafa756bcf206edacbb73b46b288c Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Sun, 27 Oct 2024 15:35:24 +0000 Subject: [PATCH 16/28] Backend Tensorflow 1.x: replace tf.layers.dense to tf.keras.layers.Dense (#1859) --- deepxde/nn/tensorflow_compat_v1/deeponet.py | 46 +++++++++++++++------ 1 file changed, 34 insertions(+), 12 deletions(-) diff --git a/deepxde/nn/tensorflow_compat_v1/deeponet.py b/deepxde/nn/tensorflow_compat_v1/deeponet.py index f7fbe992e..aa23b2263 100644 --- a/deepxde/nn/tensorflow_compat_v1/deeponet.py +++ b/deepxde/nn/tensorflow_compat_v1/deeponet.py @@ -321,7 +321,7 @@ def build_branch_net(self): if callable(self.layer_size_func[1]): # User-defined network return self.layer_size_func[1](self.X_func) - + if self.stacked: # Stacked fully connected network return self._build_stacked_branch_net() @@ -422,8 +422,7 @@ def _dense( regularizer=None, trainable=True, ): - return tf.layers.dense( - inputs, + dense = tf.keras.layers.Dense( units, activation=activation, use_bias=use_bias, @@ -431,6 +430,10 @@ def _dense( kernel_regularizer=regularizer, trainable=trainable, ) + out = dense(inputs) + if regularizer: + self.regularization_loss += tf.math.add_n(dense.losses) + return out def _stacked_dense( self, inputs, units, stack_size, activation=None, use_bias=True, trainable=True @@ -637,12 +640,11 @@ def build_branch_net(self): else: # Fully connected network for i in range(1, len(self.layer_size_func) - 1): - y_func = tf.layers.dense( + y_func = self._dense( y_func, self.layer_size_func[i], activation=self.activation_branch, - kernel_initializer=self.kernel_initializer, - kernel_regularizer=self.regularizer, + regularizer=self.regularizer, ) if self.dropout_rate_branch[i - 1] > 0: y_func = tf.layers.dropout( @@ -650,11 +652,10 @@ def build_branch_net(self): rate=self.dropout_rate_branch[i - 1], training=self.training, ) - y_func = tf.layers.dense( + y_func = self._dense( y_func, self.layer_size_func[-1], - kernel_initializer=self.kernel_initializer, - kernel_regularizer=self.regularizer, + regularizer=self.regularizer, ) return y_func @@ -664,12 +665,11 @@ def build_trunk_net(self): if self._input_transform is not None: y_loc = self._input_transform(y_loc) for i in range(1, len(self.layer_size_loc)): - y_loc = tf.layers.dense( + y_loc = self._dense( y_loc, self.layer_size_loc[i], activation=self.activation_trunk, - kernel_initializer=self.kernel_initializer, - kernel_regularizer=self.regularizer, + regularizer=self.regularizer, ) if self.dropout_rate_trunk[i - 1] > 0: y_loc = tf.layers.dropout( @@ -687,3 +687,25 @@ def merge_branch_trunk(self, branch, trunk): @staticmethod def concatenate_outputs(ys): return tf.stack(ys, axis=2) + + def _dense( + self, + inputs, + units, + activation=None, + use_bias=True, + regularizer=None, + trainable=True, + ): + dense = tf.keras.layers.Dense( + units, + activation=activation, + use_bias=use_bias, + kernel_initializer=self.kernel_initializer, + kernel_regularizer=regularizer, + trainable=trainable, + ) + out = dense(inputs) + if regularizer: + self.regularization_loss += tf.math.add_n(dense.losses) + return out From ef7e972589263e3db1f0c7b61980ab150e7f8ebd Mon Sep 17 00:00:00 2001 From: lululxvi Date: Sun, 27 Oct 2024 12:01:52 -0400 Subject: [PATCH 17/28] Update Team --- README.md | 4 ++-- docs/user/team.rst | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 0491f7a88..9bd5eb8f3 100644 --- a/README.md +++ b/README.md @@ -140,9 +140,9 @@ First off, thanks for taking the time to contribute! ## The Team -DeepXDE was developed by [Lu Lu](https://github.com/lululxvi) under the supervision of Prof. [George Karniadakis](https://www.brown.edu/research/projects/crunch/george-karniadakis) at [Brown University](https://www.brown.edu) from the summer of 2018 to 2020, supported by [PhILMs](https://www.pnnl.gov/computing/philms). DeepXDE was originally self-hosted in Subversion at Brown University, under the name SciCoNet (Scientific Computing Neural Networks). On Feb 7, 2019, SciCoNet was moved from Subversion to GitHub, renamed to DeepXDE. +DeepXDE was developed by [Lu Lu](https://github.com/lululxvi) under the supervision of Prof. [George Karniadakis](https://www.brown.edu/research/projects/crunch/george-karniadakis) at [Brown University](https://www.brown.edu) from the summer of 2018 to 2020. DeepXDE was originally self-hosted in Subversion at Brown University, under the name SciCoNet (Scientific Computing Neural Networks). On Feb 7, 2019, SciCoNet was moved from Subversion to GitHub, renamed to DeepXDE. -DeepXDE is currently maintained by [Lu Lu](https://github.com/lululxvi) at [Yale University](https://www.yale.edu) with major contributions coming from many talented individuals in various forms and means. A non-exhaustive but growing list needs to mention: [Paul Escapil-Inchauspé](https://github.com/pescap), [Jialin Li](https://github.com/lijialin03), [Saransh Chopra](https://github.com/Saransh-cpp), [Zongren Zou](https://github.com/ZongrenZou), [Sensen He](https://github.com/HydrogenSulfate), [Zhongyi Jiang](https://github.com/Jerry-Jzy), [Anran Jiao](https://github.com/anranjiao), [Shunyuan Mao](https://github.com/smao-astro). +DeepXDE is currently maintained by [Lu Lu](https://github.com/lululxvi) at [Yale University](https://www.yale.edu) with major contributions coming from many talented individuals in various forms and means. A non-exhaustive but growing list needs to mention: [Paul Escapil-Inchauspé](https://github.com/pescap), [Zongren Zou](https://github.com/ZongrenZou), [Jialin Li](https://github.com/lijialin03), [Saransh Chopra](https://github.com/Saransh-cpp), [Sensen He](https://github.com/HydrogenSulfate), [Vladimir Dudenkov](https://github.com/vl-dud), [Anran Jiao](https://github.com/anranjiao), [Zhongyi Jiang](https://github.com/Jerry-Jzy), [Shunyuan Mao](https://github.com/smao-astro). ## License diff --git a/docs/user/team.rst b/docs/user/team.rst index 684226143..9ae6a51b9 100644 --- a/docs/user/team.rst +++ b/docs/user/team.rst @@ -1,6 +1,6 @@ The Team ======== -DeepXDE was developed by `Lu Lu `_ under the supervision of Prof. `George Karniadakis `_ at `Brown University `_ from the summer of 2018 to 2020, supported by `PhILMs `_. DeepXDE was originally self-hosted in Subversion at Brown University, under the name SciCoNet (Scientific Computing Neural Networks). On Feb 7, 2019, SciCoNet was moved from Subversion to GitHub, renamed to DeepXDE. +DeepXDE was developed by `Lu Lu `_ under the supervision of Prof. `George Karniadakis `_ at `Brown University `_ from the summer of 2018 to 2020. DeepXDE was originally self-hosted in Subversion at Brown University, under the name SciCoNet (Scientific Computing Neural Networks). On Feb 7, 2019, SciCoNet was moved from Subversion to GitHub, renamed to DeepXDE. -DeepXDE is currently maintained by `Lu Lu `_ at `Yale University `_ with major contributions coming from many talented individuals in various forms and means. A non-exhaustive but growing list needs to mention: `Paul Escapil-Inchauspé `_, `Jialin Li `_, `Saransh Chopra `_, `Zongren Zou `_, `Sensen He `_, `Zhongyi Jiang `_, `Anran Jiao `_, `Shunyuan Mao `_. +DeepXDE is currently maintained by `Lu Lu `_ at `Yale University `_ with major contributions coming from many talented individuals in various forms and means. A non-exhaustive but growing list needs to mention: `Paul Escapil-Inchauspé `_, `Zongren Zou `_, `Jialin Li `_, `Saransh Chopra `_, `Sensen He `_, `Vladimir Dudenkov `_, `Anran Jiao `_, `Zhongyi Jiang `_, `Shunyuan Mao `_. From e73fe0582c1f3e055a79439159e8e825faa57a6d Mon Sep 17 00:00:00 2001 From: lululxvi Date: Sun, 27 Oct 2024 15:20:28 -0400 Subject: [PATCH 18/28] Update actions/upload-artifact to v4 --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 186726af9..07a556b42 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -18,7 +18,7 @@ jobs: run: pipx run build - name: Upload SDist and wheel - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: path: dist/* if-no-files-found: error From a53d8f737d9ba9765bef3d973649572d97a48d4b Mon Sep 17 00:00:00 2001 From: lululxvi Date: Sun, 27 Oct 2024 15:46:23 -0400 Subject: [PATCH 19/28] Update release to publish on PyPI --- .github/workflows/release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 07a556b42..53277f644 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -29,7 +29,7 @@ jobs: publish: needs: [build] runs-on: ubuntu-latest - if: github.event_name == 'release' && github.event.action == 'published' + # if: github.event_name == 'release' && github.event.action == 'published' steps: - name: Download SDist and wheel From 414e457e03ce95f9361f0b6857d70d75de13b00e Mon Sep 17 00:00:00 2001 From: lululxvi Date: Sun, 27 Oct 2024 16:18:05 -0400 Subject: [PATCH 20/28] Update pypa/gh-action-pypi-publish --- .github/workflows/release.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 53277f644..9103f0279 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -38,7 +38,7 @@ jobs: name: artifact path: dist - - name: Publish on PyPI - uses: pypa/gh-action-pypi-publish@v1.6.4 + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 with: password: ${{ secrets.PYPI_API_TOKEN }} From 40cd7e5af35b6b538cc1b9b15cc7a603975df397 Mon Sep 17 00:00:00 2001 From: lululxvi Date: Sun, 27 Oct 2024 17:01:59 -0400 Subject: [PATCH 21/28] Update actions/checkout actions/setup-python versions --- .github/workflows/build.yml | 4 ++-- .github/workflows/release.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b3395032d..214050fc5 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -23,9 +23,9 @@ jobs: os: [ubuntu-latest, macos-latest, windows-latest] steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: actions/setup-python@v4 + - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9103f0279..73e448d1f 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -10,7 +10,7 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 @@ -29,7 +29,7 @@ jobs: publish: needs: [build] runs-on: ubuntu-latest - # if: github.event_name == 'release' && github.event.action == 'published' + if: github.event_name == 'release' && github.event.action == 'published' steps: - name: Download SDist and wheel From 4c8b2d29d328d37e0ae60ceda2835e77eb37adf0 Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Wed, 6 Nov 2024 18:08:36 +0000 Subject: [PATCH 22/28] Modify Tensorflow regularizers (#1864) --- deepxde/nn/regularizers.py | 38 +++++++++++++++++++++++++++----------- 1 file changed, 27 insertions(+), 11 deletions(-) diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py index 9f65dc181..f83a871cf 100644 --- a/deepxde/nn/regularizers.py +++ b/deepxde/nn/regularizers.py @@ -2,16 +2,32 @@ def get(identifier): + """Retrieves a TensorFlow regularizer instance based on the given identifier. + + Args: + identifier (list/tuple): Specifies the type and factor(s) of the regularizer. + The first element should be one of "l1", "l2", or "l1l2" ("l1+l2"). + For "l1" and "l2", a single regularization factor is expected. + For "l1l2", provide both "l1" and "l2" factors. + """ + # TODO: other backends - if identifier is None: + if identifier is None or not identifier: return None - name, scales = identifier[0].lower(), identifier[1:] - return ( - tf.keras.regularizers.L1(l1=scales[0]) - if name == "l1" - else tf.keras.regularizers.L2(l2=scales[0]) - if name == "l2" - else tf.keras.regularizers.L1L2(l1=scales[0], l2=scales[1]) - if name in ("l1+l2", "l1l2") - else None - ) + if not isinstance(identifier, (list, tuple)): + raise ValueError("Identifier must be a list or a tuple.") + + name = identifier[0].lower() + factor = identifier[1:] + if not factor: + raise ValueError("Regularization factor must be provided.") + + if name == "l1": + return tf.keras.regularizers.L1(l1=factor[0]) + if name == "l2": + return tf.keras.regularizers.L2(l2=factor[0]) + if name in ("l1l2", "l1+l2"): + if len(factor) < 2: + raise ValueError("L1L2 regularizer requires both L1/L2 penalties.") + return tf.keras.regularizers.L1L2(l1=factor[0], l2=factor[1]) + raise ValueError(f"Unknown regularizer name: {name}") From c816a112893a88dee2d079a2b47152ed5287c2c8 Mon Sep 17 00:00:00 2001 From: Kangyu Weng <104825382+KangyuWeng@users.noreply.github.com> Date: Thu, 7 Nov 2024 11:40:31 -0500 Subject: [PATCH 23/28] Add verbose to dde.model.train and dde.model.compile (#1879) --- deepxde/model.py | 49 +++++++++++++++++++++------------------ deepxde/utils/internal.py | 3 ++- 2 files changed, 29 insertions(+), 23 deletions(-) diff --git a/deepxde/model.py b/deepxde/model.py index 24d85018d..568a16d10 100644 --- a/deepxde/model.py +++ b/deepxde/model.py @@ -65,6 +65,7 @@ def compile( decay=None, loss_weights=None, external_trainable_variables=None, + verbose=1, ): """Configures the model for training. @@ -114,8 +115,9 @@ def compile( physics systems that need to be recovered. If the backend is tensorflow.compat.v1, `external_trainable_variables` is ignored, and all trainable ``dde.Variable`` objects are automatically collected. + verbose (Integer): Controls the verbosity of the compile process. """ - if config.rank == 0: + if verbose > 0 and config.rank == 0: print("Compiling model...") self.opt_name = optimizer loss_fn = losses_module.get(loss) @@ -585,6 +587,7 @@ def train( model_restore_path=None, model_save_path=None, epochs=None, + verbose=1, ): """Trains the model. @@ -610,6 +613,7 @@ def train( model_save_path (String): Prefix of filenames created for the checkpoint. epochs (Integer): Deprecated alias to `iterations`. This will be removed in a future version. + verbose (Integer): Controls the verbosity of the train process. """ if iterations is None and epochs is not None: print( @@ -635,36 +639,36 @@ def train( if model_restore_path is not None: self.restore(model_restore_path, verbose=1) - if config.rank == 0: + if verbose > 0 and config.rank == 0: print("Training model...\n") self.stop_training = False self.train_state.set_data_train(*self.data.train_next_batch(self.batch_size)) self.train_state.set_data_test(*self.data.test()) - self._test() + self._test(verbose=verbose) self.callbacks.on_train_begin() if optimizers.is_external_optimizer(self.opt_name): if backend_name == "tensorflow.compat.v1": - self._train_tensorflow_compat_v1_scipy(display_every) + self._train_tensorflow_compat_v1_scipy(display_every, verbose=verbose) elif backend_name == "tensorflow": - self._train_tensorflow_tfp() + self._train_tensorflow_tfp(verbose=verbose) elif backend_name == "pytorch": - self._train_pytorch_lbfgs() + self._train_pytorch_lbfgs(verbose=verbose) elif backend_name == "paddle": - self._train_paddle_lbfgs() + self._train_paddle_lbfgs(verbose=verbose) else: if iterations is None: raise ValueError("No iterations for {}.".format(self.opt_name)) - self._train_sgd(iterations, display_every) + self._train_sgd(iterations, display_every, verbose=verbose) self.callbacks.on_train_end() - if config.rank == 0: + if verbose > 0 and config.rank == 0: print("") display.training_display.summary(self.train_state) if model_save_path is not None: self.save(model_save_path, verbose=1) return self.losshistory, self.train_state - def _train_sgd(self, iterations, display_every): + def _train_sgd(self, iterations, display_every, verbose=1): for i in range(iterations): self.callbacks.on_epoch_begin() self.callbacks.on_batch_begin() @@ -681,7 +685,7 @@ def _train_sgd(self, iterations, display_every): self.train_state.epoch += 1 self.train_state.step += 1 if self.train_state.step % display_every == 0 or i + 1 == iterations: - self._test() + self._test(verbose=verbose) self.callbacks.on_batch_end() self.callbacks.on_epoch_end() @@ -689,7 +693,7 @@ def _train_sgd(self, iterations, display_every): if self.stop_training: break - def _train_tensorflow_compat_v1_scipy(self, display_every): + def _train_tensorflow_compat_v1_scipy(self, display_every, verbose=1): def loss_callback(loss_train, loss_test, *args): self.train_state.epoch += 1 self.train_state.step += 1 @@ -703,7 +707,8 @@ def loss_callback(loss_train, loss_test, *args): self.train_state.loss_test, None, ) - display.training_display(self.train_state) + if verbose > 0: + display.training_display(self.train_state) for cb in self.callbacks.callbacks: if type(cb).__name__ == "VariableValue": cb.epochs_since_last += 1 @@ -736,9 +741,9 @@ def loss_callback(loss_train, loss_test, *args): fetches=fetches, loss_callback=loss_callback, ) - self._test() + self._test(verbose=verbose) - def _train_tensorflow_tfp(self): + def _train_tensorflow_tfp(self, verbose=1): # There is only one optimization step. If using multiple steps with/without # previous_optimizer_results, L-BFGS failed to reach a small error. The reason # could be that tfp.optimizer.lbfgs_minimize will start from scratch for each @@ -756,12 +761,12 @@ def _train_tensorflow_tfp(self): n_iter += results.num_iterations.numpy() self.train_state.epoch += results.num_iterations.numpy() self.train_state.step += results.num_iterations.numpy() - self._test() + self._test(verbose=verbose) if results.converged or results.failed: break - def _train_pytorch_lbfgs(self): + def _train_pytorch_lbfgs(self, verbose=1): prev_n_iter = 0 while prev_n_iter < optimizers.LBFGS_options["maxiter"]: self.callbacks.on_epoch_begin() @@ -784,7 +789,7 @@ def _train_pytorch_lbfgs(self): self.train_state.epoch += n_iter - prev_n_iter self.train_state.step += n_iter - prev_n_iter prev_n_iter = n_iter - self._test() + self._test(verbose=verbose) self.callbacks.on_batch_end() self.callbacks.on_epoch_end() @@ -792,7 +797,7 @@ def _train_pytorch_lbfgs(self): if self.stop_training: break - def _train_paddle_lbfgs(self): + def _train_paddle_lbfgs(self, verbose=1): prev_n_iter = 0 while prev_n_iter < optimizers.LBFGS_options["maxiter"]: @@ -816,7 +821,7 @@ def _train_paddle_lbfgs(self): self.train_state.epoch += n_iter - prev_n_iter self.train_state.step += n_iter - prev_n_iter prev_n_iter = n_iter - self._test() + self._test(verbose=verbose) self.callbacks.on_batch_end() self.callbacks.on_epoch_end() @@ -824,7 +829,7 @@ def _train_paddle_lbfgs(self): if self.stop_training: break - def _test(self): + def _test(self, verbose=1): # TODO Now only print the training loss in rank 0. The correct way is to print the average training loss of all ranks. ( self.train_state.y_pred_train, @@ -867,7 +872,7 @@ def _test(self): or np.isnan(self.train_state.loss_test).any() ): self.stop_training = True - if config.rank == 0: + if verbose > 0 and config.rank == 0: display.training_display(self.train_state) def predict(self, x, operator=None, callbacks=None): diff --git a/deepxde/utils/internal.py b/deepxde/utils/internal.py index c2b162f11..1b57f9d0d 100644 --- a/deepxde/utils/internal.py +++ b/deepxde/utils/internal.py @@ -21,7 +21,8 @@ def wrapper(*args, **kwargs): ts = timeit.default_timer() result = f(*args, **kwargs) te = timeit.default_timer() - if config.rank == 0: + verbose = kwargs.get('verbose', 1) + if verbose > 0 and config.rank == 0: print("%r took %f s\n" % (f.__name__, te - ts)) sys.stdout.flush() return result From 9ba2019d6f7ff4fde98d373ad4707c784b489de8 Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Sun, 10 Nov 2024 22:25:10 +0000 Subject: [PATCH 24/28] Change np.product to np.prod (#1880) --- deepxde/optimizers/tensorflow/tfp_optimizer.py | 2 +- deepxde/optimizers/tensorflow_compat_v1/tfp_optimizer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deepxde/optimizers/tensorflow/tfp_optimizer.py b/deepxde/optimizers/tensorflow/tfp_optimizer.py index b2c63c15a..aade3221b 100644 --- a/deepxde/optimizers/tensorflow/tfp_optimizer.py +++ b/deepxde/optimizers/tensorflow/tfp_optimizer.py @@ -48,7 +48,7 @@ def __init__(self, trainable_variables, build_loss): self.indices = [] # stitch indices self.partitions = [] # partition indices for i, shape in enumerate(self.shapes): - n = np.product(shape) + n = np.prod(shape) self.indices.append( tf.reshape(tf.range(count, count + n, dtype=tf.int32), shape) ) diff --git a/deepxde/optimizers/tensorflow_compat_v1/tfp_optimizer.py b/deepxde/optimizers/tensorflow_compat_v1/tfp_optimizer.py index d76b94259..7e20b863d 100644 --- a/deepxde/optimizers/tensorflow_compat_v1/tfp_optimizer.py +++ b/deepxde/optimizers/tensorflow_compat_v1/tfp_optimizer.py @@ -37,7 +37,7 @@ def __init__(self, build_loss): self.indices = [] # stitch indices self.partitions = [] # partition indices for i, shape in enumerate(self.shapes): - n = np.product(shape) + n = np.prod(shape) self.indices.append( tf.reshape(tf.range(count, count + n, dtype=tf.int32), shape) ) From bb1d3ac4f384027e89ff06c6e71213f2ef20ce29 Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Fri, 22 Nov 2024 00:25:11 +0000 Subject: [PATCH 25/28] Tensorflow 1.x backend: Unify argument names (#1887) --- deepxde/nn/tensorflow_compat_v1/deeponet.py | 34 ++++++++++----------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/deepxde/nn/tensorflow_compat_v1/deeponet.py b/deepxde/nn/tensorflow_compat_v1/deeponet.py index aa23b2263..280b1b3f7 100644 --- a/deepxde/nn/tensorflow_compat_v1/deeponet.py +++ b/deepxde/nn/tensorflow_compat_v1/deeponet.py @@ -160,8 +160,8 @@ class DeepONet(NN): is a ``dict``, then the trunk net uses the rate `dropout_rate["trunk"]`, and the branch net uses `dropout_rate["branch"]`. Both `dropout_rate["trunk"]` and `dropout_rate["branch"]` should be ``float`` or lists of ``float``. - The list length should match the length of `layer_size_trunk` - 1 for the - trunk net and `layer_size_branch` - 2 for the branch net. + The list length should match the length of `layer_sizes_trunk` - 1 for the + trunk net and `layer_sizes_branch` - 2 for the branch net. trainable_branch: Boolean. trainable_trunk: Boolean or a list of booleans. num_outputs (integer): Number of outputs. In case of multiple outputs, i.e., `num_outputs` > 1, @@ -210,7 +210,7 @@ def __init__( super().__init__() if isinstance(trainable_trunk, (list, tuple)): if len(trainable_trunk) != len(layer_sizes_trunk) - 1: - raise ValueError("trainable_trunk does not match layer_size_trunk.") + raise ValueError("trainable_trunk does not match layer_sizes_trunk.") self.layer_size_func = layer_sizes_branch self.layer_size_loc = layer_sizes_trunk @@ -490,11 +490,11 @@ class DeepONetCartesianProd(NN): """Deep operator network for dataset in the format of Cartesian product. Args: - layer_size_branch: A list of integers as the width of a fully connected network, + layer_sizes_branch: A list of integers as the width of a fully connected network, or `(dim, f)` where `dim` is the input dimension and `f` is a network function. The width of the last layer in the branch and trunk net should be the same for all strategies except "split_branch" and "split_trunk". - layer_size_trunk (list): A list of integers as the width of a fully connected + layer_sizes_trunk (list): A list of integers as the width of a fully connected network. activation: If `activation` is a ``string``, then the same activation is used in both trunk and branch nets. If `activation` is a ``dict``, then the trunk @@ -505,8 +505,8 @@ class DeepONetCartesianProd(NN): is a ``dict``, then the trunk net uses the rate `dropout_rate["trunk"]`, and the branch net uses `dropout_rate["branch"]`. Both `dropout_rate["trunk"]` and `dropout_rate["branch"]` should be ``float`` or lists of ``float``. - The list length should match the length of `layer_size_trunk` - 1 for the - trunk net and `layer_size_branch` - 2 for the branch net. + The list length should match the length of `layer_sizes_trunk` - 1 for the + trunk net and `layer_sizes_branch` - 2 for the branch net. num_outputs (integer): Number of outputs. In case of multiple outputs, i.e., `num_outputs` > 1, `multi_output_strategy` below should be set. multi_output_strategy (str or None): ``None``, "independent", "split_both", "split_branch" or @@ -537,8 +537,8 @@ class DeepONetCartesianProd(NN): def __init__( self, - layer_size_branch, - layer_size_trunk, + layer_sizes_branch, + layer_sizes_trunk, activation, kernel_initializer, regularization=None, @@ -547,8 +547,8 @@ def __init__( multi_output_strategy=None, ): super().__init__() - self.layer_size_func = layer_size_branch - self.layer_size_loc = layer_size_trunk + self.layer_size_func = layer_sizes_branch + self.layer_size_loc = layer_sizes_trunk if isinstance(activation, dict): self.activation_branch = activations.get(activation["branch"]) self.activation_trunk = activations.get(activation["trunk"]) @@ -562,24 +562,24 @@ def __init__( else: self.dropout_rate_branch = self.dropout_rate_trunk = dropout_rate if isinstance(self.dropout_rate_branch, list): - if not (len(layer_size_branch) - 2) == len(self.dropout_rate_branch): + if not (len(layer_sizes_branch) - 2) == len(self.dropout_rate_branch): raise ValueError( "Number of dropout rates of branch net must be " - f"equal to {len(layer_size_branch) - 2}" + f"equal to {len(layer_sizes_branch) - 2}" ) else: self.dropout_rate_branch = [self.dropout_rate_branch] * ( - len(layer_size_branch) - 2 + len(layer_sizes_branch) - 2 ) if isinstance(self.dropout_rate_trunk, list): - if not (len(layer_size_trunk) - 1) == len(self.dropout_rate_trunk): + if not (len(layer_sizes_trunk) - 1) == len(self.dropout_rate_trunk): raise ValueError( "Number of dropout rates of trunk net must be " - f"equal to {len(layer_size_trunk) - 1}" + f"equal to {len(layer_sizes_trunk) - 1}" ) else: self.dropout_rate_trunk = [self.dropout_rate_trunk] * ( - len(layer_size_trunk) - 1 + len(layer_sizes_trunk) - 1 ) self._inputs = None From 8275aeb6792709f51c9ebc3a7655cc0f20623c1c Mon Sep 17 00:00:00 2001 From: Pratik Rathore <76628577+pratikrathore8@users.noreply.github.com> Date: Mon, 25 Nov 2024 17:16:16 -0800 Subject: [PATCH 26/28] Add NNCG to optimizers submodule (#1661) --- deepxde/model.py | 18 +- deepxde/optimizers/__init__.py | 2 +- deepxde/optimizers/config.py | 58 ++++- deepxde/optimizers/pytorch/nncg.py | 317 +++++++++++++++++++++++ deepxde/optimizers/pytorch/optimizers.py | 22 +- docs/demos/pinn_forward/burgers.rst | 12 +- examples/pinn_forward/Burgers.py | 6 + 7 files changed, 428 insertions(+), 7 deletions(-) create mode 100644 deepxde/optimizers/pytorch/nncg.py diff --git a/deepxde/model.py b/deepxde/model.py index 568a16d10..458dfe5cf 100644 --- a/deepxde/model.py +++ b/deepxde/model.py @@ -367,11 +367,22 @@ def closure(): if self.lr_scheduler is not None: self.lr_scheduler.step() + def train_step_nncg(inputs, targets, auxiliary_vars): + def closure(): + losses = outputs_losses_train(inputs, targets, auxiliary_vars)[1] + total_loss = torch.sum(losses) + self.opt.zero_grad() + return total_loss + + self.opt.step(closure) + if self.lr_scheduler is not None: + self.lr_scheduler.step() + # Callables self.outputs = outputs self.outputs_losses_train = outputs_losses_train self.outputs_losses_test = outputs_losses_test - self.train_step = train_step + self.train_step = train_step if self.opt_name != "NNCG" else train_step_nncg def _compile_jax(self, lr, loss_fn, decay): """jax""" @@ -652,7 +663,10 @@ def train( elif backend_name == "tensorflow": self._train_tensorflow_tfp(verbose=verbose) elif backend_name == "pytorch": - self._train_pytorch_lbfgs(verbose=verbose) + if self.opt_name == "L-BFGS": + self._train_pytorch_lbfgs(verbose=verbose) + elif self.opt_name == "NNCG": + self._train_sgd(iterations, display_every, verbose=verbose) elif backend_name == "paddle": self._train_paddle_lbfgs(verbose=verbose) else: diff --git a/deepxde/optimizers/__init__.py b/deepxde/optimizers/__init__.py index e1fcfced1..556761a86 100644 --- a/deepxde/optimizers/__init__.py +++ b/deepxde/optimizers/__init__.py @@ -1,7 +1,7 @@ import importlib import sys -from .config import LBFGS_options, set_LBFGS_options +from .config import LBFGS_options, set_LBFGS_options, NNCG_options, set_NNCG_options from ..backend import backend_name diff --git a/deepxde/optimizers/config.py b/deepxde/optimizers/config.py index 01ba8bd1f..41bb19b86 100644 --- a/deepxde/optimizers/config.py +++ b/deepxde/optimizers/config.py @@ -1,9 +1,10 @@ -__all__ = ["set_LBFGS_options", "set_hvd_opt_options"] +__all__ = ["set_LBFGS_options", "set_NNCG_options", "set_hvd_opt_options"] from ..backend import backend_name from ..config import hvd LBFGS_options = {} +NNCG_options = {} if hvd is not None: hvd_opt_options = {} @@ -60,6 +61,60 @@ def set_LBFGS_options( LBFGS_options["maxls"] = maxls +def set_NNCG_options( + lr=1, + rank=50, + mu=1e-1, + updatefreq=20, + chunksz=1, + cgtol=1e-16, + cgmaxiter=1000, + lsfun="armijo", + verbose=False, +): + """Sets the hyperparameters of NysNewtonCG (NNCG). + + The NNCG optimizer only supports PyTorch. + + Args: + lr (float): + Learning rate (before line search). + rank (int): + Rank of preconditioner matrix used in preconditioned conjugate gradient. + mu (float): + Hessian damping parameter. + updatefreq (int): + How often the preconditioner matrix in preconditioned + conjugate gradient is updated. This parameter is not directly used in NNCG, + instead it is used in _train_pytorch_nncg in deepxde/model.py. + chunksz (int): + Number of Hessian-vector products to compute in parallel when constructing + preconditioner. If `chunk_size` is 1, the Hessian-vector products are + computed serially. + cgtol (float): + Convergence tolerance for the conjugate gradient method. The iteration stops + when `||r||_2 <= cgtol`, where `r` is the residual. Note that this condition + is based on the absolute tolerance, not the relative tolerance. + cgmaxiter (int): + Maximum number of iterations for the conjugate gradient method. + lsfun (str): + The line search function used to find the step size. The default value is + "armijo". The other option is None. + verbose (bool): + If `True`, prints the eigenvalues of the Nyström approximation + of the Hessian. + """ + NNCG_options["lr"] = lr + NNCG_options["rank"] = rank + NNCG_options["mu"] = mu + NNCG_options["updatefreq"] = updatefreq + NNCG_options["chunksz"] = chunksz + NNCG_options["cgtol"] = cgtol + NNCG_options["cgmaxiter"] = cgmaxiter + NNCG_options["lsfun"] = lsfun + NNCG_options["verbose"] = verbose + + def set_hvd_opt_options( compression=None, op=None, @@ -91,6 +146,7 @@ def set_hvd_opt_options( set_LBFGS_options() +set_NNCG_options() if hvd is not None: set_hvd_opt_options() diff --git a/deepxde/optimizers/pytorch/nncg.py b/deepxde/optimizers/pytorch/nncg.py new file mode 100644 index 000000000..93704ec9d --- /dev/null +++ b/deepxde/optimizers/pytorch/nncg.py @@ -0,0 +1,317 @@ +from functools import reduce + +import torch +from torch.func import vmap +from torch.optim import Optimizer + + +def _armijo(f, x, gx, dx, t, alpha=0.1, beta=0.5): + """Line search to find a step size that satisfies the Armijo condition.""" + f0 = f(x, 0, dx) + f1 = f(x, t, dx) + while f1 > f0 + alpha * t * gx.dot(dx): + t *= beta + f1 = f(x, t, dx) + return t + + +def _apply_nys_precond_inv(U, S_mu_inv, mu, lambd_r, x): + """Applies the inverse of the Nystrom approximation of the Hessian to a vector.""" + z = U.T @ x + z = (lambd_r + mu) * (U @ (S_mu_inv * z)) + (x - U @ z) + return z + + +def _nystrom_pcg(hess, b, x, mu, U, S, r, tol, max_iters): + """Solves a positive-definite linear system using NyströmPCG. + + `Frangella et al. Randomized Nyström Preconditioning. + SIAM Journal on Matrix Analysis and Applications, 2023. + ` + """ + lambd_r = S[r - 1] + S_mu_inv = (S + mu) ** (-1) + + resid = b - (hess(x) + mu * x) + with torch.no_grad(): + z = _apply_nys_precond_inv(U, S_mu_inv, mu, lambd_r, resid) + p = z.clone() + + i = 0 + + while torch.norm(resid) > tol and i < max_iters: + v = hess(p) + mu * p + with torch.no_grad(): + alpha = torch.dot(resid, z) / torch.dot(p, v) + x += alpha * p + + rTz = torch.dot(resid, z) + resid -= alpha * v + z = _apply_nys_precond_inv(U, S_mu_inv, mu, lambd_r, resid) + beta = torch.dot(resid, z) / rTz + + p = z + beta * p + + i += 1 + + if torch.norm(resid) > tol: + print( + "Warning: PCG did not converge to tolerance. " + f"Tolerance was {tol} but norm of residual is {torch.norm(resid)}" + ) + + return x + + +class NNCG(Optimizer): + """Implementation of NysNewtonCG, a damped Newton-CG method + that uses Nyström preconditioning. + + `Rathore et al. Challenges in Training PINNs: A Loss Landscape Perspective. + Preprint, 2024. ` + + .. warning:: + This optimizer doesn't support per-parameter options and parameter + groups (there can be only one). + + NOTE: This optimizer is currently a beta version. + + Our implementation is inspired by the PyTorch implementation of `L-BFGS + `. + + The parameters rank and mu will probably need to be tuned for your specific problem. + If the optimizer is running very slowly, you can try one of the following: + - Increase the rank (this should increase the + accuracy of the Nyström approximation in PCG) + - Reduce cg_tol (this will allow PCG to terminate with a less accurate solution) + - Reduce cg_max_iters (this will allow PCG to terminate after fewer iterations) + + Args: + params (iterable): iterable of parameters to optimize or dicts defining + parameter groups + lr (float, optional): learning rate (default: 1.0) + rank (int, optional): rank of the Nyström approximation (default: 10) + mu (float, optional): damping parameter (default: 1e-4) + update_freq (int, optional): frequency of updating the preconditioner + chunk_size (int, optional): number of Hessian-vector products + to be computed in parallel (default: 1) + cg_tol (float, optional): tolerance for PCG (default: 1e-16) + cg_max_iters (int, optional): maximum number of PCG iterations (default: 1000) + line_search_fn (str, optional): either 'armijo' or None (default: None) + verbose (bool, optional): verbosity (default: False) + """ + + def __init__( + self, + params, + lr=1.0, + rank=10, + mu=1e-4, + update_freq=20, + chunk_size=1, + cg_tol=1e-16, + cg_max_iters=1000, + line_search_fn=None, + verbose=False, + ): + defaults = { + "lr": lr, + "rank": rank, + "mu": mu, + "update_freq": update_freq, + "chunk_size": chunk_size, + "cg_tol": cg_tol, + "cg_max_iters": cg_max_iters, + "line_search_fn": line_search_fn, + } + self.rank = rank + self.mu = mu + self.update_freq = update_freq + self.chunk_size = chunk_size + self.cg_tol = cg_tol + self.cg_max_iters = cg_max_iters + self.line_search_fn = line_search_fn + self.verbose = verbose + self.U = None + self.S = None + self.n_iters = 0 + super().__init__(params, defaults) + + if len(self.param_groups) > 1: + raise ValueError( + "NNCG doesn't currently support " + "per-parameter options (parameter groups)" + ) + + if self.line_search_fn is not None and self.line_search_fn != "armijo": + raise ValueError("NNCG only supports Armijo line search") + + self._params = self.param_groups[0]["params"] + self._params_list = list(self._params) + self._numel_cache = None + + def step(self, closure): + """Perform a single optimization step. + + Args: + closure (callable): A closure that reevaluates the model + and returns the loss w.r.t. the parameters. + """ + if self.n_iters == 0: + # Store the previous direction for warm starting PCG + self.old_dir = torch.zeros(self._numel(), device=self._params[0].device) + + loss = closure() + # Compute gradient via torch.autograd.grad + g_tuple = torch.autograd.grad(loss, self._params_list, create_graph=True) + g = torch.cat([gi.view(-1) for gi in g_tuple if gi is not None]) + + if self.n_iters % self.update_freq == 0: + self._update_preconditioner(g) + + # One step update + for group_idx, group in enumerate(self.param_groups): + + def hvp_temp(x): + return self._hvp(g, self._params_list, x) + + # Calculate the Newton direction + d = _nystrom_pcg( + hvp_temp, + g, + self.old_dir, + self.mu, + self.U, + self.S, + self.rank, + self.cg_tol, + self.cg_max_iters, + ) + + # Store the previous direction for warm starting PCG + self.old_dir = d + + # Check if d is a descent direction + if torch.dot(d, g) <= 0: + print("Warning: d is not a descent direction") + + if self.line_search_fn == "armijo": + x_init = self._clone_param() + + def obj_func(x, t, dx): + self._add_grad(t, dx) + loss = float(closure()) + self._set_param(x) + return loss + + # Use -d for convention + t = _armijo(obj_func, x_init, g, -d, group["lr"]) + else: + t = group["lr"] + + self.state[group_idx]["t"] = t + + # update parameters + ls = 0 + for p in group["params"]: + np = torch.numel(p) + dp = d[ls : ls + np].view(p.shape) + ls += np + p.data.add_(-dp, alpha=t) + + self.n_iters += 1 + + return loss + + def _update_preconditioner(self, grad): + """Update the Nyström approximation of the Hessian. + + Args: + grad (torch.Tensor): gradient of the loss w.r.t. the parameters. + """ + # Generate test matrix (NOTE: This is transposed test matrix) + p = grad.shape[0] + Phi = torch.randn((self.rank, p), device=grad.device) / (p**0.5) + Phi = torch.linalg.qr(Phi.t(), mode="reduced")[0].t() + + Y = self._hvp_vmap(grad, self._params_list)(Phi) + + # Calculate shift + shift = torch.finfo(Y.dtype).eps + Y_shifted = Y + shift * Phi + + # Calculate Phi^T * H * Phi (w/ shift) for Cholesky + choleskytarget = torch.mm(Y_shifted, Phi.t()) + + # Perform Cholesky, if fails, do eigendecomposition + # The new shift is the abs of smallest eigenvalue (negative) + # plus the original shift + try: + C = torch.linalg.cholesky(choleskytarget) + except torch.linalg.LinAlgError: + # eigendecomposition, eigenvalues and eigenvector matrix + eigs, eigvectors = torch.linalg.eigh(choleskytarget) + shift = shift + torch.abs(torch.min(eigs)) + # add shift to eigenvalues + eigs = eigs + shift + # put back the matrix for Cholesky by eigenvector * eigenvalues + # after shift * eigenvector^T + C = torch.linalg.cholesky( + torch.mm(eigvectors, torch.mm(torch.diag(eigs), eigvectors.T)) + ) + + try: + B = torch.linalg.solve_triangular(C, Y_shifted, upper=False, left=True) + # temporary fix for issue @ https://github.com/pytorch/pytorch/issues/97211 + except RuntimeError: + B = torch.linalg.solve_triangular( + C.to("cpu"), Y_shifted.to("cpu"), upper=False, left=True + ).to(C.device) + + # B = V * S * U^T b/c we have been using transposed sketch + _, S, UT = torch.linalg.svd(B, full_matrices=False) + self.U = UT.t() + self.S = torch.max(torch.square(S) - shift, torch.tensor(0.0)) + + self.rho = self.S[-1] + + if self.verbose: + print(f"Approximate eigenvalues = {self.S}") + + def _hvp_vmap(self, grad_params, params): + return vmap( + lambda v: self._hvp(grad_params, params, v), + in_dims=0, + chunk_size=self.chunk_size, + ) + + def _hvp(self, grad_params, params, v): + Hv = torch.autograd.grad(grad_params, params, grad_outputs=v, retain_graph=True) + Hv = tuple(Hvi.detach() for Hvi in Hv) + return torch.cat([Hvi.reshape(-1) for Hvi in Hv]) + + def _numel(self): + if self._numel_cache is None: + self._numel_cache = reduce( + lambda total, p: total + p.numel(), self._params, 0 + ) + return self._numel_cache + + def _add_grad(self, step_size, update): + offset = 0 + for p in self._params: + numel = p.numel() + # Avoid in-place operation by creating a new tensor + p.data = p.data.add( + update[offset : offset + numel].view_as(p), alpha=step_size + ) + offset += numel + assert offset == self._numel() + + def _clone_param(self): + return [p.clone(memory_format=torch.contiguous_format) for p in self._params] + + def _set_param(self, params_data): + for p, pdata in zip(self._params, params_data): + # Replace the .data attribute of the tensor + p.data = pdata.data diff --git a/deepxde/optimizers/pytorch/optimizers.py b/deepxde/optimizers/pytorch/optimizers.py index 6329912dd..35ab88d24 100644 --- a/deepxde/optimizers/pytorch/optimizers.py +++ b/deepxde/optimizers/pytorch/optimizers.py @@ -2,11 +2,12 @@ import torch -from ..config import LBFGS_options +from .nncg import NNCG +from ..config import LBFGS_options, NNCG_options def is_external_optimizer(optimizer): - return optimizer in ["L-BFGS", "L-BFGS-B"] + return optimizer in ["L-BFGS", "L-BFGS-B", "NNCG"] def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0): @@ -29,6 +30,23 @@ def get(params, optimizer, learning_rate=None, decay=None, weight_decay=0): history_size=LBFGS_options["maxcor"], line_search_fn=("strong_wolfe" if LBFGS_options["maxls"] > 0 else None), ) + elif optimizer == "NNCG": + if weight_decay > 0: + raise ValueError("NNCG optimizer doesn't support weight_decay > 0") + if learning_rate is not None or decay is not None: + print("Warning: learning rate is ignored for {}".format(optimizer)) + optim = NNCG( + params, + lr=NNCG_options["lr"], + rank=NNCG_options["rank"], + mu=NNCG_options["mu"], + update_freq=NNCG_options["updatefreq"], + chunk_size=NNCG_options["chunksz"], + cg_tol=NNCG_options["cgtol"], + cg_max_iters=NNCG_options["cgmaxiter"], + line_search_fn=NNCG_options["lsfun"], + verbose=NNCG_options["verbose"], + ) else: if learning_rate is None: raise ValueError("No learning rate for {}.".format(optimizer)) diff --git a/docs/demos/pinn_forward/burgers.rst b/docs/demos/pinn_forward/burgers.rst index 76ded3e31..08e56b38a 100644 --- a/docs/demos/pinn_forward/burgers.rst +++ b/docs/demos/pinn_forward/burgers.rst @@ -87,7 +87,17 @@ After we train the network using Adam, we continue to train the network using L- .. code-block:: python model.compile("L-BFGS-B") - losshistory, train_state = model.train() + losshistory, train_state = model.train() + +However, L-BFGS can stall out early in optimization if it is unable to find a step size satisfying the strong Wolfe conditions. In such cases, we can use the NNCG optimizer (compatible with PyTorch only) to continue reducing the loss: + +.. code-block:: python + + dde.optimizers.set_NNCG_options(rank=50, mu=1e-1) + model.compile("NNCG") + losshistory, train_state = model.train(iterations=1000, display_every=100) + +By default, NNCG does not run in this demo. You will have to uncomment the NNCG code block in the demo to have it run after Adam and L-BFGS. Note that it can take some hyperparameter tuning to get the best performance from the NNCG optimizer. Complete code -------------- diff --git a/examples/pinn_forward/Burgers.py b/examples/pinn_forward/Burgers.py index 64ee46bb6..7b6883ed1 100644 --- a/examples/pinn_forward/Burgers.py +++ b/examples/pinn_forward/Burgers.py @@ -1,4 +1,5 @@ """Backend supported: tensorflow.compat.v1, tensorflow, pytorch, paddle""" + import deepxde as dde import numpy as np @@ -38,6 +39,11 @@ def pde(x, y): model.train(iterations=15000) model.compile("L-BFGS") losshistory, train_state = model.train() +# """Backend supported: pytorch""" +# # Run NNCG after Adam and L-BFGS +# dde.optimizers.set_NNCG_options(rank=50, mu=1e-1) +# model.compile("NNCG") +# losshistory, train_state = model.train(iterations=1000, display_every=100) dde.saveplot(losshistory, train_state, issave=True, isplot=True) X, y_true = gen_testdata() From 3544fdfd4e43ffb920d2b191a53d66946226199e Mon Sep 17 00:00:00 2001 From: vl-dud <60846135+vl-dud@users.noreply.github.com> Date: Mon, 2 Dec 2024 01:34:32 +0000 Subject: [PATCH 27/28] Backend PyTorch: Fix L2 regularizers for external_trainable_variables (#1884) --- deepxde/model.py | 59 ++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 24 deletions(-) diff --git a/deepxde/model.py b/deepxde/model.py index 458dfe5cf..1644c2529 100644 --- a/deepxde/model.py +++ b/deepxde/model.py @@ -112,9 +112,10 @@ def compile( weighted by the `loss_weights` coefficients. external_trainable_variables: A trainable ``dde.Variable`` object or a list of trainable ``dde.Variable`` objects. The unknown parameters in the - physics systems that need to be recovered. If the backend is - tensorflow.compat.v1, `external_trainable_variables` is ignored, and all - trainable ``dde.Variable`` objects are automatically collected. + physics systems that need to be recovered. Regularization will not be + applied to these variables. If the backend is tensorflow.compat.v1, + `external_trainable_variables` is ignored, and all trainable ``dde.Variable`` + objects are automatically collected. verbose (Integer): Controls the verbosity of the compile process. """ if verbose > 0 and config.rank == 0: @@ -330,30 +331,40 @@ def outputs_losses_test(inputs, targets, auxiliary_vars): False, inputs, targets, auxiliary_vars, self.data.losses_test ) - # Another way is using per-parameter options - # https://pytorch.org/docs/stable/optim.html#per-parameter-options, - # but not all optimizers (such as L-BFGS) support this. - trainable_variables = ( - list(self.net.parameters()) + self.external_trainable_variables - ) - if self.net.regularizer is None: - self.opt, self.lr_scheduler = optimizers.get( - trainable_variables, self.opt_name, learning_rate=lr, decay=decay - ) - else: - if self.net.regularizer[0] == "l2": - self.opt, self.lr_scheduler = optimizers.get( - trainable_variables, - self.opt_name, - learning_rate=lr, - decay=decay, - weight_decay=self.net.regularizer[1], - ) - else: + weight_decay = 0 + if self.net.regularizer is not None: + if self.net.regularizer[0] != "l2": raise NotImplementedError( f"{self.net.regularizer[0]} regularization to be implemented for " - "backend pytorch." + "backend pytorch" ) + weight_decay = self.net.regularizer[1] + + optimizer_params = self.net.parameters() + if self.external_trainable_variables: + # L-BFGS doesn't support per-parameter options. + if self.opt_name in ["L-BFGS", "L-BFGS-B"]: + optimizer_params = ( + list(optimizer_params) + self.external_trainable_variables + ) + if weight_decay > 0: + print( + "Warning: L2 regularization will also be applied to external_trainable_variables. " + "Ensure this is intended behavior." + ) + else: + optimizer_params = [ + {"params": optimizer_params}, + {"params": self.external_trainable_variables, "weight_decay": 0}, + ] + + self.opt, self.lr_scheduler = optimizers.get( + optimizer_params, + self.opt_name, + learning_rate=lr, + decay=decay, + weight_decay=weight_decay, + ) def train_step(inputs, targets, auxiliary_vars): def closure(): From ec4bdd39c3d4af853f94438506a368173e1477c9 Mon Sep 17 00:00:00 2001 From: lijialin03 <124568209+lijialin03@users.noreply.github.com> Date: Wed, 11 Dec 2024 03:19:17 +0800 Subject: [PATCH 28/28] Backend paddle: Refactor and add regularizer (#1894) --- deepxde/backend/backend.py | 39 +++++++++++++++++++ deepxde/backend/paddle/tensor.py | 8 ++++ deepxde/backend/tensorflow/tensor.py | 12 ++++++ .../backend/tensorflow_compat_v1/tensor.py | 12 ++++++ deepxde/nn/regularizers.py | 12 +++--- 5 files changed, 77 insertions(+), 6 deletions(-) diff --git a/deepxde/backend/backend.py b/deepxde/backend/backend.py index b744f9174..96c5bb2ca 100644 --- a/deepxde/backend/backend.py +++ b/deepxde/backend/backend.py @@ -502,3 +502,42 @@ def sparse_dense_matmul(x, y): Returns: Tensor: The multiplication result. """ + + +############################################################################### +# Regularization + + +def l1_regularization(l1): + """A regularizer that applies a L1 regularization penalty or L1 weight decay. + + Warning: + The implementation may vary across different backends. + + Args: + l1 (float): L1 regularization factor. + """ + + +def l2_regularization(l2): + """A regularizer that applies a L2 regularization penalty or L2 weight decay. + + Warning: + The implementation may vary across different backends. + + Args: + l2 (float): L2 regularization factor. + """ + + +def l1_l2_regularization(l1, l2): + """A regularizer that applies both L1 and L2 regularization penalties or + L1 and L2 weight decay. + + Warning: + The implementation may vary across different backends. + + Args: + l1 (float): L1 regularization factor. + l2 (float): L2 regularization factor. + """ diff --git a/deepxde/backend/paddle/tensor.py b/deepxde/backend/paddle/tensor.py index be1f58daf..713ac9f58 100644 --- a/deepxde/backend/paddle/tensor.py +++ b/deepxde/backend/paddle/tensor.py @@ -229,3 +229,11 @@ def matmul(x, y): def sparse_dense_matmul(x, y): return paddle.sparse.matmul(x, y) + + +def l1_regularization(l1): + return paddle.regularizer.L1Decay(coeff=l1) + + +def l2_regularization(l2): + return paddle.regularizer.L2Decay(coeff=l2) diff --git a/deepxde/backend/tensorflow/tensor.py b/deepxde/backend/tensorflow/tensor.py index a201ad6c7..bde0dd6be 100644 --- a/deepxde/backend/tensorflow/tensor.py +++ b/deepxde/backend/tensorflow/tensor.py @@ -210,3 +210,15 @@ def zeros_like(input_tensor): def matmul(x, y): return tf.linalg.matmul(x, y) + + +def l1_regularization(l1): + return tf.keras.regularizers.L1(l1=l1) + + +def l2_regularization(l2): + return tf.keras.regularizers.L2(l2=l2) + + +def l1_l2_regularization(l1, l2): + return tf.keras.regularizers.L1L2(l1=l1, l2=l2) diff --git a/deepxde/backend/tensorflow_compat_v1/tensor.py b/deepxde/backend/tensorflow_compat_v1/tensor.py index c0c4091ae..52270260b 100644 --- a/deepxde/backend/tensorflow_compat_v1/tensor.py +++ b/deepxde/backend/tensorflow_compat_v1/tensor.py @@ -245,3 +245,15 @@ def matmul(x, y): def sparse_dense_matmul(x, y): return tf.sparse.sparse_dense_matmul(x, y) + + +def l1_regularization(l1): + return tf.keras.regularizers.L1(l1=l1) + + +def l2_regularization(l2): + return tf.keras.regularizers.L2(l2=l2) + + +def l1_l2_regularization(l1, l2): + return tf.keras.regularizers.L1L2(l1=l1, l2=l2) diff --git a/deepxde/nn/regularizers.py b/deepxde/nn/regularizers.py index f83a871cf..b2ce664db 100644 --- a/deepxde/nn/regularizers.py +++ b/deepxde/nn/regularizers.py @@ -1,8 +1,8 @@ -from ..backend import tf +from .. import backend as bkd def get(identifier): - """Retrieves a TensorFlow regularizer instance based on the given identifier. + """Retrieves a regularizer instance based on the given identifier. Args: identifier (list/tuple): Specifies the type and factor(s) of the regularizer. @@ -11,7 +11,6 @@ def get(identifier): For "l1l2", provide both "l1" and "l2" factors. """ - # TODO: other backends if identifier is None or not identifier: return None if not isinstance(identifier, (list, tuple)): @@ -23,11 +22,12 @@ def get(identifier): raise ValueError("Regularization factor must be provided.") if name == "l1": - return tf.keras.regularizers.L1(l1=factor[0]) + return bkd.l1_regularization(factor[0]) if name == "l2": - return tf.keras.regularizers.L2(l2=factor[0]) + return bkd.l2_regularization(factor[0]) if name in ("l1l2", "l1+l2"): + # TODO: only supported by 'tensorflow.compat.v1' and 'tensorflow' now. if len(factor) < 2: raise ValueError("L1L2 regularizer requires both L1/L2 penalties.") - return tf.keras.regularizers.L1L2(l1=factor[0], l2=factor[1]) + return bkd.l1_l2_regularization(factor[0], factor[1]) raise ValueError(f"Unknown regularizer name: {name}")