Merge branch '2-migration-tf2' into 'develop'

Migration to TensorFlow2

See merge request remi.cresson/otbtf!3

.clang-format

@@ -0,0 +1,134 @@
+BasedOnStyle: Mozilla
+Language:        Cpp
+AccessModifierOffset: -2
+AlignAfterOpenBracket: Align
+AlignConsecutiveAssignments: false
+AlignConsecutiveDeclarations: true
+AlignEscapedNewlines: Left
+AlignOperands:   true
+AlignTrailingComments: true
+# clang 9.0 AllowAllArgumentsOnNextLine: true
+# clang 9.0 AllowAllConstructorInitializersOnNextLine: true
+AllowAllParametersOfDeclarationOnNextLine: false
+AllowShortBlocksOnASingleLine: false
+AllowShortCaseLabelsOnASingleLine: false
+AllowShortFunctionsOnASingleLine: Inline
+# clang 9.0 AllowShortLambdasOnASingleLine: All
+# clang 9.0 features AllowShortIfStatementsOnASingleLine: Never
+AllowShortIfStatementsOnASingleLine: false
+AllowShortLoopsOnASingleLine: false
+AlwaysBreakAfterDefinitionReturnType: None
+AlwaysBreakAfterReturnType: All
+AlwaysBreakBeforeMultilineStrings: false
+AlwaysBreakTemplateDeclarations: Yes
+BinPackArguments: false
+BinPackParameters: false
+BreakBeforeBraces: Custom
+BraceWrapping:
+  # clang 9.0 feature AfterCaseLabel:  false
+  AfterClass:      true
+  AfterControlStatement: true
+  AfterEnum:       true
+  AfterFunction:   true
+  AfterNamespace:  true
+  AfterObjCDeclaration: true
+  AfterStruct:     true
+  AfterUnion:      true
+  AfterExternBlock: true
+  BeforeCatch:     true
+  BeforeElse:      true
+## This is the big change from historical ITK formatting!
+# Historically ITK used a style similar to https://en.wikipedia.org/wiki/Indentation_style#Whitesmiths_style
+# with indented braces, and not indented code.  This style is very difficult to automatically
+# maintain with code beautification tools.  Not indenting braces is more common among
+# formatting tools.
+  IndentBraces:    false
+  SplitEmptyFunction: false
+  SplitEmptyRecord: false
+  SplitEmptyNamespace: false
+BreakBeforeBinaryOperators: None
+#clang 6.0 BreakBeforeInheritanceComma: true
+BreakInheritanceList: BeforeComma
+BreakBeforeTernaryOperators: true
+#clang 6.0 BreakConstructorInitializersBeforeComma: true
+BreakConstructorInitializers: BeforeComma
+BreakAfterJavaFieldAnnotations: false
+BreakStringLiterals: true
+## The following line allows larger lines in non-documentation code
+ColumnLimit:     120
+CommentPragmas:  '^ IWYU pragma:'
+CompactNamespaces: false
+ConstructorInitializerAllOnOneLineOrOnePerLine: false
+ConstructorInitializerIndentWidth: 2
+ContinuationIndentWidth: 2
+Cpp11BracedListStyle: false
+DerivePointerAlignment: false
+DisableFormat:   false
+ExperimentalAutoDetectBinPacking: false
+FixNamespaceComments: true
+ForEachMacros:
+  - foreach
+  - Q_FOREACH
+  - BOOST_FOREACH
+IncludeBlocks:   Preserve
+IncludeCategories:
+  - Regex:           '^"(llvm|llvm-c|clang|clang-c)/'
+    Priority:        2
+  - Regex:           '^(<|"(gtest|gmock|isl|json)/)'
+    Priority:        3
+  - Regex:           '.*'
+    Priority:        1
+IncludeIsMainRegex: '(Test)?$'
+IndentCaseLabels: true
+IndentPPDirectives: AfterHash
+IndentWidth:     2
+IndentWrappedFunctionNames: false
+JavaScriptQuotes: Leave
+JavaScriptWrapImports: true
+KeepEmptyLinesAtTheStartOfBlocks: true
+MacroBlockBegin: ''
+MacroBlockEnd:   ''
+MaxEmptyLinesToKeep: 2
+NamespaceIndentation: None
+ObjCBinPackProtocolList: Auto
+ObjCBlockIndentWidth: 2
+ObjCSpaceAfterProperty: true
+ObjCSpaceBeforeProtocolList: false
+PenaltyBreakAssignment: 2
+PenaltyBreakBeforeFirstCallParameter: 19
+PenaltyBreakComment: 300
+## The following line allows larger lines in non-documentation code
+PenaltyBreakFirstLessLess: 120
+PenaltyBreakString: 1000
+PenaltyBreakTemplateDeclaration: 10
+PenaltyExcessCharacter: 1000000
+PenaltyReturnTypeOnItsOwnLine: 200
+PointerAlignment: Middle
+ReflowComments:  true
+# We may want to sort the includes as a separate pass
+SortIncludes:    false
+# We may want to revisit this later
+SortUsingDeclarations: false
+SpaceAfterCStyleCast: false
+# SpaceAfterLogicalNot: false
+SpaceAfterTemplateKeyword: true
+SpaceBeforeAssignmentOperators: true
+SpaceBeforeCpp11BracedList: false
+SpaceBeforeCtorInitializerColon: true
+SpaceBeforeInheritanceColon: true
+SpaceBeforeParens: ControlStatements
+SpaceBeforeRangeBasedForLoopColon: true
+SpaceInEmptyParentheses: false
+SpacesBeforeTrailingComments: 1
+SpacesInAngles:  false
+SpacesInContainerLiterals: false
+SpacesInCStyleCastParentheses: false
+SpacesInParentheses: false
+SpacesInSquareBrackets: false
+Standard:        Cpp11
+StatementMacros:
+  - Q_UNUSED
+  - QT_REQUIRE_VERSION
+TabWidth:        2
+UseTab:          Never
+...

.gitattributes

@@ -0,0 +1,8 @@
+*.png filter=lfs diff=lfs merge=lfs -text
+*.jp2 filter=lfs diff=lfs merge=lfs -text
+*.tif filter=lfs diff=lfs merge=lfs -text
+*.data-* filter=lfs diff=lfs merge=lfs -text
+*.gpkg filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.jpg filter=lfs diff=lfs merge=lfs -text
+*.jpeg filter=lfs diff=lfs merge=lfs -text

.gitlab-ci.yml

@@ -0,0 +1,119 @@
+image: gitlab-registry.irstea.fr/remi.cresson/otbtf:2.4-cpu-basic-testing
+
+variables:
+    OTB_BUILD: /src/otb/build/OTB/build  # Local OTB build directory
+    OTBTF_SRC: /src/otbtf  # Local OTBTF source directory
+    OTB_TEST_DIR: $OTB_BUILD/Testing/Temporary  # OTB testing directory
+    ARTIFACT_TEST_DIR: $CI_PROJECT_DIR/testing
+    CRC_BOOK_TMP: /tmp/crc_book_tests_tmp
+
+workflow:
+  rules:
+    - if: $CI_MERGE_REQUEST_ID             # Execute jobs in merge request context
+    - if: $CI_COMMIT_BRANCH == 'develop'   # Execute jobs when a new commit is pushed to develop branch
+
+stages:
+  - Build
+  - Static Analysis
+  - Test
+  - Applications Test
+
+.update_otbtf_src: &update_otbtf_src
+  - sudo rm -rf $OTBTF_SRC && sudo ln -s $PWD $OTBTF_SRC  # Replace local OTBTF source directory
+
+.compile_otbtf: &compile_otbtf
+  - cd $OTB_BUILD && sudo make install -j$(nproc --all)  # Rebuild OTB with new OTBTF sources
+
+.install_pytest: &install_pytest
+  - pip3 install pytest pytest-cov pytest-order  # Install pytest stuff
+
+before_script:
+  - *update_otbtf_src
+
+build:
+  stage: Build
+  allow_failure: false
+  script:
+    - *compile_otbtf
+
+flake8:
+  stage: Static Analysis
+  allow_failure: true
+  script:
+    - sudo apt update && sudo apt install flake8 -y
+    - python -m flake8 --max-line-length=120 $OTBTF_SRC/python
+
+pylint:
+  stage: Static Analysis
+  allow_failure: true
+  script:
+    - sudo apt update && sudo apt install pylint -y
+    - pylint --disable=too-many-nested-blocks,too-many-locals,too-many-statements,too-few-public-methods,too-many-instance-attributes,too-many-arguments --ignored-modules=tensorflow --max-line-length=120 --logging-format-style=new $OTBTF_SRC/python
+
+codespell:
+  stage: Static Analysis
+  allow_failure: true
+  script:
+    - sudo pip install codespell && codespell
+
+cppcheck:
+  stage: Static Analysis
+  allow_failure: true
+  script:
+    - sudo apt update && sudo apt install cppcheck -y
+    - cd $OTBTF_SRC/ && cppcheck --enable=all --error-exitcode=1 -I include/ --suppress=missingInclude --suppress=unusedFunction .
+
+ctest:
+  stage: Test
+  script:
+    - *compile_otbtf
+    - sudo rm -rf $OTB_TEST_DIR/*  # Empty testing temporary folder (old files here)
+    - cd $OTB_BUILD/ && sudo ctest -L OTBTensorflow  # Run ctest
+  after_script:
+    - cp -r $OTB_TEST_DIR $ARTIFACT_TEST_DIR
+  artifacts:
+    paths:
+      - $ARTIFACT_TEST_DIR/*.*
+    expire_in: 1 week
+    when: on_failure
+
+.applications_test_base:
+  stage: Applications Test
+  rules:
+      # Only for MR targeting 'develop' branch because applications tests are slow
+    - if: $CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TARGET_BRANCH_NAME == 'develop'
+  artifacts:
+    when: on_failure
+    paths:
+      - $CI_PROJECT_DIR/report_*.xml
+      - $ARTIFACT_TEST_DIR/*.*
+    expire_in: 1 week
+
+crc_book:
+  extends: .applications_test_base
+  script:
+    - *compile_otbtf
+    - *install_pytest
+    - cd $CI_PROJECT_DIR
+    - mkdir -p $CRC_BOOK_TMP
+    - TMPDIR=$CRC_BOOK_TMP DATADIR=$CI_PROJECT_DIR/test/data python -m pytest --junitxml=$CI_PROJECT_DIR/report_tutorial.xml $OTBTF_SRC/test/tutorial_unittest.py
+  after_script:
+    - mkdir -p $ARTIFACT_TEST_DIR
+    - cp $CRC_BOOK_TMP/*.* $ARTIFACT_TEST_DIR/
+
+sr4rs:
+  extends: .applications_test_base
+  script:
+    - *compile_otbtf
+    - *install_pytest
+    - cd $CI_PROJECT_DIR
+    - wget -O sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
+      https://nextcloud.inrae.fr/s/boabW9yCjdpLPGX/download/sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
+    - unzip -o sr4rs_sentinel2_bands4328_france2020_savedmodel.zip
+    - wget -O sr4rs_data.zip https://nextcloud.inrae.fr/s/qMLLyKCDieqmgWz/download
+    - unzip -o sr4rs_data.zip
+    - rm -rf sr4rs
+    - git clone https://github.com/remicres/sr4rs.git
+    - export PYTHONPATH=$PYTHONPATH:$PWD/sr4rs
+    - python -m pytest --junitxml=$CI_PROJECT_DIR/report_sr4rs.xml $OTBTF_SRC/test/sr4rs_unittest.py
+

CMakeLists.txt

@@ -15,7 +15,7 @@ if(OTB_USE_TENSORFLOW)
   find_library(TENSORFLOW_FRAMEWORK_LIB NAMES libtensorflow_framework)
 
   set(TENSORFLOW_LIBS "${TENSORFLOW_CC_LIB}" "${TENSORFLOW_FRAMEWORK_LIB}")
-
+  set(OTBTensorflow_THIRD_PARTY "this is a hack to skip header_tests")
 else()
   message("Tensorflow support disabled")
 endif()

CONTRIBUTORS.md

@@ -0,0 +1,8 @@
+- Remi Cresson
+- Nicolas Narcon
+- Benjamin Commandre
+- Vincent Delbar
+- Loic Lozac'h
+- Pratyush Das
+- Doctor Who
+- Jordi Inglada

Dockerfile

@@ -47,7 +47,7 @@ RUN wget -qO /opt/otbtf/bin/bazelisk https://github.com/bazelbuild/bazelisk/rele
  && ln -s /opt/otbtf/bin/bazelisk /opt/otbtf/bin/bazel
 
 ARG BZL_TARGETS="//tensorflow:libtensorflow_cc.so //tensorflow/tools/pip_package:build_pip_package"
-# "--config=opt" will enable 'march=native' (otherwise read comments about CPU compatibilty and edit CC_OPT_FLAGS in build-env-tf.sh)
+# "--config=opt" will enable 'march=native' (otherwise read comments about CPU compatibility and edit CC_OPT_FLAGS in build-env-tf.sh)
 ARG BZL_CONFIGS="--config=nogcp --config=noaws --config=nohdfs --config=opt"
 # "--compilation_mode opt" is already enabled by default (see tf repo .bazelrc and configure.py)
 ARG BZL_OPTIONS="--verbose_failures --remote_cache=http://localhost:9090"
@@ -77,7 +77,7 @@ RUN git clone --single-branch -b $TF https://github.com/tensorflow/tensorflow.gi
  # Symlink external libs (required for MKL - libiomp5)
  && for f in $(find -L /opt/otbtf/include/tf -wholename "*/external/*/*.so"); do ln -s $f /opt/otbtf/lib/; done \
  # Compress and save TF binaries
- && ( ! $ZIP_TF_BIN || zip -9 -j --symlinks /opt/otbtf/tf-$TF.zip tensorflow/cc/saved_model/tag_constants.h bazel-bin/tensorflow/libtensorflow_cc.so* /tmp/tensorflow_pkg/tensorflow*.whl ) \
+ && ( ! $ZIP_TF_BIN || zip -9 -j --symlinks /opt/otbtf/tf-$TF.zip tensorflow/cc/saved_model/tag_constants.h tensorflow/cc/saved_model/signature_constants.h bazel-bin/tensorflow/libtensorflow_cc.so* /tmp/tensorflow_pkg/tensorflow*.whl ) \
  # Cleaning
  && rm -rf bazel-* /src/tf /root/.cache/ /tmp/*
 

LICENSE

@@ -188,7 +188,7 @@
       identification within third-party archives.
 
    Copyright 2018-2019 Rémi Cresson (IRSTEA)
-   Copyright 2020 Rémi Cresson (INRAE)
+   Copyright 2020-2021 Rémi Cresson (INRAE)
 
    Licensed under the Apache License, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.

README.md

@@ -18,21 +18,23 @@ Applications can be used to build OTB pipelines from Python or C++ APIs.
 
 ### Python
 
-This is a work in progress. For now, `tricks.py` provides a set of helpers to build deep nets, and `otbtf.py` provides datasets which can be used in Tensorflow pipelines to train networks from python.
+`otbtf.py` targets python developers that want to train their own model from python with TensorFlow or Keras.
+It provides various classes for datasets and iterators to handle the _patches images_ generated from the `PatchesExtraction` OTB application.
+For instance, the `otbtf.Dataset` class provides a method `get_tf_dataset()` which returns a `tf.dataset` that can be used in your favorite TensorFlow pipelines, or convert your patches into TFRecords.
 
-## Portfolio
+`tricks.py` is here for backward compatibility with codes based on OTBTF 1.x and 2.x.
 
-Below are some screen captures of deep learning applications performed at large scale with OTBTF.
- - Image to image translation (Spot-7 image --> Wikimedia Map using CGAN)
-<img src ="doc/images/pix2pix.png" />
+## Examples
 
+Below are some screen captures of deep learning applications performed at large scale with OTBTF.
  - Landcover mapping (Spot-7 images --> Building map using semantic segmentation)
 <img src ="doc/images/landcover.png" />
 
- - Image enhancement (Enhancement of Sentinel-2 images at 1.5m  using SRGAN)
+ - Super resolution (Sentinel-2 images upsampled with the [SR4RS software](https://github.com/remicres/sr4rs), which is based on OTBTF)
 <img src ="doc/images/supresol.png" />
 
-You can read more details about these applications on [this blog](https://mdl4eo.irstea.fr/2019/)
+ - Image to image translation (Spot-7 image --> Wikimedia Map using CGAN. So unnecessary but fun!)
+<img src ="doc/images/pix2pix.png" />
 
 ## How to install
 
@@ -42,8 +44,8 @@ For now you have two options: either use the existing **docker image**, or build
 
 Use the latest image from dockerhub:
 ```
-docker pull mdl4eo/otbtf2.5:cpu
-docker run -u otbuser -v $(pwd):/home/otbuser mdl4eo/otbtf2.5:cpu otbcli_PatchesExtraction -help
+docker pull mdl4eo/otbtf3.0:cpu
+docker run -u otbuser -v $(pwd):/home/otbuser mdl4eo/otbtf3.0:cpu otbcli_PatchesExtraction -help
 ```
 
 Read more in the [docker use documentation](doc/DOCKERUSE.md).
@@ -59,10 +61,11 @@ Read more in the [build from sources documentation](doc/HOWTOBUILD.md).
 - in the `python` folder are provided some [ready-to-use deep networks, with documentation and scientific references](doc/EXAMPLES.md).
 - A book: *Cresson, R. (2020). Deep Learning for Remote Sensing Images with Open Source Software. CRC Press.* Use QGIS, OTB and Tensorflow to perform various kind of deep learning sorcery on remote sensing images (patch-based classification for landcover mapping, semantic segmentation of buildings, optical image restoration from joint SAR/Optical time series).
 - Check [our repository](https://github.com/remicres/otbtf_tutorials_resources) containing stuff (data and models) to begin with with!
+- Finally, take a look in the `test` folder. You will find plenty of command lines for applications tests!
 
 ## Contribute
 
-Every one can **contribute** to OTBTF! Don't be shy.
+Every one can **contribute** to OTBTF. Just open a PR :)
 
 ## Cite