diff --git a/.gitignore b/.gitignore index 312d1f6..9a36573 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,6 @@ fastlane/report.xml fastlane/Preview.html fastlane/screenshots/**/*.png fastlane/test_output + +*.DS_Store +*.mlmodel \ No newline at end of file diff --git a/SSDMobileNet-CoreML.xcodeproj/project.pbxproj b/ObjectDetection-CoreML.xcodeproj/project.pbxproj similarity index 59% rename from SSDMobileNet-CoreML.xcodeproj/project.pbxproj rename to ObjectDetection-CoreML.xcodeproj/project.pbxproj index ccda65c..09be997 100644 --- a/SSDMobileNet-CoreML.xcodeproj/project.pbxproj +++ b/ObjectDetection-CoreML.xcodeproj/project.pbxproj @@ -7,16 +7,35 @@ objects = { /* Begin PBXBuildFile section */ + 553366C62204898800A67647 /* SSDMobileNet_CoreMLTests.swift in Sources */ = {isa = PBXBuildFile; fileRef = 553366C52204898800A67647 /* SSDMobileNet_CoreMLTests.swift */; }; + 553366CF220489DD00A67647 /* UIImage+CVPixelBuffer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 553366CE220489DD00A67647 /* UIImage+CVPixelBuffer.swift */; }; 554F149C220362AB00B828B0 /* AppDelegate.swift in Sources */ = {isa = PBXBuildFile; fileRef = 554F149B220362AB00B828B0 /* AppDelegate.swift */; }; 554F149E220362AB00B828B0 /* ViewController.swift in Sources */ = {isa = PBXBuildFile; fileRef = 554F149D220362AB00B828B0 /* ViewController.swift */; }; 554F14A1220362AB00B828B0 /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 554F149F220362AB00B828B0 /* Main.storyboard */; }; 554F14A3220362AC00B828B0 /* Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 554F14A2220362AC00B828B0 /* Assets.xcassets */; }; 554F14A6220362AC00B828B0 /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 554F14A4220362AC00B828B0 /* LaunchScreen.storyboard */; }; 554F14AE220362DA00B828B0 /* VideoCapture.swift in Sources */ = {isa = PBXBuildFile; fileRef = 554F14AD220362DA00B828B0 /* VideoCapture.swift */; }; + 558918ED2208218800F182BA /* DrawingBoundingBoxView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 558918EC2208218800F182BA /* DrawingBoundingBoxView.swift */; }; + 558918EF22089FFE00F182BA /* Measure.swift in Sources */ = {isa = PBXBuildFile; fileRef = 558918EE22089FFE00F182BA /* Measure.swift */; }; + C427E67128C2D365001279AD /* yolov5s.mlmodel in Sources */ = {isa = PBXBuildFile; fileRef = C427E67028C2D365001279AD /* yolov5s.mlmodel */; }; /* End PBXBuildFile section */ +/* Begin PBXContainerItemProxy section */ + 553366C82204898800A67647 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 554F1490220362AB00B828B0 /* Project object */; + proxyType = 1; + remoteGlobalIDString = 554F1497220362AB00B828B0; + remoteInfo = "SSDMobileNet-CoreML"; + }; +/* End PBXContainerItemProxy section */ + /* Begin PBXFileReference section */ - 554F1498220362AB00B828B0 /* SSDMobileNet-CoreML.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "SSDMobileNet-CoreML.app"; sourceTree = BUILT_PRODUCTS_DIR; }; + 553366C32204898800A67647 /* ObjectDetection-CoreMLTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = "ObjectDetection-CoreMLTests.xctest"; sourceTree = BUILT_PRODUCTS_DIR; }; + 553366C52204898800A67647 /* SSDMobileNet_CoreMLTests.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = SSDMobileNet_CoreMLTests.swift; sourceTree = ""; }; + 553366C72204898800A67647 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; + 553366CE220489DD00A67647 /* UIImage+CVPixelBuffer.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = "UIImage+CVPixelBuffer.swift"; sourceTree = ""; }; + 554F1498220362AB00B828B0 /* ObjectDetection-CoreML.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = "ObjectDetection-CoreML.app"; sourceTree = BUILT_PRODUCTS_DIR; }; 554F149B220362AB00B828B0 /* AppDelegate.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = AppDelegate.swift; sourceTree = ""; }; 554F149D220362AB00B828B0 /* ViewController.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ViewController.swift; sourceTree = ""; }; 554F14A0220362AB00B828B0 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/Main.storyboard; sourceTree = ""; }; @@ -24,9 +43,19 @@ 554F14A5220362AC00B828B0 /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 554F14A7220362AC00B828B0 /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 554F14AD220362DA00B828B0 /* VideoCapture.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = VideoCapture.swift; sourceTree = ""; }; + 558918EC2208218800F182BA /* DrawingBoundingBoxView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = DrawingBoundingBoxView.swift; sourceTree = ""; }; + 558918EE22089FFE00F182BA /* Measure.swift */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.swift; path = Measure.swift; sourceTree = ""; }; + C427E67028C2D365001279AD /* yolov5s.mlmodel */ = {isa = PBXFileReference; lastKnownFileType = file.mlmodel; path = yolov5s.mlmodel; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ + 553366C02204898800A67647 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 554F1495220362AB00B828B0 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -37,10 +66,20 @@ /* End PBXFrameworksBuildPhase section */ /* Begin PBXGroup section */ + 553366C42204898800A67647 /* ObjectDetection-CoreMLTests */ = { + isa = PBXGroup; + children = ( + 553366C52204898800A67647 /* SSDMobileNet_CoreMLTests.swift */, + 553366C72204898800A67647 /* Info.plist */, + ); + path = "ObjectDetection-CoreMLTests"; + sourceTree = ""; + }; 554F148F220362AB00B828B0 = { isa = PBXGroup; children = ( - 554F149A220362AB00B828B0 /* SSDMobileNet-CoreML */, + 554F149A220362AB00B828B0 /* ObjectDetection-CoreML */, + 553366C42204898800A67647 /* ObjectDetection-CoreMLTests */, 554F1499220362AB00B828B0 /* Products */, ); sourceTree = ""; @@ -48,31 +87,70 @@ 554F1499220362AB00B828B0 /* Products */ = { isa = PBXGroup; children = ( - 554F1498220362AB00B828B0 /* SSDMobileNet-CoreML.app */, + 554F1498220362AB00B828B0 /* ObjectDetection-CoreML.app */, + 553366C32204898800A67647 /* ObjectDetection-CoreMLTests.xctest */, ); name = Products; sourceTree = ""; }; - 554F149A220362AB00B828B0 /* SSDMobileNet-CoreML */ = { + 554F149A220362AB00B828B0 /* ObjectDetection-CoreML */ = { isa = PBXGroup; children = ( + 554F149F220362AB00B828B0 /* Main.storyboard */, 554F149B220362AB00B828B0 /* AppDelegate.swift */, 554F149D220362AB00B828B0 /* ViewController.swift */, - 554F14AD220362DA00B828B0 /* VideoCapture.swift */, - 554F149F220362AB00B828B0 /* Main.storyboard */, + 558918EC2208218800F182BA /* DrawingBoundingBoxView.swift */, + 71B8322322E2B0BC00207D05 /* ultils */, + 71B8322222E2B0B200207D05 /* mlmodel */, 554F14A2220362AC00B828B0 /* Assets.xcassets */, 554F14A4220362AC00B828B0 /* LaunchScreen.storyboard */, 554F14A7220362AC00B828B0 /* Info.plist */, ); - path = "SSDMobileNet-CoreML"; + path = "ObjectDetection-CoreML"; + sourceTree = ""; + }; + 71B8322222E2B0B200207D05 /* mlmodel */ = { + isa = PBXGroup; + children = ( + C427E67028C2D365001279AD /* yolov5s.mlmodel */, + ); + path = mlmodel; + sourceTree = ""; + }; + 71B8322322E2B0BC00207D05 /* ultils */ = { + isa = PBXGroup; + children = ( + 554F14AD220362DA00B828B0 /* VideoCapture.swift */, + 558918EE22089FFE00F182BA /* Measure.swift */, + 553366CE220489DD00A67647 /* UIImage+CVPixelBuffer.swift */, + ); + path = ultils; sourceTree = ""; }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ - 554F1497220362AB00B828B0 /* SSDMobileNet-CoreML */ = { + 553366C22204898800A67647 /* ObjectDetection-CoreMLTests */ = { isa = PBXNativeTarget; - buildConfigurationList = 554F14AA220362AC00B828B0 /* Build configuration list for PBXNativeTarget "SSDMobileNet-CoreML" */; + buildConfigurationList = 553366CA2204898800A67647 /* Build configuration list for PBXNativeTarget "ObjectDetection-CoreMLTests" */; + buildPhases = ( + 553366BF2204898800A67647 /* Sources */, + 553366C02204898800A67647 /* Frameworks */, + 553366C12204898800A67647 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + 553366C92204898800A67647 /* PBXTargetDependency */, + ); + name = "ObjectDetection-CoreMLTests"; + productName = "SSDMobileNet-CoreMLTests"; + productReference = 553366C32204898800A67647 /* ObjectDetection-CoreMLTests.xctest */; + productType = "com.apple.product-type.bundle.unit-test"; + }; + 554F1497220362AB00B828B0 /* ObjectDetection-CoreML */ = { + isa = PBXNativeTarget; + buildConfigurationList = 554F14AA220362AC00B828B0 /* Build configuration list for PBXNativeTarget "ObjectDetection-CoreML" */; buildPhases = ( 554F1494220362AB00B828B0 /* Sources */, 554F1495220362AB00B828B0 /* Frameworks */, @@ -82,9 +160,9 @@ ); dependencies = ( ); - name = "SSDMobileNet-CoreML"; + name = "ObjectDetection-CoreML"; productName = "SSDMobileNet-CoreML"; - productReference = 554F1498220362AB00B828B0 /* SSDMobileNet-CoreML.app */; + productReference = 554F1498220362AB00B828B0 /* ObjectDetection-CoreML.app */; productType = "com.apple.product-type.application"; }; /* End PBXNativeTarget section */ @@ -97,12 +175,16 @@ LastUpgradeCheck = 1010; ORGANIZATIONNAME = tucan9389; TargetAttributes = { + 553366C22204898800A67647 = { + CreatedOnToolsVersion = 10.1; + TestTargetID = 554F1497220362AB00B828B0; + }; 554F1497220362AB00B828B0 = { CreatedOnToolsVersion = 10.1; }; }; }; - buildConfigurationList = 554F1493220362AB00B828B0 /* Build configuration list for PBXProject "SSDMobileNet-CoreML" */; + buildConfigurationList = 554F1493220362AB00B828B0 /* Build configuration list for PBXProject "ObjectDetection-CoreML" */; compatibilityVersion = "Xcode 9.3"; developmentRegion = en; hasScannedForEncodings = 0; @@ -115,12 +197,20 @@ projectDirPath = ""; projectRoot = ""; targets = ( - 554F1497220362AB00B828B0 /* SSDMobileNet-CoreML */, + 554F1497220362AB00B828B0 /* ObjectDetection-CoreML */, + 553366C22204898800A67647 /* ObjectDetection-CoreMLTests */, ); }; /* End PBXProject section */ /* Begin PBXResourcesBuildPhase section */ + 553366C12204898800A67647 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 554F1496220362AB00B828B0 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; @@ -134,18 +224,38 @@ /* End PBXResourcesBuildPhase section */ /* Begin PBXSourcesBuildPhase section */ + 553366BF2204898800A67647 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + 553366C62204898800A67647 /* SSDMobileNet_CoreMLTests.swift in Sources */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 554F1494220362AB00B828B0 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( 554F149E220362AB00B828B0 /* ViewController.swift in Sources */, + 558918EF22089FFE00F182BA /* Measure.swift in Sources */, + 558918ED2208218800F182BA /* DrawingBoundingBoxView.swift in Sources */, + 553366CF220489DD00A67647 /* UIImage+CVPixelBuffer.swift in Sources */, 554F14AE220362DA00B828B0 /* VideoCapture.swift in Sources */, + C427E67128C2D365001279AD /* yolov5s.mlmodel in Sources */, 554F149C220362AB00B828B0 /* AppDelegate.swift in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ +/* Begin PBXTargetDependency section */ + 553366C92204898800A67647 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 554F1497220362AB00B828B0 /* ObjectDetection-CoreML */; + targetProxy = 553366C82204898800A67647 /* PBXContainerItemProxy */; + }; +/* End PBXTargetDependency section */ + /* Begin PBXVariantGroup section */ 554F149F220362AB00B828B0 /* Main.storyboard */ = { isa = PBXVariantGroup; @@ -166,6 +276,46 @@ /* End PBXVariantGroup section */ /* Begin XCBuildConfiguration section */ + 553366CB2204898800A67647 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = 5WXJ4Z4H69; + INFOPLIST_FILE = "SSDMobileNet-CoreMLTests/Info.plist"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "com.tucan9389.SSDMobileNet-CoreMLTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 4.2; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ObjectDetection-CoreML.app/ObjectDetection-CoreML"; + }; + name = Debug; + }; + 553366CC2204898800A67647 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + BUNDLE_LOADER = "$(TEST_HOST)"; + CODE_SIGN_STYLE = Automatic; + DEVELOPMENT_TEAM = 5WXJ4Z4H69; + INFOPLIST_FILE = "SSDMobileNet-CoreMLTests/Info.plist"; + LD_RUNPATH_SEARCH_PATHS = ( + "$(inherited)", + "@executable_path/Frameworks", + "@loader_path/Frameworks", + ); + PRODUCT_BUNDLE_IDENTIFIER = "com.tucan9389.SSDMobileNet-CoreMLTests"; + PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_VERSION = 4.2; + TARGETED_DEVICE_FAMILY = "1,2"; + TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ObjectDetection-CoreML.app/ObjectDetection-CoreML"; + }; + name = Release; + }; 554F14A8220362AC00B828B0 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -288,12 +438,13 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = 5WXJ4Z4H69; - INFOPLIST_FILE = "SSDMobileNet-CoreML/Info.plist"; + INFOPLIST_FILE = "ObjectDetection-CoreML/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - PRODUCT_BUNDLE_IDENTIFIER = "com.tucan9389.SSDMobileNet-CoreML"; + PRODUCT_BUNDLE_IDENTIFIER = "com.tucan9389.ObjectDetection-CoreML"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 4.2; TARGETED_DEVICE_FAMILY = "1,2"; @@ -306,12 +457,13 @@ ASSETCATALOG_COMPILER_APPICON_NAME = AppIcon; CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = 5WXJ4Z4H69; - INFOPLIST_FILE = "SSDMobileNet-CoreML/Info.plist"; + INFOPLIST_FILE = "ObjectDetection-CoreML/Info.plist"; + IPHONEOS_DEPLOYMENT_TARGET = 12.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", ); - PRODUCT_BUNDLE_IDENTIFIER = "com.tucan9389.SSDMobileNet-CoreML"; + PRODUCT_BUNDLE_IDENTIFIER = "com.tucan9389.ObjectDetection-CoreML"; PRODUCT_NAME = "$(TARGET_NAME)"; SWIFT_VERSION = 4.2; TARGETED_DEVICE_FAMILY = "1,2"; @@ -321,7 +473,16 @@ /* End XCBuildConfiguration section */ /* Begin XCConfigurationList section */ - 554F1493220362AB00B828B0 /* Build configuration list for PBXProject "SSDMobileNet-CoreML" */ = { + 553366CA2204898800A67647 /* Build configuration list for PBXNativeTarget "ObjectDetection-CoreMLTests" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 553366CB2204898800A67647 /* Debug */, + 553366CC2204898800A67647 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; + 554F1493220362AB00B828B0 /* Build configuration list for PBXProject "ObjectDetection-CoreML" */ = { isa = XCConfigurationList; buildConfigurations = ( 554F14A8220362AC00B828B0 /* Debug */, @@ -330,7 +491,7 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 554F14AA220362AC00B828B0 /* Build configuration list for PBXNativeTarget "SSDMobileNet-CoreML" */ = { + 554F14AA220362AC00B828B0 /* Build configuration list for PBXNativeTarget "ObjectDetection-CoreML" */ = { isa = XCConfigurationList; buildConfigurations = ( 554F14AB220362AC00B828B0 /* Debug */, diff --git a/ObjectDetection-CoreML.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/ObjectDetection-CoreML.xcodeproj/project.xcworkspace/contents.xcworkspacedata new file mode 100644 index 0000000..bb61b6c --- /dev/null +++ b/ObjectDetection-CoreML.xcodeproj/project.xcworkspace/contents.xcworkspacedata @@ -0,0 +1,7 @@ + + + + + diff --git a/SSDMobileNet-CoreML.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist b/ObjectDetection-CoreML.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist similarity index 100% rename from SSDMobileNet-CoreML.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist rename to ObjectDetection-CoreML.xcodeproj/project.xcworkspace/xcshareddata/IDEWorkspaceChecks.plist diff --git a/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcbaselines/553366C22204898800A67647.xcbaseline/0BCEFA5F-115F-452C-A40D-3ECF2FEFD78E.plist b/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcbaselines/553366C22204898800A67647.xcbaseline/0BCEFA5F-115F-452C-A40D-3ECF2FEFD78E.plist new file mode 100644 index 0000000..55dd37a --- /dev/null +++ b/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcbaselines/553366C22204898800A67647.xcbaseline/0BCEFA5F-115F-452C-A40D-3ECF2FEFD78E.plist @@ -0,0 +1,22 @@ + + + + + classNames + + SSDMobileNet_CoreMLTests + + testPerformanceSSDMobileNetV1Model() + + com.apple.XCTPerformanceMetric_WallClockTime + + baselineAverage + 0.102 + baselineIntegrationDisplayName + Local Baseline + + + + + + diff --git a/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcbaselines/553366C22204898800A67647.xcbaseline/Info.plist b/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcbaselines/553366C22204898800A67647.xcbaseline/Info.plist new file mode 100644 index 0000000..92f4b50 --- /dev/null +++ b/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcbaselines/553366C22204898800A67647.xcbaseline/Info.plist @@ -0,0 +1,21 @@ + + + + + runDestinationsByUUID + + 0BCEFA5F-115F-452C-A40D-3ECF2FEFD78E + + targetArchitecture + arm64 + targetDevice + + modelCode + iPhone10,6 + platformIdentifier + com.apple.platform.iphoneos + + + + + diff --git a/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcschemes/ObjectDetection-CoreML.xcscheme b/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcschemes/ObjectDetection-CoreML.xcscheme new file mode 100644 index 0000000..32fccf3 --- /dev/null +++ b/ObjectDetection-CoreML.xcodeproj/xcshareddata/xcschemes/ObjectDetection-CoreML.xcscheme @@ -0,0 +1,101 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/SSDMobileNet-CoreML/AppDelegate.swift b/ObjectDetection-CoreML/AppDelegate.swift similarity index 100% rename from SSDMobileNet-CoreML/AppDelegate.swift rename to ObjectDetection-CoreML/AppDelegate.swift diff --git a/SSDMobileNet-CoreML/Assets.xcassets/AppIcon.appiconset/Contents.json b/ObjectDetection-CoreML/Assets.xcassets/AppIcon.appiconset/Contents.json similarity index 100% rename from SSDMobileNet-CoreML/Assets.xcassets/AppIcon.appiconset/Contents.json rename to ObjectDetection-CoreML/Assets.xcassets/AppIcon.appiconset/Contents.json diff --git a/SSDMobileNet-CoreML/Assets.xcassets/Contents.json b/ObjectDetection-CoreML/Assets.xcassets/Contents.json similarity index 100% rename from SSDMobileNet-CoreML/Assets.xcassets/Contents.json rename to ObjectDetection-CoreML/Assets.xcassets/Contents.json diff --git a/ObjectDetection-CoreML/Assets.xcassets/adult-building-business-1436289.imageset/Contents.json b/ObjectDetection-CoreML/Assets.xcassets/adult-building-business-1436289.imageset/Contents.json new file mode 100644 index 0000000..f3c9c3d --- /dev/null +++ b/ObjectDetection-CoreML/Assets.xcassets/adult-building-business-1436289.imageset/Contents.json @@ -0,0 +1,21 @@ +{ + "images" : [ + { + "idiom" : "universal", + "filename" : "adult-building-business-1436289.jpg", + "scale" : "1x" + }, + { + "idiom" : "universal", + "scale" : "2x" + }, + { + "idiom" : "universal", + "scale" : "3x" + } + ], + "info" : { + "version" : 1, + "author" : "xcode" + } +} \ No newline at end of file diff --git a/ObjectDetection-CoreML/Assets.xcassets/adult-building-business-1436289.imageset/adult-building-business-1436289.jpg b/ObjectDetection-CoreML/Assets.xcassets/adult-building-business-1436289.imageset/adult-building-business-1436289.jpg new file mode 100644 index 0000000..ff142ee Binary files /dev/null and b/ObjectDetection-CoreML/Assets.xcassets/adult-building-business-1436289.imageset/adult-building-business-1436289.jpg differ diff --git a/SSDMobileNet-CoreML/Base.lproj/LaunchScreen.storyboard b/ObjectDetection-CoreML/Base.lproj/LaunchScreen.storyboard similarity index 100% rename from SSDMobileNet-CoreML/Base.lproj/LaunchScreen.storyboard rename to ObjectDetection-CoreML/Base.lproj/LaunchScreen.storyboard diff --git a/ObjectDetection-CoreML/Base.lproj/Main.storyboard b/ObjectDetection-CoreML/Base.lproj/Main.storyboard new file mode 100644 index 0000000..c32cfe7 --- /dev/null +++ b/ObjectDetection-CoreML/Base.lproj/Main.storyboard @@ -0,0 +1,147 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/ObjectDetection-CoreML/DrawingBoundingBoxView.swift b/ObjectDetection-CoreML/DrawingBoundingBoxView.swift new file mode 100644 index 0000000..bff2705 --- /dev/null +++ b/ObjectDetection-CoreML/DrawingBoundingBoxView.swift @@ -0,0 +1,81 @@ +// +// DrawingBoundingBoxView.swift +// SSDMobileNet-CoreML +// +// Created by GwakDoyoung on 04/02/2019. +// Copyright Β© 2019 tucan9389. All rights reserved. +// + +import UIKit +import Vision + +class DrawingBoundingBoxView: UIView { + + static private var colors: [String: UIColor] = [:] + + public func labelColor(with label: String) -> UIColor { + if let color = DrawingBoundingBoxView.colors[label] { + return color + } else { + let color = UIColor(hue: .random(in: 0...1), saturation: 1, brightness: 1, alpha: 0.8) + DrawingBoundingBoxView.colors[label] = color + return color + } + } + + public var predictedObjects: [VNRecognizedObjectObservation] = [] { + didSet { + self.drawBoxs(with: predictedObjects) + self.setNeedsDisplay() + } + } + + func drawBoxs(with predictions: [VNRecognizedObjectObservation]) { + subviews.forEach({ $0.removeFromSuperview() }) + + for prediction in predictions { + createLabelAndBox(prediction: prediction) + } + } + + func createLabelAndBox(prediction: VNRecognizedObjectObservation) { + let labelString: String? = prediction.label + let color: UIColor = labelColor(with: labelString ?? "N/A") + + let scale = CGAffineTransform.identity.scaledBy(x: bounds.width, y: bounds.height) + let transform = CGAffineTransform(scaleX: 1, y: -1).translatedBy(x: 0, y: -1) + let bgRect = prediction.boundingBox.applying(transform).applying(scale) + + let bgView = UIView(frame: bgRect) + bgView.layer.borderColor = color.cgColor + bgView.layer.borderWidth = 4 + bgView.backgroundColor = UIColor.clear + addSubview(bgView) + + let label = UILabel(frame: CGRect(x: 0, y: 0, width: 300, height: 300)) + label.text = labelString ?? "N/A" + label.font = UIFont.systemFont(ofSize: 13) + label.textColor = UIColor.black + label.backgroundColor = color + label.sizeToFit() + label.frame = CGRect(x: bgRect.origin.x, y: bgRect.origin.y - label.frame.height, + width: label.frame.width, height: label.frame.height) + addSubview(label) + } +} + +extension VNRecognizedObjectObservation { + var label: String? { + return self.labels.first?.identifier + } +} + +extension CGRect { + func toString(digit: Int) -> String { + let xStr = String(format: "%.\(digit)f", origin.x) + let yStr = String(format: "%.\(digit)f", origin.y) + let wStr = String(format: "%.\(digit)f", width) + let hStr = String(format: "%.\(digit)f", height) + return "(\(xStr), \(yStr), \(wStr), \(hStr))" + } +} diff --git a/SSDMobileNet-CoreML/Info.plist b/ObjectDetection-CoreML/Info.plist similarity index 100% rename from SSDMobileNet-CoreML/Info.plist rename to ObjectDetection-CoreML/Info.plist diff --git a/ObjectDetection-CoreML/ViewController.swift b/ObjectDetection-CoreML/ViewController.swift new file mode 100644 index 0000000..4e62f1a --- /dev/null +++ b/ObjectDetection-CoreML/ViewController.swift @@ -0,0 +1,295 @@ +// +// ViewController.swift +// SSDMobileNet-CoreML +// +// Created by GwakDoyoung on 01/02/2019. +// Copyright Β© 2019 tucan9389. All rights reserved. +// + +import UIKit +import Vision +import CoreMedia + +class ViewController: UIViewController { + + // MARK: - UI Properties + @IBOutlet weak var videoPreview: UIView! + @IBOutlet weak var boxesView: DrawingBoundingBoxView! + @IBOutlet weak var labelsTableView: UITableView! + + @IBOutlet weak var inferenceLabel: UILabel! + @IBOutlet weak var etimeLabel: UILabel! + @IBOutlet weak var fpsLabel: UILabel! + + // MARK - Core ML model + // YOLOv3(iOS12+), YOLOv3FP16(iOS12+), YOLOv3Int8LUT(iOS12+) + // YOLOv3Tiny(iOS12+), YOLOv3TinyFP16(iOS12+), YOLOv3TinyInt8LUT(iOS12+) + // MobileNetV2_SSDLite(iOS12+), ObjectDetector(iOS12+) + let objectDectectionModel = YOLOv3Tiny() + + //XS + //XS Max + //XR + //X + //8 + //8+ + //7 + //6S+ + //6+ + + //XS + // YOLOv3 : 108 108 8 + // YOLOv3FP16 : 104 104 9 + // YOLOv3Int8LUT : 101 102 9 + // YOLOv3Tiny : 46 46 21 + // YOLOv3TinyFP16 : 51 51 19 + // YOLOv3TinyInt8LUT : 45 45 21 + // MobileNetV2_SSDLite : 31 32 23 + // ObjectDetector : 24 25 23 + // + //XS Max + // YOLOv3 : 93 93 10 + // YOLOv3FP16 : 89 89 10 + // YOLOv3Int8LUT : 92 92 10 + // YOLOv3Tiny : 41 42 22 + // YOLOv3TinyFP16 : 41 41 23 + // YOLOv3TinyInt8LUT : 39 39 24 + // MobileNetV2_SSDLite : 31 31 23 + // ObjectDetector : 26 26 23 + // + //XR + // YOLOv3 : 100 100 9 + // YOLOv3FP16 : 101 101 8 + // YOLOv3Int8LUT : 100 102 9 + // YOLOv3Tiny : 47 48 20 + // YOLOv3TinyFP16 : 44 44 21 + // YOLOv3TinyInt8LUT : 39 40 23 + // MobileNetV2_SSDLite : 31 32 23 + // ObjectDetector : 23 23 24 + // + //X + // YOLOv3 : 356 357 2 + // YOLOv3FP16 : 348 348 2 + // YOLOv3Int8LUT : 337 338 2 + // YOLOv3Tiny : 106 106 8 + // YOLOv3TinyFP16 : 103 104 9 + // YOLOv3TinyInt8LUT : 106 107 8 + // MobileNetV2_SSDLite : 109 109 8 + // ObjectDetector : 63 64 14 + // + //7+ + // YOLOv3 : 569 569 1 + // YOLOv3FP16 : 572 572 1 + // YOLOv3Int8LUT : 575 576 1 + // YOLOv3Tiny : 165 166 5 + // YOLOv3TinyFP16 : 165 165 5 + // YOLOv3TinyInt8LUT : 160 160 5 + // MobileNetV2_SSDLite : 141 142 6 + // ObjectDetector : 86 87 10 + // + //7 + // YOLOv3 : 561 561 1 + // YOLOv3FP16 : 565 565 1 + // YOLOv3Int8LUT : 572 573 1 + // YOLOv3Tiny : 168 169 5 + // YOLOv3TinyFP16 : 167 167 5 + // YOLOv3TinyInt8LUT : 161 161 5 + // MobileNetV2_SSDLite : 134 134 6 + // ObjectDetector : 84 85 11 + + // MARK: - Vision Properties + var request: VNCoreMLRequest? + var visionModel: VNCoreMLModel? + var isInferencing = false + + // MARK: - AV Property + var videoCapture: VideoCapture! + let semaphore = DispatchSemaphore(value: 1) + var lastExecution = Date() + + // MARK: - TableView Data + var predictions: [VNRecognizedObjectObservation] = [] + + // MARK - Performance Measurement Property + private let πŸ‘¨β€πŸ”§ = πŸ“() + + let maf1 = MovingAverageFilter() + let maf2 = MovingAverageFilter() + let maf3 = MovingAverageFilter() + + // MARK: - View Controller Life Cycle + override func viewDidLoad() { + super.viewDidLoad() + + // setup the model + setUpModel() + + // setup camera + setUpCamera() + + // setup delegate for performance measurement + πŸ‘¨β€πŸ”§.delegate = self + } + + override func didReceiveMemoryWarning() { + super.didReceiveMemoryWarning() + } + + override func viewWillAppear(_ animated: Bool) { + super.viewWillAppear(animated) + self.videoCapture.start() + } + + override func viewWillDisappear(_ animated: Bool) { + super.viewWillDisappear(animated) + self.videoCapture.stop() + } + + // MARK: - Setup Core ML + func setUpModel() { + if let visionModel = try? VNCoreMLModel(for: objectDectectionModel.model) { + self.visionModel = visionModel + request = VNCoreMLRequest(model: visionModel, completionHandler: visionRequestDidComplete) + request?.imageCropAndScaleOption = .scaleFill + } else { + fatalError("fail to create vision model") + } + } + + // MARK: - SetUp Video + func setUpCamera() { + videoCapture = VideoCapture() + videoCapture.delegate = self + videoCapture.fps = 30 + videoCapture.setUp(sessionPreset: .vga640x480) { success in + + if success { + // add preview view on the layer + if let previewLayer = self.videoCapture.previewLayer { + self.videoPreview.layer.addSublayer(previewLayer) + self.resizePreviewLayer() + } + + // start video preview when setup is done + self.videoCapture.start() + } + } + } + + override func viewDidLayoutSubviews() { + super.viewDidLayoutSubviews() + resizePreviewLayer() + } + + func resizePreviewLayer() { + videoCapture.previewLayer?.frame = videoPreview.bounds + } +} + +// MARK: - VideoCaptureDelegate +extension ViewController: VideoCaptureDelegate { + func videoCapture(_ capture: VideoCapture, didCaptureVideoFrame pixelBuffer: CVPixelBuffer?, timestamp: CMTime) { + // the captured image from camera is contained on pixelBuffer + if !self.isInferencing, let pixelBuffer = pixelBuffer { + self.isInferencing = true + + // start of measure + self.πŸ‘¨β€πŸ”§.πŸŽ¬πŸ‘() + + // predict! + self.predictUsingVision(pixelBuffer: pixelBuffer) + } + } +} + +extension ViewController { + func predictUsingVision(pixelBuffer: CVPixelBuffer) { + guard let request = request else { fatalError() } + // vision framework configures the input size of image following our model's input configuration automatically + self.semaphore.wait() + let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer) + try? handler.perform([request]) + } + + // MARK: - Post-processing + func visionRequestDidComplete(request: VNRequest, error: Error?) { + self.πŸ‘¨β€πŸ”§.🏷(with: "endInference") + if let predictions = request.results as? [VNRecognizedObjectObservation] { +// print(predictions.first?.labels.first?.identifier ?? "nil") +// print(predictions.first?.labels.first?.confidence ?? -1) + + self.predictions = predictions + DispatchQueue.main.async { + self.boxesView.predictedObjects = predictions + self.labelsTableView.reloadData() + + // end of measure + self.πŸ‘¨β€πŸ”§.🎬🀚() + + self.isInferencing = false + } + } else { + // end of measure + self.πŸ‘¨β€πŸ”§.🎬🀚() + + self.isInferencing = false + } + self.semaphore.signal() + } +} + +extension ViewController: UITableViewDataSource { + func tableView(_ tableView: UITableView, numberOfRowsInSection section: Int) -> Int { + return predictions.count + } + + func tableView(_ tableView: UITableView, cellForRowAt indexPath: IndexPath) -> UITableViewCell { + guard let cell = tableView.dequeueReusableCell(withIdentifier: "InfoCell") else { + return UITableViewCell() + } + + let rectString = predictions[indexPath.row].boundingBox.toString(digit: 2) + let confidence = predictions[indexPath.row].labels.first?.confidence ?? -1 + let confidenceString = String(format: "%.3f", confidence/*Math.sigmoid(confidence)*/) + + cell.textLabel?.text = predictions[indexPath.row].label ?? "N/A" + cell.detailTextLabel?.text = "\(rectString), \(confidenceString)" + return cell + } +} + +// MARK: - πŸ“(Performance Measurement) Delegate +extension ViewController: πŸ“Delegate { + func updateMeasure(inferenceTime: Double, executionTime: Double, fps: Int) { + //print(executionTime, fps) + DispatchQueue.main.async { + self.maf1.append(element: Int(inferenceTime*1000.0)) + self.maf2.append(element: Int(executionTime*1000.0)) + self.maf3.append(element: fps) + + self.inferenceLabel.text = "inference: \(self.maf1.averageValue) ms" + self.etimeLabel.text = "execution: \(self.maf2.averageValue) ms" + self.fpsLabel.text = "fps: \(self.maf3.averageValue)" + } + } +} + +class MovingAverageFilter { + private var arr: [Int] = [] + private let maxCount = 10 + + public func append(element: Int) { + arr.append(element) + if arr.count > maxCount { + arr.removeFirst() + } + } + + public var averageValue: Int { + guard !arr.isEmpty else { return 0 } + let sum = arr.reduce(0) { $0 + $1 } + return Int(Double(sum) / Double(arr.count)) + } +} + + diff --git a/ObjectDetection-CoreML/ultils/Measure.swift b/ObjectDetection-CoreML/ultils/Measure.swift new file mode 100644 index 0000000..29b797a --- /dev/null +++ b/ObjectDetection-CoreML/ultils/Measure.swift @@ -0,0 +1,86 @@ +// +// Measure.swift +// TurtleApp-CoreML +// +// Created by GwakDoyoung on 03/07/2018. +// Copyright Β© 2018 GwakDoyoung. All rights reserved. +// + +import UIKit + +protocol πŸ“Delegate { + func updateMeasure(inferenceTime: Double, executionTime: Double, fps: Int) +} +// Performance Measurement +class πŸ“ { + + var delegate: πŸ“Delegate? + + var index: Int = -1 + var measurements: [Dictionary] + + init() { + let measurement = [ + "start": CACurrentMediaTime(), + "end": CACurrentMediaTime() + ] + measurements = Array>(repeating: measurement, count: 30) + } + + // start + func πŸŽ¬πŸ‘() { + index += 1 + index %= 30 + measurements[index] = [:] + + 🏷(for: index, with: "start") + } + + // stop + func 🎬🀚() { + 🏷(for: index, with: "end") + + let beforeMeasurement = getBeforeMeasurment(for: index) + let currentMeasurement = measurements[index] + if let startTime = currentMeasurement["start"], + let endInferenceTime = currentMeasurement["endInference"], + let endTime = currentMeasurement["end"], + let beforeStartTime = beforeMeasurement["start"] { + delegate?.updateMeasure(inferenceTime: endInferenceTime - startTime, + executionTime: endTime - startTime, + fps: Int(1/(startTime - beforeStartTime))) + } + + } + + // labeling with + func 🏷(with msg: String? = "") { + 🏷(for: index, with: msg) + } + + private func 🏷(for index: Int, with msg: String? = "") { + if let message = msg { + measurements[index][message] = CACurrentMediaTime() + } + } + + private func getBeforeMeasurment(for index: Int) -> Dictionary { + return measurements[(index + 30 - 1) % 30] + } + + // log + func πŸ–¨() { + + } +} + +class MeasureLogView: UIView { + let etimeLabel = UILabel(frame: .zero) + let fpsLabel = UILabel(frame: .zero) + + + required init?(coder aDecoder: NSCoder) { + fatalError("init(coder:) has not been implemented") + } + +} diff --git a/ObjectDetection-CoreML/ultils/UIImage+CVPixelBuffer.swift b/ObjectDetection-CoreML/ultils/UIImage+CVPixelBuffer.swift new file mode 100644 index 0000000..aef6132 --- /dev/null +++ b/ObjectDetection-CoreML/ultils/UIImage+CVPixelBuffer.swift @@ -0,0 +1,51 @@ +import UIKit + +extension UIImage { + func pixelBufferFromImage() -> CVPixelBuffer { + let ciimage = CIImage(image: self) + //let cgimage = convertCIImageToCGImage(inputImage: ciimage!) + let tmpcontext = CIContext(options: nil) + let cgimage = tmpcontext.createCGImage(ciimage!, from: ciimage!.extent) + + let cfnumPointer = UnsafeMutablePointer.allocate(capacity: 1) + let cfnum = CFNumberCreate(kCFAllocatorDefault, .intType, cfnumPointer) + let keys: [CFString] = [kCVPixelBufferCGImageCompatibilityKey, kCVPixelBufferCGBitmapContextCompatibilityKey, kCVPixelBufferBytesPerRowAlignmentKey] + let values: [CFTypeRef] = [kCFBooleanTrue, kCFBooleanTrue, cfnum!] + let keysPointer = UnsafeMutablePointer.allocate(capacity: 1) + let valuesPointer = UnsafeMutablePointer.allocate(capacity: 1) + keysPointer.initialize(to: keys) + valuesPointer.initialize(to: values) + + let options = CFDictionaryCreate(kCFAllocatorDefault, keysPointer, valuesPointer, keys.count, nil, nil) + + let width = cgimage!.width + let height = cgimage!.height + + var pxbuffer: CVPixelBuffer? + // if pxbuffer = nil, you will get status = -6661 + var status = CVPixelBufferCreate(kCFAllocatorDefault, width, height, + kCVPixelFormatType_32BGRA, options, &pxbuffer) + status = CVPixelBufferLockBaseAddress(pxbuffer!, CVPixelBufferLockFlags(rawValue: 0)); + + let bufferAddress = CVPixelBufferGetBaseAddress(pxbuffer!); + + + let rgbColorSpace = CGColorSpaceCreateDeviceRGB(); + let bytesperrow = CVPixelBufferGetBytesPerRow(pxbuffer!) + let context = CGContext(data: bufferAddress, + width: width, + height: height, + bitsPerComponent: 8, + bytesPerRow: bytesperrow, + space: rgbColorSpace, + bitmapInfo: CGImageAlphaInfo.premultipliedFirst.rawValue | CGBitmapInfo.byteOrder32Little.rawValue); + context?.concatenate(CGAffineTransform(rotationAngle: 0)) + context?.concatenate(__CGAffineTransformMake( 1, 0, 0, -1, 0, CGFloat(height) )) //Flip Vertical +// context?.concatenate(__CGAffineTransformMake( -1.0, 0.0, 0.0, 1.0, CGFloat(width), 0.0)) //Flip Horizontal + + + context?.draw(cgimage!, in: CGRect(x:0, y:0, width:CGFloat(width), height:CGFloat(height))); + status = CVPixelBufferUnlockBaseAddress(pxbuffer!, CVPixelBufferLockFlags(rawValue: 0)); + return pxbuffer!; + + }} diff --git a/SSDMobileNet-CoreML/VideoCapture.swift b/ObjectDetection-CoreML/ultils/VideoCapture.swift similarity index 100% rename from SSDMobileNet-CoreML/VideoCapture.swift rename to ObjectDetection-CoreML/ultils/VideoCapture.swift diff --git a/ObjectDetection-CoreMLTests/Info.plist b/ObjectDetection-CoreMLTests/Info.plist new file mode 100644 index 0000000..6c40a6c --- /dev/null +++ b/ObjectDetection-CoreMLTests/Info.plist @@ -0,0 +1,22 @@ + + + + + CFBundleDevelopmentRegion + $(DEVELOPMENT_LANGUAGE) + CFBundleExecutable + $(EXECUTABLE_NAME) + CFBundleIdentifier + $(PRODUCT_BUNDLE_IDENTIFIER) + CFBundleInfoDictionaryVersion + 6.0 + CFBundleName + $(PRODUCT_NAME) + CFBundlePackageType + BNDL + CFBundleShortVersionString + 1.0 + CFBundleVersion + 1 + + diff --git a/ObjectDetection-CoreMLTests/SSDMobileNet_CoreMLTests.swift b/ObjectDetection-CoreMLTests/SSDMobileNet_CoreMLTests.swift new file mode 100644 index 0000000..0862795 --- /dev/null +++ b/ObjectDetection-CoreMLTests/SSDMobileNet_CoreMLTests.swift @@ -0,0 +1,51 @@ +// +// SSDMobileNet_CoreMLTests.swift +// SSDMobileNet-CoreMLTests +// +// Created by GwakDoyoung on 01/02/2019. +// Copyright Β© 2019 tucan9389. All rights reserved. +// + +import XCTest +import Vision + +class SSDMobileNet_CoreMLTests: XCTestCase { + + // MARK: - Vision Properties + var cpmRequest: VNCoreMLRequest? + var cpmModel: VNCoreMLModel? + + var ssdRequest: VNCoreMLRequest? + var ssdModel: VNCoreMLModel? + + let image = UIImage(named: "adult-building-business-1436289") + var pixelBuffer: CVPixelBuffer? + + override func setUp() { + // <# SSD Mobilenet v1 model #> + ssdModel = try? VNCoreMLModel(for: ssd_mobilenet_feature_extractor().model) + if let visionModel = ssdModel { + ssdRequest = VNCoreMLRequest(model: visionModel, completionHandler: nil) + } + ssdRequest?.imageCropAndScaleOption = .scaleFill + + // image configuration + pixelBuffer = image?.pixelBufferFromImage() + } + + override func tearDown() { + // Put teardown code here. This method is called after the invocation of each test method in the class. + } + + func testPerformanceSSDMobileNetV1Model() { + guard let pixelBuffer = pixelBuffer, + let request = ssdRequest else { + fatalError() + } + self.measure { + let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer) + try? handler.perform([request]) + } + } + +} diff --git a/README.md b/README.md index 52ebb69..2e1b868 100644 --- a/README.md +++ b/README.md @@ -1 +1,244 @@ -# SSDMobileNet-CoreML \ No newline at end of file +# ObjectDetection-CoreML + +- supporting model: `yolov3` +- work in process: `yolov5` + +![platform-ios](https://img.shields.io/badge/platform-ios-lightgrey.svg) +![swift-version](https://img.shields.io/badge/swift-4.2-red.svg) +![lisence](https://img.shields.io/badge/license-MIT-black.svg) + +This project is Object Detection on iOS with Core ML.
If you are interested in iOS + Machine Learning, visit [here](https://github.com/motlabs/iOS-Proejcts-with-ML-Models) you can see various DEMOs.
![SSDMobileNetV2-DEMO](https://user-images.githubusercontent.com/37643248/188248210-2c02790b-6231-4549-8211-e3edcccba9e8.gif) + +## How it works + +(preparing...) + +## Requirements + +- Xcode 10.3+ +- iOS 13.0+ +- Swift 4.2 + +## Model + +### Model Matadata + +image + +### Model Size, Minimum iOS Version, Download Link + +| Model | Size
(MB) | Minimum
iOS Version | Download
Link | Trained Dataset +| :---- | :----: | :----: | ---- | --- | +| yolov5n.mlmodel | 7.52 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5n.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5s.mlmodel | 28 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5s.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5m.mlmodel | 81.2 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5m.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5l.mlmodel | 178 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5l.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5x.mlmodel | 331 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5x.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5n6.mlmodel | 12.8 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5n6.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5s6.mlmodel | 48.5 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5s6.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5m6.mlmodel | 137 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5m6.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5l6.mlmodel | 293 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5l6.mlmodel) | [COCO](#trained-dataset-infos) +| yolov5x6.mlmodel | 537 | iOS13 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov5-support/yolov5x6.mlmodel) | [COCO](#trained-dataset-infos) +| YOLOv3.mlmodel | 248.4 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov3-support/YOLOv3.mlmodel) | [COCO](#trained-dataset-infos) +| YOLOv3FP16.mlmodel | 124.2 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov3-support/YOLOv3FP16.mlmodel) | [COCO](#trained-dataset-infos) +| YOLOv3Int8LUT.mlmodel | 62.2 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov3-support/YOLOv3Int8LUT.mlmodel) | [COCO](#trained-dataset-infos) +| YOLOv3Tiny.mlmodel | 35.5 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov3-support/YOLOv3Tiny.mlmodel) | [COCO](#trained-dataset-infos) +| YOLOv3TinyFP16.mlmodel | 17.8 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov3-support/YOLOv3TinyFP16.mlmodel) | [COCO](#trained-dataset-infos) +| YOLOv3TinyInt8LUT.mlmodel | 8.9 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/yolov3-support/YOLOv3TinyInt8LUT.mlmodel) | [COCO](#trained-dataset-infos) +| MobileNetV2_SSDLite.mlmodel | 9.3 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/etc-support/MobileNetV2_SSDLite.mlmodel) | [COCO](#trained-dataset-infos) +| ObjectDetector.mlmodel | 63.7 | iOS12 | [Link](https://github.com/tucan9389/ObjectDetection-CoreML/releases/download/etc-support/ObjectDetector.mlmodel) | [6 Label Dataset](#trained-dataset-infos) + +#### Trained Dataset Infos + +
+ COCO Dataset + +- https://github.com/ultralytics/yolov5/blob/9da6d0f9f5bc37fa386b7b82d2a963f94650949a/data/coco.yaml#L17-L97 + +
+ +
+ 6 Label Dataset(Apple's DEMO) + +- Bagel +- Banana +- Coffee +- Croissant +- Egg +- Waffle + +
+ +### Infernece Time (ms) + +| Model vs. Device | 11
Pro | XS | XS
Max | XR | X | 7+ | 7 | +| :---- | :----: | :----: | :----: | :----: | :----: | :----: | :----: | +| yolov5n | | | | | | | | +| yolov5s | | | | | | | | +| yolov5m | | | | | | | | +| yolov5l | | | | | | | | +| yolov5x | | | | | | | | +| yolov5n6 | | | | | | | | +| yolov5s6 | | | | | | | | +| yolov5m6 | | | | | | | | +| yolov5l6 | | | | | | | | +| yolov5x6 | | | | | | | | +| YOLOv3 | 83 | 108 | 93 | 100 | 356 | 569 | 561 | +| YOLOv3FP16 | 84 | 104 | 89 | 101 | 348 | 572 | 565 | +| YOLOv3Int8LUT | 86 | 101 | 92 | 100 | 337 | 575 | 572 | +| YOLOv3Tiny | 44 | 46 | 41 | 47 | 106 | 165 | 168 | +| YOLOv3TinyFP16 | 44 | 51 | 41 | 44 | 103 | 165 | 167 | +| YOLOv3TinyInt8LUT | 44 | 45 | 39 | 39 | 106 | 160 | 161 | +| MobileNetV2_SSDLite | 18 | 31 | 31 | 31 | 109 | 141 | 134 | +| ObjectDetector | 18 | 24 | 26 | 23 | 63 | 86 | 84 | + +### Total Time (ms) + +| Model vs. Device | 11
Pro | XS | XS
Max | XR | X | 7+ | 7 | +| :---- | :----: | :----: | :----: | :----: | :----: | :----: | :----: | +| yolov5n | | | | | | | | +| yolov5s | | | | | | | | +| yolov5m | | | | | | | | +| yolov5l | | | | | | | | +| yolov5x | | | | | | | | +| yolov5n6 | | | | | | | | +| yolov5s6 | | | | | | | | +| yolov5m6 | | | | | | | | +| yolov5l6 | | | | | | | | +| yolov5x6 | | | | | | | | +| YOLOv3 | 84 | 108 | 93 | 100 | 357 | 569 | 561 | +| YOLOv3FP16 | 85 | 104 | 89 | 101 | 348 | 572 | 565 | +| YOLOv3Int8LUT | 86 | 102 | 92 | 102 | 338 | 576 | 573 | +| YOLOv3Tiny | 45 | 46 | 42 | 48 | 106 | 166 | 169 | +| YOLOv3TinyFP16 | 45 | 51 | 41 | 44 | 104 | 165 | 167 | +| YOLOv3TinyInt8LUT | 45 | 45 | 39 | 40 | 107 | 160 | 161 | +| MobileNetV2_SSDLite | 19 | 32 | 31 | 32 | 109 | 142 | 134 | +| ObjectDetector | 18 | 25 | 26 | 23 | 64 | 87 | 85 | + +### FPS + +| Model vs. Device | 11
Pro | XS | XS
Max | XR | X | 7+ | 7 | +| :---- | :----: | :----: | :----: | :----: | :----: | :----: | :----: | +| yolov5n | | | | | | | | +| yolov5s | | | | | | | | +| yolov5m | | | | | | | | +| yolov5l | | | | | | | | +| yolov5x | | | | | | | | +| yolov5n6 | | | | | | | | +| yolov5s6 | | | | | | | | +| yolov5m6 | | | | | | | | +| yolov5l6 | | | | | | | | +| yolov5x6 | | | | | | | | +| YOLOv3 | 9 | 8 | 10 | 9 | 2 | 1 | 1 | +| YOLOv3FP16 | 9 | 9 | 10 | 8 | 2 | 1 | 1 | +| YOLOv3Int8LUT | 9 | 9 | 10 | 9 | 2 | 1 | 1 | +| YOLOv3Tiny | 14 | 21 | 22 | 20 | 8 | 5 | 5 | +| YOLOv3TinyFP16 | 14 | 19 | 23 | 21 | 9 | 5 | 5 | +| YOLOv3TinyInt8LUT | 14 | 21 | 24 | 23 | 8 | 5 | 5 | +| MobileNetV2_SSDLite | 29 | 23 | 23 | 23 | 8 | 6 | 6 | +| ObjectDetector | 29 | 23 | 23 | 24 | 14 | 10 | 11 | + +### Get your own model + +> Or you can use your own object detection model + +## Build & Run + +### 1. Prerequisites + +#### 1.1 Import object detection model + +![λͺ¨λΈ 뢈러였기.png](https://github.com/tucan9389/MobileNetApp-CoreML/blob/master/resource/%EB%AA%A8%EB%8D%B8%20%EB%B6%88%EB%9F%AC%EC%98%A4%EA%B8%B0.png?raw=true) + +Once you import the model, compiler generates model helper class on build path automatically. You can access the model through model helper class by creating an instance, not through build path. + +#### 1.2 Add permission in info.plist for device's camera access + +image + +### 2. Dependencies + +No external library yet. + +### 3. Code + +#### 3.1 Import Vision framework + +```swift +import Vision +``` + +#### 3.2 Define properties for Core ML + +```swift +class ViewController: UIViewController { + // + // ... + // + + // MARK: - Vision Properties + var request: VNCoreMLRequest? + var visionModel: VNCoreMLModel? + + // + // ... + // +} + +``` + +#### 3.3 Configure and prepare the model + +```swift +// MARK: - Setup Core ML +extension ViewController { + func setupModel() { + if let visionModel = try? VNCoreMLModel(for: objectDectectionModel.model) { + self.visionModel = visionModel + request = VNCoreMLRequest(model: visionModel, completionHandler: visionRequestDidComplete) + request?.imageCropAndScaleOption = .scaleFill + } else { + fatalError("fail to create vision model") + } + } +} +``` + +```swift +// MARK: - Post-processing +extension ViewController { + func visionRequestDidComplete(request: VNRequest, error: Error?) { + if let predictions = request.results as? [VNRecognizedObjectObservation] { + // <# TODO #> + } + } +} +``` + +#### 3.4 Inference πŸƒβ€β™‚οΈ + +```swift +// MARK: - Inference! +extension ViewController { + func predictUsingVision(pixelBuffer: CVPixelBuffer) { + let handler = VNImageRequestHandler(cvPixelBuffer: pixelBuffer) + try? handler.perform([request]) + } +} +``` + +## Performance Test + +(preparing...) + + +## See also + +- [motlabs/awesome-ml-demos-with-ios](https://github.com/motlabs/awesome-ml-demos-with-ios)
+ : The challenge using machine learning model created from tensorflow on iOS +- [Machine Learning - Models - Apple Developer](https://developer.apple.com/machine-learning/models) +- [hollance/coreml-survival-guide](https://github.com/hollance/coreml-survival-guide) +- [vonholst/SSDMobileNet_CoreML](https://github.com/vonholst/SSDMobileNet_CoreML)
+ : iOS project for object detection(SSDMobileNet V1) using Core ML. +- [ultralytics/yolov5](https://github.com/ultralytics/yolov5)
+ : YOLOv5 repository \ No newline at end of file diff --git a/SSDMobileNet-CoreML.xcodeproj/project.xcworkspace/contents.xcworkspacedata b/SSDMobileNet-CoreML.xcodeproj/project.xcworkspace/contents.xcworkspacedata deleted file mode 100644 index b98c72c..0000000 --- a/SSDMobileNet-CoreML.xcodeproj/project.xcworkspace/contents.xcworkspacedata +++ /dev/null @@ -1,7 +0,0 @@ - - - - - diff --git a/SSDMobileNet-CoreML/Base.lproj/Main.storyboard b/SSDMobileNet-CoreML/Base.lproj/Main.storyboard deleted file mode 100644 index 247365f..0000000 --- a/SSDMobileNet-CoreML/Base.lproj/Main.storyboard +++ /dev/null @@ -1,46 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/SSDMobileNet-CoreML/ViewController.swift b/SSDMobileNet-CoreML/ViewController.swift deleted file mode 100644 index 70e2f4d..0000000 --- a/SSDMobileNet-CoreML/ViewController.swift +++ /dev/null @@ -1,81 +0,0 @@ -// -// ViewController.swift -// SSDMobileNet-CoreML -// -// Created by GwakDoyoung on 01/02/2019. -// Copyright Β© 2019 tucan9389. All rights reserved. -// - -import UIKit -import CoreMedia - -class ViewController: UIViewController { - - // MARK: - UI Properties - @IBOutlet weak var videoPreview: UIView! - // @IBOutlet weak var jointView: DrawingJointView! - // @IBOutlet weak var labelsTableView: UITableView! - - // MARK: - AV Property - var videoCapture: VideoCapture! - - // MARK: - View Controller Life Cycle - override func viewDidLoad() { - super.viewDidLoad() - // setup camera - setUpCamera() - } - - override func didReceiveMemoryWarning() { - super.didReceiveMemoryWarning() - } - - override func viewWillAppear(_ animated: Bool) { - super.viewWillAppear(animated) - self.videoCapture.start() - } - - override func viewWillDisappear(_ animated: Bool) { - super.viewWillDisappear(animated) - self.videoCapture.stop() - } - - // MARK: - SetUp Video - func setUpCamera() { - videoCapture = VideoCapture() - videoCapture.delegate = self - videoCapture.fps = 30 - videoCapture.setUp(sessionPreset: .vga640x480) { success in - - if success { - // add preview view on the layer - if let previewLayer = self.videoCapture.previewLayer { - self.videoPreview.layer.addSublayer(previewLayer) - self.resizePreviewLayer() - } - - // start video preview when setup is done - self.videoCapture.start() - } - } - } - - override func viewDidLayoutSubviews() { - super.viewDidLayoutSubviews() - resizePreviewLayer() - } - - func resizePreviewLayer() { - videoCapture.previewLayer?.frame = videoPreview.bounds - } -} - -// MARK: - VideoCaptureDelegate -extension ViewController: VideoCaptureDelegate { - func videoCapture(_ capture: VideoCapture, didCaptureVideoFrame pixelBuffer: CVPixelBuffer?, timestamp: CMTime) { - // the captured image from camera is contained on pixelBuffer - if let pixelBuffer = pixelBuffer { - // <# DO SOMETHING! #> - } - } -}