diff --git a/.env b/.env index a303bb8b9dfd9..127bec1da801a 100644 --- a/.env +++ b/.env @@ -7,3 +7,4 @@ DATABASE_URL=jdbc:postgresql://db:5432/dataline CONFIG_ROOT=/data WORKSPACE_ROOT=/tmp/workspace WORKSPACE_DOCKER_MOUNT=workspace +TRACKING_STRATEGY=segment diff --git a/dataline-analytics/build.gradle b/dataline-analytics/build.gradle new file mode 100644 index 0000000000000..8a217b7289c49 --- /dev/null +++ b/dataline-analytics/build.gradle @@ -0,0 +1,7 @@ +dependencies { + implementation 'com.segment.analytics.java:analytics:2.1.1' + + + implementation project(':dataline-config:models') + implementation project(':dataline-config:persistence') +} diff --git a/dataline-analytics/src/main/java/io/dataline/analytics/LoggingTrackingClient.java b/dataline-analytics/src/main/java/io/dataline/analytics/LoggingTrackingClient.java new file mode 100644 index 0000000000000..4bbe276e57f50 --- /dev/null +++ b/dataline-analytics/src/main/java/io/dataline/analytics/LoggingTrackingClient.java @@ -0,0 +1,58 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class LoggingTrackingClient implements TrackingClient { + + private static final Logger LOGGER = LoggerFactory.getLogger(LoggingTrackingClient.class); + + private final Supplier identitySupplier; + + public LoggingTrackingClient(Supplier identitySupplier) { + this.identitySupplier = identitySupplier; + } + + @Override + public void identify() { + LOGGER.info("identify. userId: {}", identitySupplier.get().getCustomerId()); + } + + @Override + public void track(String action) { + track(action, Collections.emptyMap()); + } + + @Override + public void track(String action, Map metadata) { + LOGGER.info("track. userId: {} action: {}, metadata: {}", identitySupplier.get().getCustomerId(), action, metadata); + } + +} diff --git a/dataline-analytics/src/main/java/io/dataline/analytics/SegmentTrackingClient.java b/dataline-analytics/src/main/java/io/dataline/analytics/SegmentTrackingClient.java new file mode 100644 index 0000000000000..4fb4b732e411c --- /dev/null +++ b/dataline-analytics/src/main/java/io/dataline/analytics/SegmentTrackingClient.java @@ -0,0 +1,81 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.ImmutableMap; +import com.segment.analytics.Analytics; +import com.segment.analytics.messages.IdentifyMessage; +import com.segment.analytics.messages.TrackMessage; +import java.util.Collections; +import java.util.Map; +import java.util.function.Supplier; + +public class SegmentTrackingClient implements TrackingClient { + + private static final String SEGMENT_WRITE_KEY = "7UDdp5K55CyiGgsauOr2pNNujGvmhaeu"; + + // Analytics is threadsafe. + private final Analytics analytics; + private final Supplier identitySupplier; + + @VisibleForTesting + SegmentTrackingClient(Supplier identitySupplier, Analytics analytics) { + this.identitySupplier = identitySupplier; + this.analytics = analytics; + } + + public SegmentTrackingClient(Supplier identitySupplier) { + this.analytics = Analytics.builder(SEGMENT_WRITE_KEY).build(); + this.identitySupplier = identitySupplier; + } + + @Override + public void identify() { + final TrackingIdentity trackingIdentity = identitySupplier.get(); + final ImmutableMap.Builder identityMetadataBuilder = ImmutableMap.builder() + .put("anonymized", trackingIdentity.isAnonymousDataCollection()) + .put("subscribed_newsletter", trackingIdentity.isNews()) + .put("subscribed_security", trackingIdentity.isSecurityUpdates()); + trackingIdentity.getEmail().ifPresent(email -> identityMetadataBuilder.put("email", email)); + + analytics.enqueue(IdentifyMessage.builder() + .userId(trackingIdentity.getCustomerId().toString()) + .traits(identityMetadataBuilder.build())); + } + + @Override + public void track(String action) { + track(action, Collections.emptyMap()); + } + + @Override + public void track(String action, Map metadata) { + analytics.enqueue(TrackMessage.builder(action) + .userId(identitySupplier.get().getCustomerId().toString()) + .properties(metadata)); + } + +} diff --git a/dataline-analytics/src/main/java/io/dataline/analytics/TrackingClient.java b/dataline-analytics/src/main/java/io/dataline/analytics/TrackingClient.java new file mode 100644 index 0000000000000..eba916784f321 --- /dev/null +++ b/dataline-analytics/src/main/java/io/dataline/analytics/TrackingClient.java @@ -0,0 +1,37 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import java.util.Map; + +public interface TrackingClient { + + void identify(); + + void track(String action); + + void track(String action, Map metadata); + +} diff --git a/dataline-analytics/src/main/java/io/dataline/analytics/TrackingClientSingleton.java b/dataline-analytics/src/main/java/io/dataline/analytics/TrackingClientSingleton.java new file mode 100644 index 0000000000000..5e096a0a3d886 --- /dev/null +++ b/dataline-analytics/src/main/java/io/dataline/analytics/TrackingClientSingleton.java @@ -0,0 +1,110 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import com.google.common.annotations.VisibleForTesting; +import io.dataline.config.Configs; +import io.dataline.config.StandardWorkspace; +import io.dataline.config.persistence.ConfigNotFoundException; +import io.dataline.config.persistence.ConfigRepository; +import io.dataline.config.persistence.JsonValidationException; +import io.dataline.config.persistence.PersistenceConstants; +import java.io.IOException; +import java.util.function.Supplier; + +public class TrackingClientSingleton { + + private static final Object lock = new Object(); + private static TrackingClient trackingClient; + + public static TrackingClient get() { + synchronized (lock) { + if (trackingClient == null) { + initialize(); + } + return trackingClient; + } + } + + @VisibleForTesting + static void initialize(TrackingClient trackingClient) { + synchronized (lock) { + TrackingClientSingleton.trackingClient = trackingClient; + } + } + + public static void initialize(Configs.TrackingStrategy trackingStrategy, ConfigRepository configRepository) { + initialize(createTrackingClient(trackingStrategy, () -> getTrackingIdentity(configRepository))); + } + + // fallback on a logging client with an empty identity. + private static void initialize() { + initialize(new LoggingTrackingClient(TrackingIdentity::empty)); + } + + @VisibleForTesting + static TrackingIdentity getTrackingIdentity(ConfigRepository configRepository) { + try { + final StandardWorkspace workspace = configRepository.getStandardWorkspace(PersistenceConstants.DEFAULT_WORKSPACE_ID); + String email = null; + if (workspace.getEmail() != null && workspace.getAnonymousDataCollection() != null && !workspace.getAnonymousDataCollection()) { + email = workspace.getEmail(); + } + return new TrackingIdentity( + workspace.getCustomerId(), + email, + workspace.getAnonymousDataCollection(), + workspace.getNews(), + workspace.getSecurityUpdates()); + } catch (ConfigNotFoundException e) { + throw new RuntimeException("could not find workspace with id: " + PersistenceConstants.DEFAULT_WORKSPACE_ID, e); + } catch (JsonValidationException | IOException e) { + throw new RuntimeException(e); + } + } + + /** + * Creates a tracking client that uses the appropriate strategy from an identity supplier. + * + * @param trackingStrategy - what type of tracker we want to use. + * @param trackingIdentitySupplier - how we get the identity of the user. we have a supplier, + * because we if the identity updates over time (which happens during initial setup), we + * always want the most recent info. + * @return tracking client + */ + @VisibleForTesting + static TrackingClient createTrackingClient(Configs.TrackingStrategy trackingStrategy, Supplier trackingIdentitySupplier) { + + switch (trackingStrategy) { + case SEGMENT: + return new SegmentTrackingClient(trackingIdentitySupplier); + case LOGGING: + return new LoggingTrackingClient(trackingIdentitySupplier); + default: + throw new RuntimeException("unrecognized tracking strategy"); + } + } + +} diff --git a/dataline-analytics/src/main/java/io/dataline/analytics/TrackingIdentity.java b/dataline-analytics/src/main/java/io/dataline/analytics/TrackingIdentity.java new file mode 100644 index 0000000000000..907330c029e71 --- /dev/null +++ b/dataline-analytics/src/main/java/io/dataline/analytics/TrackingIdentity.java @@ -0,0 +1,92 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import java.util.Objects; +import java.util.Optional; +import java.util.UUID; + +public class TrackingIdentity { + + private final UUID customerId; + private final String email; + private final Boolean anonymousDataCollection; + private final Boolean news; + private final Boolean securityUpdates; + + public static TrackingIdentity empty() { + return new TrackingIdentity(null, null, null, null, null); + } + + public TrackingIdentity(UUID customerId, String email, Boolean anonymousDataCollection, Boolean news, Boolean securityUpdates) { + this.customerId = customerId; + this.email = email; + this.anonymousDataCollection = anonymousDataCollection; + this.news = news; + this.securityUpdates = securityUpdates; + } + + public UUID getCustomerId() { + return customerId; + } + + public Optional getEmail() { + return Optional.ofNullable(email); + } + + public boolean isAnonymousDataCollection() { + return anonymousDataCollection != null && anonymousDataCollection; + } + + public boolean isNews() { + return news != null && news; + } + + public boolean isSecurityUpdates() { + return securityUpdates != null && securityUpdates; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + TrackingIdentity that = (TrackingIdentity) o; + return anonymousDataCollection == that.anonymousDataCollection && + news == that.news && + securityUpdates == that.securityUpdates && + Objects.equals(customerId, that.customerId) && + Objects.equals(email, that.email); + } + + @Override + public int hashCode() { + return Objects.hash(customerId, email, anonymousDataCollection, news, securityUpdates); + } + +} diff --git a/dataline-analytics/src/test/java/io/dataline/analytics/SegmentTrackingClientTest.java b/dataline-analytics/src/test/java/io/dataline/analytics/SegmentTrackingClientTest.java new file mode 100644 index 0000000000000..2f6c53b6a33ae --- /dev/null +++ b/dataline-analytics/src/test/java/io/dataline/analytics/SegmentTrackingClientTest.java @@ -0,0 +1,101 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import com.google.common.collect.ImmutableMap; +import com.segment.analytics.Analytics; +import com.segment.analytics.messages.IdentifyMessage; +import com.segment.analytics.messages.TrackMessage; +import java.util.Map; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; + +class SegmentTrackingClientTest { + + private static final TrackingIdentity identity = new TrackingIdentity(UUID.randomUUID(), "a@dataline.io", false, false, true); + + private Analytics analytics; + private SegmentTrackingClient segmentTrackingClient; + + @BeforeEach + void setup() { + analytics = mock(Analytics.class); + segmentTrackingClient = new SegmentTrackingClient(() -> identity, analytics); + } + + @SuppressWarnings("OptionalGetWithoutIsPresent") + @Test + void testIdentify() { + // equals is not defined on MessageBuilder, so we need to use ArgumentCaptor to inspect each field + // manually. + ArgumentCaptor mockBuilder = ArgumentCaptor.forClass(IdentifyMessage.Builder.class); + + segmentTrackingClient.identify(); + + verify(analytics).enqueue(mockBuilder.capture()); + final IdentifyMessage actual = mockBuilder.getValue().build(); + final Map expectedTraits = ImmutableMap.builder() + .put("email", identity.getEmail().get()) + .put("anonymized", identity.isAnonymousDataCollection()) + .put("subscribed_newsletter", identity.isNews()) + .put("subscribed_security", identity.isSecurityUpdates()) + .build(); + assertEquals(identity.getCustomerId().toString(), actual.userId()); + assertEquals(expectedTraits, actual.traits()); + } + + @Test + void testTrack() { + ArgumentCaptor mockBuilder = ArgumentCaptor.forClass(TrackMessage.Builder.class); + + segmentTrackingClient.track("jump"); + + verify(analytics).enqueue(mockBuilder.capture()); + TrackMessage actual = mockBuilder.getValue().build(); + assertEquals("jump", actual.event()); + assertEquals(identity.getCustomerId().toString(), actual.userId()); + } + + @Test + void testTrackWithMetadata() { + ArgumentCaptor mockBuilder = ArgumentCaptor.forClass(TrackMessage.Builder.class); + final ImmutableMap metadata = ImmutableMap.of("height", "80 meters"); + + segmentTrackingClient.track("jump", metadata); + + verify(analytics).enqueue(mockBuilder.capture()); + TrackMessage actual = mockBuilder.getValue().build(); + assertEquals("jump", actual.event()); + assertEquals(identity.getCustomerId().toString(), actual.userId()); + assertEquals(metadata, actual.properties()); + } + +} diff --git a/dataline-analytics/src/test/java/io/dataline/analytics/TrackingClientSingletonTest.java b/dataline-analytics/src/test/java/io/dataline/analytics/TrackingClientSingletonTest.java new file mode 100644 index 0000000000000..3f52b90a23363 --- /dev/null +++ b/dataline-analytics/src/test/java/io/dataline/analytics/TrackingClientSingletonTest.java @@ -0,0 +1,124 @@ +/* + * MIT License + * + * Copyright (c) 2020 Dataline + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package io.dataline.analytics; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import io.dataline.config.Configs; +import io.dataline.config.StandardWorkspace; +import io.dataline.config.persistence.ConfigNotFoundException; +import io.dataline.config.persistence.ConfigRepository; +import io.dataline.config.persistence.JsonValidationException; +import io.dataline.config.persistence.PersistenceConstants; +import java.io.IOException; +import java.util.UUID; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +class TrackingClientSingletonTest { + + private ConfigRepository configRepository; + + @BeforeEach + void setup() { + configRepository = mock(ConfigRepository.class); + // equivalent of resetting TrackingClientSingleton to uninitialized state. + TrackingClientSingleton.initialize(null); + } + + @Test + void testCreateTrackingClientLogging() { + assertTrue( + TrackingClientSingleton.createTrackingClient(Configs.TrackingStrategy.LOGGING, TrackingIdentity::empty) instanceof LoggingTrackingClient); + } + + @Test + void testCreateTrackingClientSegment() { + assertTrue( + TrackingClientSingleton.createTrackingClient(Configs.TrackingStrategy.SEGMENT, TrackingIdentity::empty) instanceof SegmentTrackingClient); + } + + @Test + void testGet() { + TrackingClient client = mock(TrackingClient.class); + TrackingClientSingleton.initialize(client); + assertEquals(client, TrackingClientSingleton.get()); + } + + @Test + void testGetUninitialized() { + assertTrue(TrackingClientSingleton.get() instanceof LoggingTrackingClient); + } + + @Test + void testGetTrackingIdentityInitialSetupNotComplete() throws JsonValidationException, IOException, ConfigNotFoundException { + final StandardWorkspace workspace = new StandardWorkspace().withCustomerId(UUID.randomUUID()); + + when(configRepository.getStandardWorkspace(PersistenceConstants.DEFAULT_WORKSPACE_ID)).thenReturn(workspace); + + final TrackingIdentity actual = TrackingClientSingleton.getTrackingIdentity(configRepository); + final TrackingIdentity expected = new TrackingIdentity(workspace.getCustomerId(), null, null, null, null); + + assertEquals(expected, actual); + } + + @Test + void testGetTrackingIdentityNonAnonymous() throws JsonValidationException, IOException, ConfigNotFoundException { + final StandardWorkspace workspace = new StandardWorkspace() + .withCustomerId(UUID.randomUUID()) + .withEmail("a@dataline.io") + .withAnonymousDataCollection(false) + .withNews(true) + .withSecurityUpdates(true); + + when(configRepository.getStandardWorkspace(PersistenceConstants.DEFAULT_WORKSPACE_ID)).thenReturn(workspace); + + final TrackingIdentity actual = TrackingClientSingleton.getTrackingIdentity(configRepository); + final TrackingIdentity expected = new TrackingIdentity(workspace.getCustomerId(), workspace.getEmail(), false, true, true); + + assertEquals(expected, actual); + } + + @Test + void testGetTrackingIdentityAnonymous() throws JsonValidationException, IOException, ConfigNotFoundException { + final StandardWorkspace workspace = new StandardWorkspace() + .withCustomerId(UUID.randomUUID()) + .withEmail("a@dataline.io") + .withAnonymousDataCollection(true) + .withNews(true) + .withSecurityUpdates(true); + + when(configRepository.getStandardWorkspace(PersistenceConstants.DEFAULT_WORKSPACE_ID)).thenReturn(workspace); + + final TrackingIdentity actual = TrackingClientSingleton.getTrackingIdentity(configRepository); + final TrackingIdentity expected = new TrackingIdentity(workspace.getCustomerId(), null, true, true, true); + + assertEquals(expected, actual); + } + +} diff --git a/dataline-api/src/main/openapi/config.yaml b/dataline-api/src/main/openapi/config.yaml index f4118cae08e20..036a5e6780afe 100644 --- a/dataline-api/src/main/openapi/config.yaml +++ b/dataline-api/src/main/openapi/config.yaml @@ -723,6 +723,9 @@ components: WorkspaceId: type: string format: uuid + CustomerId: + type: string + format: uuid WorkspaceIdRequestBody: type: object required: @@ -734,12 +737,15 @@ components: type: object required: - workspaceId + - customerId - name - slug - initialSetupComplete properties: workspaceId: $ref: "#/components/schemas/WorkspaceId" + customerId: + $ref: "#/components/schemas/CustomerId" name: type: string slug: diff --git a/dataline-config/models/src/main/java/io/dataline/config/Configs.java b/dataline-config/models/src/main/java/io/dataline/config/Configs.java index fd1e532c265ac..a873be1f659d4 100644 --- a/dataline-config/models/src/main/java/io/dataline/config/Configs.java +++ b/dataline-config/models/src/main/java/io/dataline/config/Configs.java @@ -36,4 +36,11 @@ public interface Configs { String getDockerNetwork(); + TrackingStrategy getTrackingStrategy(); + + enum TrackingStrategy { + SEGMENT, + LOGGING + } + } diff --git a/dataline-config/models/src/main/java/io/dataline/config/EnvConfigs.java b/dataline-config/models/src/main/java/io/dataline/config/EnvConfigs.java index 575e5c5f7ae98..25423a58889ca 100644 --- a/dataline-config/models/src/main/java/io/dataline/config/EnvConfigs.java +++ b/dataline-config/models/src/main/java/io/dataline/config/EnvConfigs.java @@ -37,6 +37,7 @@ public class EnvConfigs implements Configs { public static final String WORKSPACE_DOCKER_MOUNT = "WORKSPACE_DOCKER_MOUNT"; public static final String CONFIG_ROOT = "CONFIG_ROOT"; public static final String DOCKER_NETWORK = "DOCKER_NETWORK"; + public static final String TRACKING_STRATEGY = "TRACKING_STRATEGY"; public static final String DEFAULT_NETWORK = "host"; @@ -83,6 +84,22 @@ public String getDockerNetwork() { return DEFAULT_NETWORK; } + @Override + public TrackingStrategy getTrackingStrategy() { + final String trackingStrategy = getEnv.apply(TRACKING_STRATEGY); + if (trackingStrategy == null) { + LOGGER.info("TRACKING_STRATEGY not set, defaulting to " + TrackingStrategy.LOGGING); + return TrackingStrategy.LOGGING; + } + + try { + return TrackingStrategy.valueOf(trackingStrategy.toUpperCase()); + } catch (IllegalArgumentException e) { + LOGGER.info(trackingStrategy + " not recognized, defaulting to " + TrackingStrategy.LOGGING); + return TrackingStrategy.LOGGING; + } + } + private Path getPath(final String name) { final String value = getEnv.apply(name); if (value == null) { diff --git a/dataline-config/models/src/main/resources/json/StandardWorkspace.json b/dataline-config/models/src/main/resources/json/StandardWorkspace.json index 7185c2b2cde7b..991220eaf776c 100644 --- a/dataline-config/models/src/main/resources/json/StandardWorkspace.json +++ b/dataline-config/models/src/main/resources/json/StandardWorkspace.json @@ -11,6 +11,10 @@ "type": "string", "format": "uuid" }, + "customerId": { + "type": "string", + "format": "uuid" + }, "name": { "type": "string" }, diff --git a/dataline-config/models/src/test/java/io/dataline/config/EnvConfigsTest.java b/dataline-config/models/src/test/java/io/dataline/config/EnvConfigsTest.java index eefb36b705662..305d5b54a28e0 100644 --- a/dataline-config/models/src/test/java/io/dataline/config/EnvConfigsTest.java +++ b/dataline-config/models/src/test/java/io/dataline/config/EnvConfigsTest.java @@ -91,4 +91,22 @@ void testDockerNetwork() { Assertions.assertEquals("abc", config.getDockerNetwork()); } + @Test + void testTrackingStrategy() { + when(function.apply(EnvConfigs.TRACKING_STRATEGY)).thenReturn(null); + Assertions.assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); + + when(function.apply(EnvConfigs.TRACKING_STRATEGY)).thenReturn("abc"); + Assertions.assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); + + when(function.apply(EnvConfigs.TRACKING_STRATEGY)).thenReturn("logging"); + Assertions.assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); + + when(function.apply(EnvConfigs.TRACKING_STRATEGY)).thenReturn("segment"); + Assertions.assertEquals(Configs.TrackingStrategy.SEGMENT, config.getTrackingStrategy()); + + when(function.apply(EnvConfigs.TRACKING_STRATEGY)).thenReturn("LOGGING"); + Assertions.assertEquals(Configs.TrackingStrategy.LOGGING, config.getTrackingStrategy()); + } + } diff --git a/dataline-server/build.gradle b/dataline-server/build.gradle index e4ef9ed41bf6c..f187272b7ad53 100644 --- a/dataline-server/build.gradle +++ b/dataline-server/build.gradle @@ -16,6 +16,7 @@ dependencies { implementation group: 'com.networknt', name: 'json-schema-validator', version: '1.0.42' + implementation project(':dataline-analytics') implementation project(':dataline-api') implementation project(':dataline-config:models') implementation project(':dataline-config:persistence') @@ -30,7 +31,7 @@ application { run { // default for running on local machine. - environment "CONFIG_ROOT", rootProject.file('data/config') + environment "CONFIG_ROOT", rootProject.file('data') environment "VERSION", "0.1.0" environment "DATABASE_USER", "postgres" environment "DATABASE_PASSWORD", "" diff --git a/dataline-server/src/main/java/io/dataline/server/ConfigurationApiFactory.java b/dataline-server/src/main/java/io/dataline/server/ConfigurationApiFactory.java index 959239cff5ba4..1eb8036c311a3 100644 --- a/dataline-server/src/main/java/io/dataline/server/ConfigurationApiFactory.java +++ b/dataline-server/src/main/java/io/dataline/server/ConfigurationApiFactory.java @@ -24,18 +24,18 @@ package io.dataline.server; +import io.dataline.config.persistence.ConfigRepository; import io.dataline.server.apis.ConfigurationApi; -import java.nio.file.Path; import org.apache.commons.dbcp2.BasicDataSource; import org.glassfish.hk2.api.Factory; public class ConfigurationApiFactory implements Factory { - private static Path dbRoot; + private static ConfigRepository configRepository; private static BasicDataSource connectionPool; - public static void setConfigPersistenceRoot(final Path dbRoot) { - ConfigurationApiFactory.dbRoot = dbRoot; + public static void setConfigRepository(final ConfigRepository configRepository) { + ConfigurationApiFactory.configRepository = configRepository; } public static void setDbConnectionPool(final BasicDataSource connectionPool) { @@ -44,8 +44,7 @@ public static void setDbConnectionPool(final BasicDataSource connectionPool) { @Override public ConfigurationApi provide() { - return new ConfigurationApi( - ConfigurationApiFactory.dbRoot, ConfigurationApiFactory.connectionPool); + return new ConfigurationApi(ConfigurationApiFactory.configRepository, ConfigurationApiFactory.connectionPool); } @Override diff --git a/dataline-server/src/main/java/io/dataline/server/ServerApp.java b/dataline-server/src/main/java/io/dataline/server/ServerApp.java index f121caf49f983..d0a6aa4255ac7 100644 --- a/dataline-server/src/main/java/io/dataline/server/ServerApp.java +++ b/dataline-server/src/main/java/io/dataline/server/ServerApp.java @@ -24,8 +24,15 @@ package io.dataline.server; +import io.dataline.analytics.TrackingClientSingleton; import io.dataline.config.Configs; import io.dataline.config.EnvConfigs; +import io.dataline.config.StandardWorkspace; +import io.dataline.config.persistence.ConfigNotFoundException; +import io.dataline.config.persistence.ConfigRepository; +import io.dataline.config.persistence.DefaultConfigPersistence; +import io.dataline.config.persistence.JsonValidationException; +import io.dataline.config.persistence.PersistenceConstants; import io.dataline.db.DatabaseHelper; import io.dataline.server.apis.ConfigurationApi; import io.dataline.server.errors.InvalidInputExceptionMapper; @@ -33,7 +40,9 @@ import io.dataline.server.errors.InvalidJsonInputExceptionMapper; import io.dataline.server.errors.KnownExceptionMapper; import io.dataline.server.errors.UncaughtExceptionMapper; +import java.io.IOException; import java.nio.file.Path; +import java.util.UUID; import java.util.logging.Level; import org.apache.commons.dbcp2.BasicDataSource; import org.eclipse.jetty.server.Server; @@ -52,20 +61,23 @@ public class ServerApp { private static final Logger LOGGER = LoggerFactory.getLogger(ServerApp.class); - private final Path configRoot; + private final ConfigRepository configRepository; + private final BasicDataSource connectionPool; - public ServerApp(final Path configRoot) { - this.configRoot = configRoot; + public ServerApp(ConfigRepository configRepository, BasicDataSource connectionPool) { + + this.configRepository = configRepository; + this.connectionPool = connectionPool; } public void start() throws Exception { - BasicDataSource connectionPool = DatabaseHelper.getConnectionPoolFromEnv(); + TrackingClientSingleton.get().identify(); Server server = new Server(8001); ServletContextHandler handler = new ServletContextHandler(); - ConfigurationApiFactory.setConfigPersistenceRoot(configRoot); + ConfigurationApiFactory.setConfigRepository(configRepository); ConfigurationApiFactory.setDbConnectionPool(connectionPool); ResourceConfig rc = @@ -115,14 +127,43 @@ public void configure() { server.join(); } + private static void setCustomerIdIfNotSet(final ConfigRepository configRepository) { + final StandardWorkspace workspace; + try { + workspace = configRepository.getStandardWorkspace(PersistenceConstants.DEFAULT_WORKSPACE_ID); + + if (workspace.getCustomerId() == null) { + final UUID customerId = UUID.randomUUID(); + LOGGER.info("customerId not set for workspace. Setting it to " + customerId); + workspace.setCustomerId(customerId); + + configRepository.writeStandardWorkspace(workspace); + } + } catch (ConfigNotFoundException e) { + throw new RuntimeException("could not find workspace with id: " + PersistenceConstants.DEFAULT_WORKSPACE_ID, e); + } catch (JsonValidationException | IOException e) { + throw new RuntimeException(e); + } + } + public static void main(String[] args) throws Exception { final Configs configs = new EnvConfigs(); final Path configRoot = configs.getConfigRoot(); LOGGER.info("configRoot = " + configRoot); + final ConfigRepository configRepository = new ConfigRepository(new DefaultConfigPersistence(configRoot)); + + // hack: upon installation we need to assign a random customerId so that when + // tracking we can associate all action with the correct anonymous id. + setCustomerIdIfNotSet(configRepository); + + TrackingClientSingleton.initialize(configs.getTrackingStrategy(), configRepository); + + BasicDataSource connectionPool = DatabaseHelper.getConnectionPoolFromEnv(); + LOGGER.info("Starting server..."); - new ServerApp(configRoot).start(); + new ServerApp(configRepository, connectionPool).start(); } } diff --git a/dataline-server/src/main/java/io/dataline/server/apis/ConfigurationApi.java b/dataline-server/src/main/java/io/dataline/server/apis/ConfigurationApi.java index 76c42329c6791..24ce45129e9a1 100644 --- a/dataline-server/src/main/java/io/dataline/server/apis/ConfigurationApi.java +++ b/dataline-server/src/main/java/io/dataline/server/apis/ConfigurationApi.java @@ -62,7 +62,6 @@ import io.dataline.api.model.WorkspaceUpdate; import io.dataline.config.persistence.ConfigNotFoundException; import io.dataline.config.persistence.ConfigRepository; -import io.dataline.config.persistence.DefaultConfigPersistence; import io.dataline.config.persistence.JsonValidationException; import io.dataline.scheduler.persistence.DefaultSchedulerPersistence; import io.dataline.scheduler.persistence.SchedulerPersistence; @@ -80,7 +79,6 @@ import io.dataline.server.handlers.WorkspacesHandler; import io.dataline.server.validation.IntegrationSchemaValidation; import java.io.IOException; -import java.nio.file.Path; import javax.validation.Valid; import org.apache.commons.dbcp2.BasicDataSource; import org.eclipse.jetty.http.HttpStatus; @@ -100,8 +98,7 @@ public class ConfigurationApi implements io.dataline.api.V1Api { private final JobHistoryHandler jobHistoryHandler; private final WebBackendConnectionsHandler webBackendConnectionsHandler; - public ConfigurationApi(final Path dbRoot, BasicDataSource connectionPool) { - final ConfigRepository configRepository = new ConfigRepository(new DefaultConfigPersistence(dbRoot)); + public ConfigurationApi(final ConfigRepository configRepository, BasicDataSource connectionPool) { final IntegrationSchemaValidation integrationSchemaValidation = new IntegrationSchemaValidation(); workspacesHandler = new WorkspacesHandler(configRepository); sourcesHandler = new SourcesHandler(configRepository); diff --git a/dataline-server/src/main/java/io/dataline/server/handlers/SchedulerHandler.java b/dataline-server/src/main/java/io/dataline/server/handlers/SchedulerHandler.java index 92c7808ec6810..d1bf5bc17a68e 100644 --- a/dataline-server/src/main/java/io/dataline/server/handlers/SchedulerHandler.java +++ b/dataline-server/src/main/java/io/dataline/server/handlers/SchedulerHandler.java @@ -24,6 +24,8 @@ package io.dataline.server.handlers; +import com.google.common.collect.ImmutableMap; +import io.dataline.analytics.TrackingClientSingleton; import io.dataline.api.model.CheckConnectionRead; import io.dataline.api.model.ConnectionIdRequestBody; import io.dataline.api.model.ConnectionSyncRead; @@ -69,7 +71,18 @@ public CheckConnectionRead checkSourceImplementationConnection(SourceImplementat final long jobId = schedulerPersistence.createSourceCheckConnectionJob(connectionImplementation); LOGGER.debug("jobId = " + jobId); - return reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); + final CheckConnectionRead checkConnectionRead = reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); + + TrackingClientSingleton.get().track("check_connection", ImmutableMap.builder() + .put("type", "source") + .put("name", connectionImplementation.getName()) + .put("source_specification_id", connectionImplementation.getSourceSpecificationId()) + .put("source_implementation_id", connectionImplementation.getSourceImplementationId()) + .put("check_connection_result", checkConnectionRead.getStatus()) + .put("job_id", jobId) + .build()); + + return checkConnectionRead; } public CheckConnectionRead checkDestinationImplementationConnection(DestinationImplementationIdRequestBody destinationImplementationIdRequestBody) @@ -79,7 +92,18 @@ public CheckConnectionRead checkDestinationImplementationConnection(DestinationI final long jobId = schedulerPersistence.createDestinationCheckConnectionJob(connectionImplementation); LOGGER.debug("jobId = " + jobId); - return reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); + final CheckConnectionRead checkConnectionRead = reportConnectionStatus(waitUntilJobIsTerminalOrTimeout(jobId)); + + TrackingClientSingleton.get().track("check_connection", ImmutableMap.builder() + .put("type", "destination") + .put("name", connectionImplementation.getName()) + .put("destination_specification_id", connectionImplementation.getDestinationSpecificationId()) + .put("destination_implementation_id", connectionImplementation.getDestinationImplementationId()) + .put("check_connection_result", checkConnectionRead.getStatus()) + .put("job_id", jobId) + .build()); + + return checkConnectionRead; } public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementation(SourceImplementationIdRequestBody sourceImplementationIdRequestBody) @@ -98,8 +122,14 @@ public SourceImplementationDiscoverSchemaRead discoverSchemaForSourceImplementat LOGGER.debug("output = " + output); - return new SourceImplementationDiscoverSchemaRead() - .schema(SchemaConverter.toApiSchema(output.getSchema())); + TrackingClientSingleton.get().track("discover_schema", ImmutableMap.builder() + .put("name", connectionImplementation.getName()) + .put("source_specification_id", connectionImplementation.getSourceSpecificationId()) + .put("source_implementation_id", connectionImplementation.getSourceImplementationId()) + .put("job_id", jobId) + .build()); + + return new SourceImplementationDiscoverSchemaRead().schema(SchemaConverter.toApiSchema(output.getSchema())); } public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectionIdRequestBody) @@ -115,6 +145,17 @@ public ConnectionSyncRead syncConnection(final ConnectionIdRequestBody connectio final long jobId = schedulerPersistence.createSyncJob(sourceConnectionImplementation, destinationConnectionImplementation, standardSync); final Job job = waitUntilJobIsTerminalOrTimeout(jobId); + TrackingClientSingleton.get().track("sync", ImmutableMap.builder() + .put("name", standardSync.getName()) + .put("connection_id", standardSync.getConnectionId()) + .put("sync_mode", standardSync.getSyncMode()) + .put("source_specification_id", sourceConnectionImplementation.getSourceSpecificationId()) + .put("source_implementation_id", sourceConnectionImplementation.getSourceImplementationId()) + .put("destination_specification_id", destinationConnectionImplementation.getDestinationSpecificationId()) + .put("destination_implementation_id", destinationConnectionImplementation.getDestinationImplementationId()) + .put("job_id", jobId) + .build()); + return new ConnectionSyncRead() .status(job.getStatus().equals(JobStatus.COMPLETED) ? ConnectionSyncRead.StatusEnum.SUCCESS : ConnectionSyncRead.StatusEnum.FAIL); } diff --git a/dataline-server/src/main/java/io/dataline/server/handlers/WorkspacesHandler.java b/dataline-server/src/main/java/io/dataline/server/handlers/WorkspacesHandler.java index b5e52c5045532..2f4a47f2cc8bb 100644 --- a/dataline-server/src/main/java/io/dataline/server/handlers/WorkspacesHandler.java +++ b/dataline-server/src/main/java/io/dataline/server/handlers/WorkspacesHandler.java @@ -24,6 +24,7 @@ package io.dataline.server.handlers; +import io.dataline.analytics.TrackingClientSingleton; import io.dataline.api.model.SlugRequestBody; import io.dataline.api.model.WorkspaceIdRequestBody; import io.dataline.api.model.WorkspaceRead; @@ -72,6 +73,9 @@ public WorkspaceRead updateWorkspace(WorkspaceUpdate workspaceUpdate) configRepository.writeStandardWorkspace(persistedWorkspace); + // after updating email or tracking info, we need to re-identify the instance. + TrackingClientSingleton.get().identify(); + return buildWorkspaceReadFromId(workspaceUpdate.getWorkspaceId()); } @@ -81,6 +85,7 @@ private WorkspaceRead buildWorkspaceReadFromId(UUID workspaceId) return new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) + .customerId(workspace.getCustomerId()) .name(workspace.getName()) .slug(workspace.getSlug()) .initialSetupComplete(workspace.getInitialSetupComplete()); diff --git a/dataline-server/src/test/java/io/dataline/server/handlers/WorkspacesHandlerTest.java b/dataline-server/src/test/java/io/dataline/server/handlers/WorkspacesHandlerTest.java index dcc0031032e6a..5b5da894ba896 100644 --- a/dataline-server/src/test/java/io/dataline/server/handlers/WorkspacesHandlerTest.java +++ b/dataline-server/src/test/java/io/dataline/server/handlers/WorkspacesHandlerTest.java @@ -61,6 +61,7 @@ private StandardWorkspace generateWorkspace() { return new StandardWorkspace() .withWorkspaceId(workspaceId) + .withCustomerId(UUID.randomUUID()) .withEmail("test@dataline.io") .withName("test workspace") .withSlug("default") @@ -76,6 +77,7 @@ void testGetWorkspace() throws JsonValidationException, ConfigNotFoundException, final WorkspaceRead workspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) + .customerId(workspace.getCustomerId()) .name("test workspace") .slug("default") .initialSetupComplete(false); @@ -92,6 +94,7 @@ void testGetWorkspaceBySlug() throws JsonValidationException, ConfigNotFoundExce final WorkspaceRead workspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) + .customerId(workspace.getCustomerId()) .name("test workspace") .slug("default") .initialSetupComplete(false); @@ -111,6 +114,7 @@ void testUpdateWorkspace() throws JsonValidationException, ConfigNotFoundExcepti final StandardWorkspace expectedWorkspace = new StandardWorkspace() .withWorkspaceId(workspace.getWorkspaceId()) + .withCustomerId(workspace.getCustomerId()) .withEmail("test@dataline.io") .withName("test workspace") .withSlug("default") @@ -127,6 +131,7 @@ void testUpdateWorkspace() throws JsonValidationException, ConfigNotFoundExcepti final WorkspaceRead expectedWorkspaceRead = new WorkspaceRead() .workspaceId(workspace.getWorkspaceId()) + .customerId(workspace.getCustomerId()) .name("test workspace") .slug("default") .initialSetupComplete(true); diff --git a/settings.gradle b/settings.gradle index ed2d5fbd5252e..49af9b028a997 100644 --- a/settings.gradle +++ b/settings.gradle @@ -2,6 +2,7 @@ import groovy.io.FileType rootProject.name = 'dataline' +include 'dataline-analytics' include 'dataline-api' include 'dataline-commons' include ':dataline-config:models'