diff --git a/assemblies/plugins/tech/google/pom.xml b/assemblies/plugins/tech/google/pom.xml index a3b1fb09ab7..016c1e6c875 100644 --- a/assemblies/plugins/tech/google/pom.xml +++ b/assemblies/plugins/tech/google/pom.xml @@ -32,9 +32,10 @@ + 2.27.0 2.9.0 - v4-rev612-1.25.0 - v3-rev197-1.25.0 + v4-rev20230526-2.0.0 + v3-rev20230610-2.0.0 v3-rev169-1.25.0 v3-rev197-1.25.0 6.1.26 @@ -51,12 +52,17 @@ com.google.analytics google-analytics-data - 0.31.0 + 0.28.0 com.google.api.grpc proto-google-analytics-data-v1beta - 0.31.0 + 0.28.0 + + + com.google.protobuf + protobuf-java + 3.23.3 com.google.protobuf @@ -90,20 +96,10 @@ gax-grpc ${gax.version} - - com.google.protobuf - protobuf-java-util - 3.21.9 - - - com.google.protobuf - protobuf-java - 3.21.12 - com.google.api.grpc proto-google-common-protos - 2.14.0 + 2.21.0 joda-time @@ -133,23 +129,6 @@ google-analytics-data 0.28.0 - - com.google.protobuf - protobuf-java-util - 3.23.3 - - com.google.oauth-client google-oauth-client diff --git a/assemblies/plugins/tech/google/src/assembly/assembly.xml b/assemblies/plugins/tech/google/src/assembly/assembly.xml index 95c9db26533..3e000037711 100644 --- a/assemblies/plugins/tech/google/src/assembly/assembly.xml +++ b/assemblies/plugins/tech/google/src/assembly/assembly.xml @@ -85,7 +85,6 @@ com.google.code.gson:gson com.google.analytics:google-analytics-data:jar com.google.api.grpc:proto-google-analytics-data-v1beta:jar - com.google.protobuf:protobuf-java-util:jar diff --git a/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-analytics.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-analytics.svg new file mode 100644 index 00000000000..2d71695dd99 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-analytics.svg @@ -0,0 +1,4 @@ + + + + diff --git a/plugins/tech/google/src/main/resources/GoogleSheetsInput.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-sheets-input.svg similarity index 100% rename from plugins/tech/google/src/main/resources/GoogleSheetsInput.svg rename to docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-sheets-input.svg diff --git a/plugins/tech/google/src/main/resources/GoogleSheetsOutput.svg b/docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-sheets-output.svg similarity index 100% rename from plugins/tech/google/src/main/resources/GoogleSheetsOutput.svg rename to docs/hop-user-manual/modules/ROOT/assets/images/transforms/icons/google-sheets-output.svg diff --git a/docs/hop-user-manual/modules/ROOT/nav.adoc b/docs/hop-user-manual/modules/ROOT/nav.adoc index fa0257de1ca..e3a12444f40 100644 --- a/docs/hop-user-manual/modules/ROOT/nav.adoc +++ b/docs/hop-user-manual/modules/ROOT/nav.adoc @@ -150,6 +150,9 @@ under the License. *** xref:pipeline/transforms/getsystemdata.adoc[Get System Info] *** xref:pipeline/transforms/gettablenames.adoc[Get table names] *** xref:pipeline/transforms/getvariable.adoc[Get variables] +*** xref:pipeline/transforms/google-analytics.adoc[Google Analytics Input] +*** xref:pipeline/transforms/google-sheets-input.adoc[Google Sheets Input] +*** xref:pipeline/transforms/google-sheets-output.adoc[Google Sheets Output] *** xref:pipeline/transforms/groupby.adoc[Group By] *** xref:pipeline/transforms/http.adoc[HTTP client] *** xref:pipeline/transforms/httppost.adoc[HTTP Post] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc index 026fd3f9061..bd7dd26240c 100644 --- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms.adoc @@ -103,6 +103,9 @@ The pages nested under this topic contain information on how to use the transfor * xref:pipeline/transforms/getsystemdata.adoc[Get System Info] * xref:pipeline/transforms/gettablenames.adoc[Get table names] * xref:pipeline/transforms/getvariable.adoc[Get variables] +* xref:pipeline/transforms/google-analytics.adoc[Google Analytics Input] +* xref:pipeline/transforms/google-sheets-input.adoc[Google Sheets Input] +* xref:pipeline/transforms/google-sheets-output.adoc[Google Sheets Output] * xref:pipeline/transforms/groupby.adoc[Group By] * xref:pipeline/transforms/http.adoc[HTTP client] * xref:pipeline/transforms/httppost.adoc[HTTP Post] diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc new file mode 100644 index 00000000000..9d0191aa9b1 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-analytics.adoc @@ -0,0 +1,84 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The Google Analytics Input transform reads data from a Google Analytics 4 account. + += image:transforms/icons/google-analytics.svg[Google Analytics Input transform Icon, role="image-doc-icon"] Google Analytics Input + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The Google Analytics Input transform reads data from a Google Analytics 4 account, using the Google Analytics Data API. + +The https://ga-dev-tools.google/ga4/query-explorer/[GA4 Query Explorer] provides a dedicated website to developing and testing queries with the Google Analytics API (outside of Apache Hop). + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:question_mark.svg[Maybe Supported, 24] +!Flink! image:question_mark.svg[Maybe Supported, 24] +!Dataflow! image:question_mark.svg[Maybe Supported, 24] +!=== +|=== + + +== Configuration + +* You need a Google Analytics account with access and sufficient permissions to a GA4 property. +* You need a Google Cloud project with a service account. The Google Analytics api needs to be enabled for this project. + +Check the Google Cloud docs for more information on how the https://cloud.google.com/iam/docs/service-accounts-create[create a service account^] and to https://developers.google.com/analytics/devguides/reporting/data/v1/quickstart-client-libraries[set up a project^] for use with the Google Analytics Data API. + + +== Options + +=== Google Analytics Connection Settings + +[%header, width="90%"] +|=== +|Option|Description +|Application name|Enter an application name such as "Apache Hop" or leave the default. +|OAuth service email|Your Google Developer Service Account's Email Address (e.g. ""). +|Key file|The path to the P12 private key associated with your OAuth Service account. +|Property Id|the GA4 property to read from +|=== + +=== Query Definition + +[%header, width="90%"] +|=== +|Option|Description +|Start date (YYYY-MM-DD|Specifies the start date associated with the query. The date must be entered in the following format: YYYY-MM-DD. +|End date (YYYY-MM-DD|Specifies the end date associated with the query. The date must be entered in the following format: YYYY-MM-DD. +|Dimension|Specifies the dimension fields for which you want to query. The Google Analytics Data API documentation provides you with a list of valid inputs and metrics that can be combined. +|Metrics|Specifies the metrics fields you want returned. At least one metric must be provided. +|Sort|Specifies a dimension field on which to sort. +|=== + +=== Fields + +Click **Get Fields** to retrieve a list of possible fields based on the query you defined. +Click **Preview** to preview data based on the defined query. + +=== Limit Size + +Limit the number of rows to retrieve for the specified GA4 property to a number of rows. \ No newline at end of file diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-sheets-input.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-sheets-input.adoc new file mode 100644 index 00000000000..ede552409dd --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-sheets-input.adoc @@ -0,0 +1,62 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The Google Sheets Input transform reads data from a Google Sheets worksheets. + += image:transforms/icons/google-sheets-input.svg[Google Sheets Input transform Icon, role="image-doc-icon"] Google Sheets Input + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The Google Sheets Input transform reads data from a Google Sheets worksheets. + +This transform requires a Google service account (JSON file) and a Google Cloud project with the Sheets API enabled. + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:question_mark.svg[Maybe Supported, 24] +!Flink! image:question_mark.svg[Maybe Supported, 24] +!Dataflow! image:question_mark.svg[Maybe Supported, 24] +!=== +|=== + +== Options + +=== Service account tab + +Click the `browse` button to pick your google service account client secret json file. + +The `Test` button Lets you test your access to the API. On success, you should see a success message. + +=== Spreadsheet tab + +Lets you specify or browse for spreadsheets existing in the service account drive or for the ones that are shared with the service account email. + +=== Fields tab + +Lets you select from the fields of the sheet. + +Fields name are always defined in the first line of the google spreadsheet. + +**Get Fields** lets you get fields and guess their types, format, precision, decimal and group delimiter as well as trim type. + diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-sheets-output.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-sheets-output.adoc new file mode 100644 index 00000000000..5e2018fa3c9 --- /dev/null +++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/google-sheets-output.adoc @@ -0,0 +1,63 @@ +//// +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + http://www.apache.org/licenses/LICENSE-2.0 +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +//// +:documentationPath: /pipeline/transforms/ +:language: en_US +:description: The Google Sheets Output transform writes data to a Google Sheets worksheets. + += image:transforms/icons/google-sheets-output.svg[Google Sheets Output transform Icon, role="image-doc-icon"] Google Sheets Output + +[%noheader,cols="3a,1a", role="table-no-borders" ] +|=== +| +== Description + +The Google Sheets Output transform writes data to a Google Sheets worksheets. + +This transform requires a Google service account (JSON file) and a Google Cloud project with the Sheets API enabled. + +| +== Supported Engines +[%noheader,cols="2,1a",frame=none, role="table-supported-engines"] +!=== +!Hop Engine! image:check_mark.svg[Supported, 24] +!Spark! image:question_mark.svg[Maybe Supported, 24] +!Flink! image:question_mark.svg[Maybe Supported, 24] +!Dataflow! image:question_mark.svg[Maybe Supported, 24] +!=== +|=== + +== Options + +=== Service account tab + +Click the `browse` button to pick your google service account client secret json file. + +The `Test` button Lets you test your access to the API. On success, you should see a success message. + +=== Spreadsheet tab + +[%header, width="90%"] +|=== +|Option|Description +|Spreadsheet key|Lets you specify or browse for spreadsheets existing in the service account drive or for the ones that are shared with the service account email. If you type in a sheet name (that does not exist in the drive) it will attempt to create a sheet it the "create" checkbox is ticked. +|Worksheet Id|Should be browsed form the selected spreadsheet key. If you want to create a new file, type in any key that will become the name of the worksheet in the created spreadsheet +|Append to sheet|Appends the lines **without the header** to an existing spreadsheet. This is incompatible with the create option below. +|Create new sheet if it does not exist|If the checkbox is checked then if the Spreadsheet Key specified in the field Spreadsheet key does not exist it will create a new spreadsheet within the service account drive (note that this account has no UI) +|Share Full Rights (RW) with|the Share with user email field lets you specify the email of a user who will get full rights on the freshly created file. +|Domain Wide Permission|Enables to share with your whole domain (if configured in Google Drive). +|=== + diff --git a/plugins/tech/google/pom.xml b/plugins/tech/google/pom.xml index c55e67502f3..faac28f3dca 100755 --- a/plugins/tech/google/pom.xml +++ b/plugins/tech/google/pom.xml @@ -48,10 +48,10 @@ --> 2.9.0 - v4-rev612-1.25.0 - v3-rev197-1.25.0 - - v3-rev197-1.25.0 + v4-rev20230526-2.0.0 + + v3-rev20230610-2.0.0 + 6.1.26 @@ -97,7 +97,6 @@ - com.google.oauth-client google-oauth-client-jetty @@ -115,7 +114,6 @@ - com.google.oauth-client google-oauth-client-java6 @@ -133,7 +131,6 @@ - com.google.cloud google-cloud-storage @@ -141,7 +138,6 @@ - junit junit diff --git a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java index ca44afe0403..8c0a15a547c 100644 --- a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java +++ b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalytics.java @@ -23,7 +23,6 @@ import com.google.analytics.data.v1beta.Dimension; import com.google.analytics.data.v1beta.DimensionHeader; import com.google.analytics.data.v1beta.Metric; -import com.google.analytics.data.v1beta.MetricAggregation; import com.google.analytics.data.v1beta.MetricHeader; import com.google.analytics.data.v1beta.MetricType; import com.google.analytics.data.v1beta.Row; @@ -32,8 +31,6 @@ import com.google.api.gax.core.FixedCredentialsProvider; import com.google.auth.Credentials; import com.google.auth.oauth2.ServiceAccountCredentials; -import com.google.protobuf.Struct; -import com.google.protobuf.util.JsonFormat; import org.apache.hop.core.exception.HopException; import org.apache.hop.core.row.IValueMeta; import org.apache.hop.core.row.RowDataUtil; @@ -109,25 +106,16 @@ public boolean init(){ } private RunReportRequest getRequest(){ -// try { - -// Struct.Builder structBuilder = Struct.newBuilder(); -// JsonFormat.parser().merge(meta.getDimensionFilters(), structBuilder); return RunReportRequest.newBuilder() .setProperty("properties/" + meta.getGaProperty()) .addAllDimensions(dimensionList) .addAllMetrics(metricList) -// .setMetricFilter(FilterExpression.parseFrom(new ByteArrayInputStream(meta.getMetricFilters().getBytes()))) -// .setDimensionFilter(FilterExpression.parseFrom(IOUtils.toInputStream(meta.getDimensionFilters(), "UTF-8"))) // .setMetricAggregations(0, MetricAggregation.valueOf(MetricAggregation.COUNT_VALUE)) .addDateRanges(DateRange.newBuilder().setStartDate(meta.getStartDate()).setEndDate(meta.getEndDate())) .setLimit(REQUEST_ROW_SIZE) .setOffset(requestOffset) .build(); -// } catch (IOException e) { -// throw new RuntimeException(e); -// } } @Override diff --git a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java index 5d13ab2f3b8..b3de34e3fec 100644 --- a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java +++ b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsDialog.java @@ -94,8 +94,6 @@ public class GoogleAnalyticsDialog extends BaseTransformDialog implements ITrans private TextVar wQuMetrics; -// private TextVar wQuDimFilters; -// private TextVar wQuMetricFilters; private TextVar wQuMetricAggs; private TextVar wQuSort; @@ -114,17 +112,17 @@ public class GoogleAnalyticsDialog extends BaseTransformDialog implements ITrans private ModifyListener lsMod; static final String REFERENCE_SORT_URI = - "https://developers.google.com/analytics/devguides/reporting/core/v3/reference#sort"; + "https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/OrderBy"; static final String REFERENCE_METRICS_URI = "https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#metrics"; static final String REFERENCE_DIMENSIONS_URI = "https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#dimensions"; - static final String REFERENCE_FILTERS_URI = - "https://developers.google.com/analytics/devguides/reporting/core/v3/reference#filters"; +// static final String REFERENCE_FILTERS_URI = +// "https://developers.google.com/analytics/devguides/reporting/core/v3/reference#filters"; static final String REFERENCE_DIMENSION_AND_METRIC_URI = - "https://developers.google.com/analytics/devguides/reporting/core/v3/"; + "https://support.google.com/analytics/answer/9143382?hl=en"; static final String REFERENCE_METRICAGGS_URI = - "https://developers.google.com/analytics/devguides/reporting/core/v3/reference#filters"; + "https://developers.google.com/analytics/devguides/reporting/data/v1/rest/v1beta/MetricAggregation"; public GoogleAnalyticsDialog( Shell parent, IVariables variables, Object in, PipelineMeta pipelineMeta, String sname) { @@ -452,7 +450,7 @@ public String open() { BaseMessages.getString(PKG, "GoogleAnalyticsDialog.Query.Reference.Label")); props.setLook(wQuMetricAggsReference); wQuMetricAggsReference.addListener( - SWT.Selection, ev -> BareBonesBrowserLaunch.openURL(REFERENCE_FILTERS_URI)); + SWT.Selection, ev -> BareBonesBrowserLaunch.openURL(REFERENCE_METRICAGGS_URI)); wQuMetricAggsReference.pack(true); diff --git a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsMeta.java b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsMeta.java index 7b4690d6935..8b7e43dfb24 100644 --- a/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsMeta.java +++ b/plugins/tech/google/src/main/java/org/apache/hop/pipeline/transforms/googleanalytics/GoogleAnalyticsMeta.java @@ -98,18 +98,6 @@ public class GoogleAnalyticsMeta extends BaseTransformMeta + + + + + + + + \ No newline at end of file diff --git a/plugins/tech/google/src/main/resources/google-sheets-output.svg b/plugins/tech/google/src/main/resources/google-sheets-output.svg new file mode 100644 index 00000000000..459ff692ec7 --- /dev/null +++ b/plugins/tech/google/src/main/resources/google-sheets-output.svg @@ -0,0 +1,37 @@ + + + + + + + + + \ No newline at end of file diff --git a/pom.xml b/pom.xml index 36a5fab3112..6cfa10b2f57 100644 --- a/pom.xml +++ b/pom.xml @@ -126,7 +126,7 @@ 1.1.3 5.4.0 1.2 - 2.27.0 + 2.19.6 2.13.0 2.2.0 v1-rev20220705-2.0.0