From 757abe57c991db0f11dcec41d99e7029bb2bd7c6 Mon Sep 17 00:00:00 2001 From: pj-cegeka Date: Fri, 3 Nov 2023 20:37:36 +0100 Subject: [PATCH 1/3] feat: add crowdscan --- .../infra/TripleRepositoryRDF4JImpl.java | 14 ++ .../resources/crowdscan/crowdscan-locaties.nq | 2 + crowdscan-locations.config.yml | 43 +++++ demonstrator.env | 4 + docker-compose.yml | 24 ++- frontend/streams.json | 5 + gipod.config.yml | 151 ++++++++++++------ jsonld/count.jsonld | 14 ++ jsonld/map.jsonld | 11 ++ sparql/count.to-observation.rq | 50 ++++++ sparql/crowdscan.add-query.rq | 30 ++++ sparql/map.to-zone.rq | 53 ++++++ sparql/observation.to-oslo.rq | 26 +++ sparql/zone.to-oslo.rq | 60 +++++++ 14 files changed, 440 insertions(+), 47 deletions(-) create mode 100644 backend/src/main/resources/crowdscan/crowdscan-locaties.nq create mode 100644 crowdscan-locations.config.yml create mode 100644 jsonld/count.jsonld create mode 100644 jsonld/map.jsonld create mode 100644 sparql/count.to-observation.rq create mode 100644 sparql/crowdscan.add-query.rq create mode 100644 sparql/map.to-zone.rq create mode 100644 sparql/observation.to-oslo.rq create mode 100644 sparql/zone.to-oslo.rq diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java index 492ca78..0039d9a 100644 --- a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/triple/infra/TripleRepositoryRDF4JImpl.java @@ -50,6 +50,20 @@ protected void initRepo(RepositoryManager repositoryManager) { } catch (Exception e) { log.error("Could not create repository. Reason: {}", e.getMessage()); } + // possibly temp solution to separate locations from other members + try { + String locationsRepoId = "locations"; + if(!repositoryManager.hasRepositoryConfig(locationsRepoId)) { + String indeces = "spoc,cspo"; + NativeStoreConfig storeConfig = new NativeStoreConfig(indeces); + RepositoryImplConfig repositoryImplConfig = new SailRepositoryConfig(storeConfig); + RepositoryConfig config = new RepositoryConfig(locationsRepoId, repositoryImplConfig); + repositoryManager.addRepositoryConfig(config); + log.info("Created repository with id: {}", locationsRepoId); + } + } catch (Exception e) { + log.error("Could not create repository. Reason: {}", e.getMessage()); + } } @Override diff --git a/backend/src/main/resources/crowdscan/crowdscan-locaties.nq b/backend/src/main/resources/crowdscan/crowdscan-locaties.nq new file mode 100644 index 0000000..4851cc4 --- /dev/null +++ b/backend/src/main/resources/crowdscan/crowdscan-locaties.nq @@ -0,0 +1,2 @@ + . + "${geometry}"^^ . diff --git a/crowdscan-locations.config.yml b/crowdscan-locations.config.yml new file mode 100644 index 0000000..49d73e2 --- /dev/null +++ b/crowdscan-locations.config.yml @@ -0,0 +1,43 @@ +server: + port: 8085 +orchestrator: + pipelines: + - name: map-pipeline + description: "HTTP polling, OSLO transformation, version creation & HTTP sending." + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + config: + url: https://test.crowdscan.be/dataapi/antwerpen/beacon/map + interval: P1D + adapter: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter + config: + core-context: file:///ldio/jsonld/map.jsonld + transformers: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/map.to-zone.rq + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/zone.to-oslo.rq + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator + config: + member-type: http://def.isotc211.org/iso19156/2011/SamplingSurface#SF_SamplingSurface + delimiter: "/" + date-observed-property: http://purl.org/dc/terms/modified + generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime + versionOf-property: http://purl.org/dc/terms/isVersionOf +# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser +# config: +# versionOf-property: http://purl.org/dc/terms/isVersionOf + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: locations + named-graph: http://crowdscan-locations +# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut +# config: +# endpoint: https://webhook.site/51437bbb-a96c-4900-9535-836d6707aa83 +# content-type: application/n-quads \ No newline at end of file diff --git a/demonstrator.env b/demonstrator.env index 0fcc799..5af8dc9 100644 --- a/demonstrator.env +++ b/demonstrator.env @@ -22,6 +22,10 @@ LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_FULLNAME=http://schema.org/name #LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_CAPACITY=http://schema.mobivoc.org/#totalCapacity LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_AVAILABLE=http://schema.mobivoc.org/#currentValue #LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_USED=https://w3id.org/gbfs#bikes_in_use +LDES_STREAMS_CROWDSCAN_MEMBERTYPE=http://def.isotc211.org/iso19156/2011/Observation#OM_Observation +LDES_STREAMS_CROWDSCAN_TIMESTAMPPATH=http://www.w3.org/ns/prov#generatedAtTime +LDES_STREAMS_CROWDSCAN_VERSIONOFPATH=http://purl.org/dc/terms/isVersionOf +LDES_STREAMS_CROWDSCAN_PROPERTYPREDICATES_COUNT=https://crowdscan.be/ns/value GRAPHDB_URL=http://rdf4j-server:8080/rdf4j-server/repositories/ GRAPHDB_REPOSITORYID=test SERVER_PORT=8080 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 32c2b12..222dde7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,7 +1,7 @@ services: data-provider: - image: ghcr.io/informatievlaanderen/ldi-orchestrator:20230927142432 + image: ghcr.io/informatievlaanderen/ldi-orchestrator:latest container_name: demonstrator-data-provider ports: - 8082:8080 @@ -9,11 +9,14 @@ services: - ./gipod.config.yml:/ldio/application.yml:ro - ./rml:/ldio/rml:ro - ./sparql:/ldio/sparql:ro + - ./jsonld:/ldio/jsonld:ro depends_on: rdf4j-server: condition: service_healthy demonstrator: condition: service_started + crowdscan-location-provider: + condition: service_started networks: - demonstrator @@ -64,6 +67,25 @@ services: networks: - demonstrator + crowdscan-location-provider: + image: ghcr.io/informatievlaanderen/ldi-orchestrator:20230927142432 + container_name: demonstrator-crowdscan-location-provider + ports: + - 8085:8080 + volumes: + - ./crowdscan-locations.config.yml:/ldio/application.yml:ro + - ./rml:/ldio/rml:ro + - ./sparql:/ldio/sparql:ro + - ./jsonld:/ldio/jsonld:ro + depends_on: + rdf4j-server: + condition: service_healthy + demonstrator: + condition: service_started + networks: + - demonstrator + + networks: demonstrator: driver: bridge diff --git a/frontend/streams.json b/frontend/streams.json index 3b8c65e..0d326b0 100644 --- a/frontend/streams.json +++ b/frontend/streams.json @@ -14,6 +14,11 @@ "id": "bluebikes", "fullName": "Blue Bikes", "color": "#05c" + }, + { + "id": "crowdscan", + "fullName": "CrowdScan", + "color": "#A4E2EB" } ] } \ No newline at end of file diff --git a/gipod.config.yml b/gipod.config.yml index d3a6b10..f74f725 100644 --- a/gipod.config.yml +++ b/gipod.config.yml @@ -2,72 +2,131 @@ server: port: 8082 orchestrator: pipelines: - - name: gipod - input: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient - config: - url: https://private-api.gipod.vlaanderen.be/api/v1/ldes/mobility-hindrances?generatedAtTime=2024-10-05T10:28:37.923Z - sourceFormat: application/ld+json - outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser - config: - sparql-host: http://rdf4j-server:8080/rdf4j-server - repository-id: test - named-graph: http://gipod - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut - config: - content-type: application/n-quads - endpoint: http://host.docker.internal:8084/api/gipod/members - - name: verkeersmetingen-pipeline +# - name: gipod +# input: +# name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient +# config: +# url: https://private-api.gipod.vlaanderen.be/api/v1/ldes/mobility-hindrances?generatedAtTime=2024-10-05T10:28:37.923Z +# sourceFormat: application/ld+json +# outputs: +# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser +# config: +# sparql-host: http://rdf4j-server:8080/rdf4j-server +# repository-id: test +# named-graph: http://gipod +# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut +# config: +# content-type: application/n-quads +# endpoint: http://host.docker.internal:8084/api/gipod/members +# - name: verkeersmetingen-pipeline +# description: "Simple http in, version creation, http out pipeline allowing to pause output." +# input: +# name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller +# config: +# url: http://miv.opendata.belfla.be/miv/verkeersdata +# interval: PT10S +# adapter: +# name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter +# config: +# mapping: /ldio/rml/verkeersmetingen.ttl +# transformers: +# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer +# config: +# query: /ldio/sparql/meting.to-oslo.rq +# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator +# config: +# member-type: https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting +# delimiter: "/" +# date-observed-property: "//" +# generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime +# versionOf-property: http://purl.org/dc/terms/isVersionOf +# outputs: +# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser +# config: +# sparql-host: http://rdf4j-server:8080/rdf4j-server +# repository-id: test +# named-graph: http://verkeersmeting +# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut +# config: +# endpoint: http://host.docker.internal:8084/api/verkeersmeting/members +# content-type: application/n-quads +# - name: stations-pipeline +# description: "HTTP polling, OSLO transformation, version creation & HTTP sending." +# input: +# name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller +# config: +# url: https://api.blue-bike.be/pub/location +# interval: PT2M +# adapter: +# name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter +# config: +# mapping: /ldio/rml/stations.ttl +# outputs: +# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser +# config: +# sparql-host: http://rdf4j-server:8080/rdf4j-server +# repository-id: test +# named-graph: http://bluebikes +# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut +# config: +# endpoint: http://host.docker.internal:8084/api/bluebikes/members +# content-type: application/n-quads + - name: observations-pipeline description: "Simple http in, version creation, http out pipeline allowing to pause output." input: name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller config: - url: http://miv.opendata.belfla.be/miv/verkeersdata - interval: PT10S + url: https://test.crowdscan.be/dataapi/antwerpen/beacon/count + interval: PT1M adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter config: - mapping: /ldio/rml/verkeersmetingen.ttl + core-context: file:///ldio/jsonld/count.jsonld transformers: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer config: - query: /ldio/sparql/meting.to-oslo.rq + query: ./sparql/count.to-observation.rq + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/observation.to-oslo.rq - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator config: - member-type: https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting + member-type: http://def.isotc211.org/iso19156/2011/Observation#OM_Observation delimiter: "/" - date-observed-property: "//" + date-observed-property: http://def.isotc211.org/iso19156/2011/Observation#OM_Observation.resultTime generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime versionOf-property: http://purl.org/dc/terms/isVersionOf - outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser - config: - sparql-host: http://rdf4j-server:8080/rdf4j-server - repository-id: test - named-graph: http://verkeersmeting - - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer config: - endpoint: http://host.docker.internal:8084/api/verkeersmeting/members - content-type: application/n-quads - - name: stations-pipeline - description: "HTTP polling, OSLO transformation, version creation & HTTP sending." - input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller - config: - url: https://api.blue-bike.be/pub/location - interval: PT2M - adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + query: ./sparql/crowdscan.add-query.rq + infer: true + - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpEnricher config: - mapping: /ldio/rml/stations.ttl + url-property-path: https://crowdscan.be/ns/HttpRequest.url + body-property-path: https://crowdscan.be/ns/HttpRequest.body + http-method-property-path: https://crowdscan.be/ns/HttpRequest.method + header-property-path: https://crowdscan.be/ns/HttpRequest.header +# auth: +# type: OAUTH2_CLIENT_CREDENTIALS +# token-endpoint: https://apieco.eco-counter-tools.com/token +# client-id: ${OAUTH2_CLIENT_ID} +# client-secret: ${OAUTH2_CLIENT_SECRET} +# rate-limit: +# enabled: true +# max-requests-per-minute: ${REQUESTS_PER_MINUTE} + adapter: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter outputs: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: test - named-graph: http://bluebikes + named-graph: http://crowdscan - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut config: - endpoint: http://host.docker.internal:8084/api/bluebikes/members - content-type: application/n-quads \ No newline at end of file + endpoint: http://host.docker.internal:8084/api/crowdscan/members + content-type: application/n-quads +# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut +# config: +# endpoint: https://webhook.site/51437bbb-a96c-4900-9535-836d6707aa83 +# content-type: text/turtle \ No newline at end of file diff --git a/jsonld/count.jsonld b/jsonld/count.jsonld new file mode 100644 index 0000000..02c9b56 --- /dev/null +++ b/jsonld/count.jsonld @@ -0,0 +1,14 @@ +{ + "@context": { + "@vocab": "https://www.crowdscan.be/ns/count#", + "time": { + "@type": "http://www.w3.org/2001/XMLSchema#DateTime" + }, + "timedelta": { + "@type": "http://www.w3.org/2001/XMLSchema#int" + }, + "regions": { + "@container": "@list" + } + } +} \ No newline at end of file diff --git a/jsonld/map.jsonld b/jsonld/map.jsonld new file mode 100644 index 0000000..c59f805 --- /dev/null +++ b/jsonld/map.jsonld @@ -0,0 +1,11 @@ +{ + "@context": [ + { + "@vocab": "https://www.crowdscan.be/ns/map#", + "last_updated": { + "@type": "http://www.w3.org/2001/XMLSchema#DateTime" + } + }, + "http://geojson.org/geojson-ld/geojson-context.jsonld" + ] +} \ No newline at end of file diff --git a/sparql/count.to-observation.rq b/sparql/count.to-observation.rq new file mode 100644 index 0000000..f2823f1 --- /dev/null +++ b/sparql/count.to-observation.rq @@ -0,0 +1,50 @@ +PREFIX rdf: +PREFIX xsd: +PREFIX schema: +PREFIX time: +PREFIX : + +CONSTRUCT { + + GRAPH ?observation { + ?observation a :Observation ; + :time ?time ; + :value ?value ; + :environment ?environment ; + :timedelta ?delta ; + :region ?zone . + } + +} WHERE { + + ?header :environment ?environment . + ?header :time ?time . + ?header :timedelta ?timedelta . + bind(strdt(concat("-PT", str(?timedelta), "M"), xsd:duration) as ?delta) . + + ?payload :payload ?regions . + { + SELECT (count(?first) as ?zoneCount) WHERE { ?list rdf:first ?first . } + } + + OPTIONAL { filter(?zoneCount = 1) + ?list rdf:first ?value . + bind(0 as ?zone) . + } + + OPTIONAL { filter(?zoneCount > 1) + { + SELECT ?value (count(?mid) as ?zone) + WHERE + { + ?regions rdf:rest* ?mid . + ?mid rdf:rest ?node . + ?node rdf:first ?value . + } + GROUP BY ?node ?value + } + } + + bind(uri(concat("https://crowdscan.be/id/observation/", ?environment, "/", str(?zone))) as ?observation) . + +} diff --git a/sparql/crowdscan.add-query.rq b/sparql/crowdscan.add-query.rq new file mode 100644 index 0000000..ab20ca9 --- /dev/null +++ b/sparql/crowdscan.add-query.rq @@ -0,0 +1,30 @@ +PREFIX rdf: +PREFIX xsd: + +PREFIX : +PREFIX http: +PREFIX obs: +PREFIX prov: + +CONSTRUCT { + ?request a :HttpRequest . + ?request http:url ?url . + ?request http:body ?body . + ?request http:method "POST" . + ?request http:header ?type . + ?request http:header ?accept . +} WHERE { + ?obs rdf:type obs:OM_Observation . + ?obs obs:OM_Observation.featureOfInterest ?loc . + ?obs prov:generatedAtTime ?obstime . + + + bind(concat(str(?loc), "/") as ?locpart) . + + # set HTTP request parameters + bind(bnode() as ?request) . + bind("http://rdf4j-server:8080/rdf4j-server/repositories/locations" as ?url) . + bind(concat("describe ?zone From Where { { select (max(?t) as ?max) Where { ?z ?time . Filter(?t < \"", str(?obstime), "\") . bind(str(?time) as ?t) . } } bind(IRI(concat(\"", ?locpart, "\", ?max)) as ?zone) . }") as ?body) . + bind("Content-Type: application/sparql-query" as ?type) . + bind("Accept: application/n-quads" as ?accept) . +} diff --git a/sparql/map.to-zone.rq b/sparql/map.to-zone.rq new file mode 100644 index 0000000..3ab834a --- /dev/null +++ b/sparql/map.to-zone.rq @@ -0,0 +1,53 @@ +PREFIX rdf: +PREFIX xsd: +PREFIX schema: +PREFIX time: +PREFIX geojson: +PREFIX locn: +PREFIX gsp: +PREFIX sf: +PREFIX dcterms: + +PREFIX : + +CONSTRUCT { + + GRAPH ?graph { + ?entity a ?type ; + dcterms:modified ?last_updated ; + :region ?region ; + :sn ?sn ; + :installationTime ?installation_time ; + :environment ?env ; + locn:geometry [a ?geometryType ; gsp:asWKT ?geometry ] . + } + +} WHERE { + + ?feature geojson:properties ?properties . + ?feature locn:geometry ?geometry . + + OPTIONAL { + ?properties rdf:type :gateway . + ?properties :last_updated ?last_updated . + ?properties :env ?env . + ?properties :installation_time ?installation_time . + ?properties :sn ?sn . + bind(sf:Polygon as ?geometryType) . + bind( as ?graph) . + bind(:Gateway as ?type) . + bind(bnode() as ?entity) . + } + + OPTIONAL { + ?properties rdf:type :zone . + ?properties :region ?region . + ?properties :last_updated ?last_updated . + bind(sf:Point as ?geometryType) . + bind(:Zone as ?type) . + ?sub :env ?environment . + bind(uri(concat("https://www.crowdscan.be/id/zone/", ?environment, "/", STR(?region))) as ?entity) . + bind(?entity as ?graph) . + } + +} diff --git a/sparql/observation.to-oslo.rq b/sparql/observation.to-oslo.rq new file mode 100644 index 0000000..a1a5476 --- /dev/null +++ b/sparql/observation.to-oslo.rq @@ -0,0 +1,26 @@ +PREFIX rdf: +PREFIX xsd: +PREFIX schema: +PREFIX time: +PREFIX iso19156-om: +PREFIX sosa: +PREFIX dv-seb: +PREFIX : + +CONSTRUCT { + ?observation a iso19156-om:OM_Observation ; + iso19156-om:OM_Observation.resultTime ?time ; + iso19156-om:OM_Observation.phenomenonTime [ a time:Interval; time:hasXSDDuration ?delta; time:hasEnd [a time:Instant; time:inXSDDateTimeStamp ?time ] ]; + iso19156-om:OM_Observation.result [ a schema:QuantitativeValue ; schema:value ?value ] ; + iso19156-om:OM_Observation.observedProperty :PeopleEstimate ; + iso19156-om:OM_Observation.featureOfInterest ?feature ; + iso19156-om:OM_Observation.procedure [ a sosa:Procedure ; dv-seb:Observatieprocedure.specificatie ] . +} WHERE { + ?observation rdf:type :Observation . + ?observation :time ?time . + ?observation :value ?value . + ?observation :timedelta ?delta . + ?observation :environment ?env . + ?observation :region ?zone . + bind(IRI(concat("https://www.crowdscan.be/id/zone/", ?env, "/", STR(?zone))) AS ?feature) . +} \ No newline at end of file diff --git a/sparql/zone.to-oslo.rq b/sparql/zone.to-oslo.rq new file mode 100644 index 0000000..4f2c9a5 --- /dev/null +++ b/sparql/zone.to-oslo.rq @@ -0,0 +1,60 @@ +PREFIX rdfs: +PREFIX schema: +PREFIX time: +PREFIX xsd: +PREFIX dcterms: +PREFIX sosa: +PREFIX ssn: +PREFIX gsp: +PREFIX sf: +PREFIX locn: +PREFIX iso19156-sf: +PREFIX iso19156-ss: +PREFIX iso19156-sp: +PREFIX iso19156-gfi: +PREFIX iso19156-dssf: + +PREFIX : + +CONSTRUCT { + + ?zone + a iso19156-ss:SF_SamplingSurface ; + dcterms:modified ?modified ; + iso19156-sf:SF_SamplingFeature.sampledFeature ?zoneFeature; + iso19156-ss:SF_SamplingSurface.shape [ + a gsp:Surface ; + gsp:asWKT ?zoneGeometry + ] ; + sosa:isResultOf ?zoneSampling . + + ?zoneFeature + a iso19156-gfi:GFI_DomainFeature, dcterms:Location ; + rdfs:label ?placeName . + + ?zoneSampling + a sosa:Sampling ; + sosa:madeBySampler [ + a sosa:Sampler ; + locn:location [ + a locn:Geometry ; + gsp:asWKT ?gatewayGeometry + ] + ] . + +} WHERE { + + ?zone a :Zone . + ?zone dcterms:modified ?modified . + ?zone :region ?region . + ?zone locn:geometry ?zoneShape . + ?zoneShape gsp:asWKT ?zoneGeometry . + bind(BNODE() as ?zoneSampling) . + bind(BNODE() as ?zoneFeature) . + ?gateway a :Gateway . + ?gateway :environment ?environment . + ?gateway locn:geometry ?gatewayLocation . + ?gatewayLocation gsp:asWKT ?gatewayGeometry . + bind(STRLANG(concat(?environment, " zone ", STR(?region)), "nl") AS ?placeName) . + +} \ No newline at end of file From a29bf09063c59ab4a7698628a71ca2a9305f5b60 Mon Sep 17 00:00:00 2001 From: pj-cegeka Date: Mon, 6 Nov 2023 09:28:39 +0100 Subject: [PATCH 2/3] feat: add crowdscan --- .../application/config/EventStreamConfig.java | 9 ++ .../services/PropertyExtractor.java | 12 ++ .../services/PropertyPathExtractor.java | 48 ++++++ .../valueobjects/IngestedMemberDto.java | 14 +- crowdscan-locations.config.yml | 11 +- demonstrator.env | 3 +- docker-compose.yml | 20 --- .../components/map/composables/usePopup.js | 4 + gipod.config.yml | 152 ++++++++++-------- 9 files changed, 174 insertions(+), 99 deletions(-) create mode 100644 backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java create mode 100644 backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java index 06590a1..523c234 100644 --- a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/config/EventStreamConfig.java @@ -13,6 +13,7 @@ public class EventStreamConfig { private String versionOfPath; private String timestampPath; private String timezoneId; + private String geoLocationPath; private Map propertyPredicates; public String getMemberType() { @@ -60,4 +61,12 @@ public String getVersionOfPath() { public void setVersionOfPath(String versionOfPath) { this.versionOfPath = versionOfPath; } + + public String getGeoLocationPath() { + return geoLocationPath; + } + + public void setGeoLocationPath(String geoLocationPath) { + this.geoLocationPath = geoLocationPath; + } } \ No newline at end of file diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java new file mode 100644 index 0000000..4ca583b --- /dev/null +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyExtractor.java @@ -0,0 +1,12 @@ +package be.informatievlaanderen.vsds.demonstrator.member.application.services; + +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; + +import java.util.List; + +public interface PropertyExtractor { + + List getProperties(Model model); + +} diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java new file mode 100644 index 0000000..e112947 --- /dev/null +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/services/PropertyPathExtractor.java @@ -0,0 +1,48 @@ +package be.informatievlaanderen.vsds.demonstrator.member.application.services; + +import org.apache.jena.query.*; +import org.apache.jena.rdf.model.Model; +import org.apache.jena.rdf.model.RDFNode; + +import java.util.ArrayList; +import java.util.List; + +public class PropertyPathExtractor implements PropertyExtractor { + + private static final String OBJECT_VAR_NAME = "object"; + private static final String IRI_OPENING_SYMBOL = "<"; + private final String queryString; + + private PropertyPathExtractor(String propertyPath) { + queryString = "SELECT * where { ?subject %s ?object }".formatted(propertyPath); + } + + /** + * This factory method was provided for backwards compatibility. + * In the past we supported properties to be provided as strings in a non IRI + * format. + * When a property is provided as a plain string, we wrap it to an IRI. + * NOTE: Does not work with property paths -> ex:foo/ex:bar will not get auto + * wrapping. + */ + public static PropertyPathExtractor from(String propertyPath) { + return propertyPath.startsWith(IRI_OPENING_SYMBOL) + ? new PropertyPathExtractor(propertyPath) + : new PropertyPathExtractor("<%s>".formatted(propertyPath)); + } + + @Override + public List getProperties(Model model) { + final Query query = QueryFactory.create(queryString); + try (QueryExecution queryExecution = QueryExecutionFactory.create(query, model)) { + ResultSet resultSet = queryExecution.execSelect(); + + List results = new ArrayList<>(); + while (resultSet.hasNext()) { + results.add(resultSet.next().get(OBJECT_VAR_NAME)); + } + return results; + } + } + +} diff --git a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java index 9412687..b5a2e5a 100644 --- a/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java +++ b/backend/src/main/java/be/informatievlaanderen/vsds/demonstrator/member/application/valueobjects/IngestedMemberDto.java @@ -2,6 +2,7 @@ import be.informatievlaanderen.vsds.demonstrator.member.application.config.EventStreamConfig; import be.informatievlaanderen.vsds.demonstrator.member.application.exceptions.NoGeometryProvidedException; +import be.informatievlaanderen.vsds.demonstrator.member.application.services.PropertyPathExtractor; import be.informatievlaanderen.vsds.demonstrator.member.domain.member.entities.Member; import org.apache.jena.geosparql.implementation.GeometryWrapper; import org.apache.jena.geosparql.implementation.vocabulary.SRS_URI; @@ -15,6 +16,7 @@ import java.time.ZoneId; import java.time.ZoneOffset; import java.time.ZonedDateTime; +import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -42,7 +44,7 @@ public Model getModel() { } public Member getMember(EventStreamConfig eventStreamConfig) throws FactoryException, TransformException { - Geometry geometry = getGeometry(); + Geometry geometry = getGeometry(eventStreamConfig); String memberId = getMemberId(eventStreamConfig); String isVersionOf = getIsVersionOf(eventStreamConfig); String timestampString = getTimestamp(eventStreamConfig); @@ -64,8 +66,14 @@ private LocalDateTime getTimestamp(ZoneId timezoneId, String timestampString) { return timestamp; } - private Geometry getGeometry() throws FactoryException, TransformException { - List wktNodes = model.listObjectsOfProperty(model.createProperty("http://www.opengis.net/ont/geosparql#asWKT")).toList(); + private Geometry getGeometry(EventStreamConfig config) throws FactoryException, TransformException { + List wktNodes = new ArrayList<>(); + if(config.getGeoLocationPath() != null) { + wktNodes.addAll(PropertyPathExtractor.from(config.getGeoLocationPath()).getProperties(model)); + } + else { + wktNodes.addAll(model.listObjectsOfProperty(model.createProperty("http://www.opengis.net/ont/geosparql#asWKT")).toList()); + } if (wktNodes.isEmpty()) { throw new NoGeometryProvidedException(); } else { diff --git a/crowdscan-locations.config.yml b/crowdscan-locations.config.yml index 49d73e2..93d97bb 100644 --- a/crowdscan-locations.config.yml +++ b/crowdscan-locations.config.yml @@ -2,7 +2,7 @@ server: port: 8085 orchestrator: pipelines: - - name: map-pipeline + - name: crowdscan-locations-pipeline description: "HTTP polling, OSLO transformation, version creation & HTTP sending." input: name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller @@ -28,16 +28,9 @@ orchestrator: date-observed-property: http://purl.org/dc/terms/modified generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime versionOf-property: http://purl.org/dc/terms/isVersionOf -# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionMaterialiser -# config: -# versionOf-property: http://purl.org/dc/terms/isVersionOf outputs: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser config: sparql-host: http://rdf4j-server:8080/rdf4j-server repository-id: locations - named-graph: http://crowdscan-locations -# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut -# config: -# endpoint: https://webhook.site/51437bbb-a96c-4900-9535-836d6707aa83 -# content-type: application/n-quads \ No newline at end of file + named-graph: http://crowdscan-locations \ No newline at end of file diff --git a/demonstrator.env b/demonstrator.env index 5af8dc9..41ce699 100644 --- a/demonstrator.env +++ b/demonstrator.env @@ -25,7 +25,8 @@ LDES_STREAMS_BLUEBIKES_PROPERTYPREDICATES_AVAILABLE=http://schema.mobivoc.org/#c LDES_STREAMS_CROWDSCAN_MEMBERTYPE=http://def.isotc211.org/iso19156/2011/Observation#OM_Observation LDES_STREAMS_CROWDSCAN_TIMESTAMPPATH=http://www.w3.org/ns/prov#generatedAtTime LDES_STREAMS_CROWDSCAN_VERSIONOFPATH=http://purl.org/dc/terms/isVersionOf -LDES_STREAMS_CROWDSCAN_PROPERTYPREDICATES_COUNT=https://crowdscan.be/ns/value +LDES_STREAMS_CROWDSCAN_GEOLOCATIONPATH=/ +LDES_STREAMS_CROWDSCAN_PROPERTYPREDICATES_DENSITY=http://schema.org/value GRAPHDB_URL=http://rdf4j-server:8080/rdf4j-server/repositories/ GRAPHDB_REPOSITORYID=test SERVER_PORT=8080 \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 222dde7..068a710 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,8 +15,6 @@ services: condition: service_healthy demonstrator: condition: service_started - crowdscan-location-provider: - condition: service_started networks: - demonstrator @@ -67,24 +65,6 @@ services: networks: - demonstrator - crowdscan-location-provider: - image: ghcr.io/informatievlaanderen/ldi-orchestrator:20230927142432 - container_name: demonstrator-crowdscan-location-provider - ports: - - 8085:8080 - volumes: - - ./crowdscan-locations.config.yml:/ldio/application.yml:ro - - ./rml:/ldio/rml:ro - - ./sparql:/ldio/sparql:ro - - ./jsonld:/ldio/jsonld:ro - depends_on: - rdf4j-server: - condition: service_healthy - demonstrator: - condition: service_started - networks: - - demonstrator - networks: demonstrator: diff --git a/frontend/src/components/map/composables/usePopup.js b/frontend/src/components/map/composables/usePopup.js index 1451816..d04628b 100644 --- a/frontend/src/components/map/composables/usePopup.js +++ b/frontend/src/components/map/composables/usePopup.js @@ -29,6 +29,10 @@ export function usePopup(collection, properties) { ${properties.fullname} ${properties.available}beschikbare ${getBikeString(properties.available)} +` + case "crowdscan": + return `
+personen dichtheid: ${properties.density}
` } diff --git a/gipod.config.yml b/gipod.config.yml index f74f725..556d996 100644 --- a/gipod.config.yml +++ b/gipod.config.yml @@ -18,60 +18,60 @@ orchestrator: # config: # content-type: application/n-quads # endpoint: http://host.docker.internal:8084/api/gipod/members -# - name: verkeersmetingen-pipeline -# description: "Simple http in, version creation, http out pipeline allowing to pause output." -# input: -# name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller -# config: -# url: http://miv.opendata.belfla.be/miv/verkeersdata -# interval: PT10S -# adapter: -# name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter -# config: -# mapping: /ldio/rml/verkeersmetingen.ttl -# transformers: -# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer -# config: -# query: /ldio/sparql/meting.to-oslo.rq -# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator -# config: -# member-type: https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting -# delimiter: "/" -# date-observed-property: "//" -# generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime -# versionOf-property: http://purl.org/dc/terms/isVersionOf -# outputs: -# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser -# config: -# sparql-host: http://rdf4j-server:8080/rdf4j-server -# repository-id: test -# named-graph: http://verkeersmeting -# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut -# config: -# endpoint: http://host.docker.internal:8084/api/verkeersmeting/members -# content-type: application/n-quads -# - name: stations-pipeline -# description: "HTTP polling, OSLO transformation, version creation & HTTP sending." -# input: -# name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller -# config: -# url: https://api.blue-bike.be/pub/location -# interval: PT2M -# adapter: -# name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter -# config: -# mapping: /ldio/rml/stations.ttl -# outputs: -# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser -# config: -# sparql-host: http://rdf4j-server:8080/rdf4j-server -# repository-id: test -# named-graph: http://bluebikes -# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut -# config: -# endpoint: http://host.docker.internal:8084/api/bluebikes/members -# content-type: application/n-quads - - name: observations-pipeline + - name: verkeersmetingen-pipeline + description: "Simple http in, version creation, http out pipeline allowing to pause output." + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + config: + url: http://miv.opendata.belfla.be/miv/verkeersdata + interval: PT10S + adapter: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + config: + mapping: /ldio/rml/verkeersmetingen.ttl + transformers: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: /ldio/sparql/meting.to-oslo.rq + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator + config: + member-type: https://data.vlaanderen.be/ns/verkeersmetingen#Verkeersmeting + delimiter: "/" + date-observed-property: "//" + generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime + versionOf-property: http://purl.org/dc/terms/isVersionOf + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: test + named-graph: http://verkeersmeting + - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + config: + endpoint: http://host.docker.internal:8084/api/verkeersmeting/members + content-type: application/n-quads + - name: stations-pipeline + description: "HTTP polling, OSLO transformation, version creation & HTTP sending." + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + config: + url: https://api.blue-bike.be/pub/location + interval: PT2M + adapter: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.RmlAdapter + config: + mapping: /ldio/rml/stations.ttl + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: test + named-graph: http://bluebikes + - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + config: + endpoint: http://host.docker.internal:8084/api/bluebikes/members + content-type: application/n-quads + - name: crowdscan-observations-pipeline description: "Simple http in, version creation, http out pipeline allowing to pause output." input: name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller @@ -106,14 +106,6 @@ orchestrator: body-property-path: https://crowdscan.be/ns/HttpRequest.body http-method-property-path: https://crowdscan.be/ns/HttpRequest.method header-property-path: https://crowdscan.be/ns/HttpRequest.header -# auth: -# type: OAUTH2_CLIENT_CREDENTIALS -# token-endpoint: https://apieco.eco-counter-tools.com/token -# client-id: ${OAUTH2_CLIENT_ID} -# client-secret: ${OAUTH2_CLIENT_SECRET} -# rate-limit: -# enabled: true -# max-requests-per-minute: ${REQUESTS_PER_MINUTE} adapter: name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter outputs: @@ -126,7 +118,35 @@ orchestrator: config: endpoint: http://host.docker.internal:8084/api/crowdscan/members content-type: application/n-quads -# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut -# config: -# endpoint: https://webhook.site/51437bbb-a96c-4900-9535-836d6707aa83 -# content-type: text/turtle \ No newline at end of file + - name: crowdscan-locations-pipeline + description: "HTTP polling, OSLO transformation, version creation & HTTP sending." + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + config: + url: https://test.crowdscan.be/dataapi/antwerpen/beacon/map + interval: P1D + adapter: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter + config: + core-context: file:///ldio/jsonld/map.jsonld + transformers: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/map.to-zone.rq + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/zone.to-oslo.rq + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator + config: + member-type: http://def.isotc211.org/iso19156/2011/SamplingSurface#SF_SamplingSurface + delimiter: "/" + date-observed-property: http://purl.org/dc/terms/modified + generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime + versionOf-property: http://purl.org/dc/terms/isVersionOf + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: locations + named-graph: http://crowdscan-locations \ No newline at end of file From a800588f18d74f5ef023c5abb276e38a4ebad1ab Mon Sep 17 00:00:00 2001 From: pj-cegeka Date: Mon, 6 Nov 2023 15:37:45 +0100 Subject: [PATCH 3/3] feat: remove query request from model --- crowdscan-locations.config.yml | 36 -------------- gipod.config.yml | 84 +++++++++----------------------- sparql/crowdscan.add-query.rq | 2 +- sparql/crowdscan.remove-query.rq | 15 ++++++ 4 files changed, 40 insertions(+), 97 deletions(-) delete mode 100644 crowdscan-locations.config.yml create mode 100644 sparql/crowdscan.remove-query.rq diff --git a/crowdscan-locations.config.yml b/crowdscan-locations.config.yml deleted file mode 100644 index 93d97bb..0000000 --- a/crowdscan-locations.config.yml +++ /dev/null @@ -1,36 +0,0 @@ -server: - port: 8085 -orchestrator: - pipelines: - - name: crowdscan-locations-pipeline - description: "HTTP polling, OSLO transformation, version creation & HTTP sending." - input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller - config: - url: https://test.crowdscan.be/dataapi/antwerpen/beacon/map - interval: P1D - adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter - config: - core-context: file:///ldio/jsonld/map.jsonld - transformers: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer - config: - query: ./sparql/map.to-zone.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer - config: - query: ./sparql/zone.to-oslo.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator - config: - member-type: http://def.isotc211.org/iso19156/2011/SamplingSurface#SF_SamplingSurface - delimiter: "/" - date-observed-property: http://purl.org/dc/terms/modified - generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime - versionOf-property: http://purl.org/dc/terms/isVersionOf - outputs: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser - config: - sparql-host: http://rdf4j-server:8080/rdf4j-server - repository-id: locations - named-graph: http://crowdscan-locations \ No newline at end of file diff --git a/gipod.config.yml b/gipod.config.yml index 556d996..f4cafef 100644 --- a/gipod.config.yml +++ b/gipod.config.yml @@ -2,22 +2,22 @@ server: port: 8082 orchestrator: pipelines: -# - name: gipod -# input: -# name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient -# config: -# url: https://private-api.gipod.vlaanderen.be/api/v1/ldes/mobility-hindrances?generatedAtTime=2024-10-05T10:28:37.923Z -# sourceFormat: application/ld+json -# outputs: -# - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser -# config: -# sparql-host: http://rdf4j-server:8080/rdf4j-server -# repository-id: test -# named-graph: http://gipod -# - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut -# config: -# content-type: application/n-quads -# endpoint: http://host.docker.internal:8084/api/gipod/members + - name: gipod + input: + name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient + config: + url: https://private-api.gipod.vlaanderen.be/api/v1/ldes/mobility-hindrances?generatedAtTime=2024-10-05T10:28:37.923Z + sourceFormat: application/ld+json + outputs: + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser + config: + sparql-host: http://rdf4j-server:8080/rdf4j-server + repository-id: test + named-graph: http://gipod + - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpOut + config: + content-type: application/n-quads + endpoint: http://host.docker.internal:8084/api/gipod/members - name: verkeersmetingen-pipeline description: "Simple http in, version creation, http out pipeline allowing to pause output." input: @@ -72,30 +72,11 @@ orchestrator: endpoint: http://host.docker.internal:8084/api/bluebikes/members content-type: application/n-quads - name: crowdscan-observations-pipeline - description: "Simple http in, version creation, http out pipeline allowing to pause output." input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient config: - url: https://test.crowdscan.be/dataapi/antwerpen/beacon/count - interval: PT1M - adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter - config: - core-context: file:///ldio/jsonld/count.jsonld + url: https://azure.crowdscan.be/ldes-scewc/observations transformers: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer - config: - query: ./sparql/count.to-observation.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer - config: - query: ./sparql/observation.to-oslo.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator - config: - member-type: http://def.isotc211.org/iso19156/2011/Observation#OM_Observation - delimiter: "/" - date-observed-property: http://def.isotc211.org/iso19156/2011/Observation#OM_Observation.resultTime - generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime - versionOf-property: http://purl.org/dc/terms/isVersionOf - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer config: query: ./sparql/crowdscan.add-query.rq @@ -108,6 +89,10 @@ orchestrator: header-property-path: https://crowdscan.be/ns/HttpRequest.header adapter: name: be.vlaanderen.informatievlaanderen.ldes.ldi.RdfAdapter + - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer + config: + query: ./sparql/crowdscan.remove-query.rq + infer: false outputs: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser config: @@ -119,31 +104,10 @@ orchestrator: endpoint: http://host.docker.internal:8084/api/crowdscan/members content-type: application/n-quads - name: crowdscan-locations-pipeline - description: "HTTP polling, OSLO transformation, version creation & HTTP sending." input: - name: be.vlaanderen.informatievlaanderen.ldes.ldio.LdioHttpInPoller + name: be.vlaanderen.informatievlaanderen.ldes.ldi.client.LdioLdesClient config: - url: https://test.crowdscan.be/dataapi/antwerpen/beacon/map - interval: P1D - adapter: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.JsonToLdAdapter - config: - core-context: file:///ldio/jsonld/map.jsonld - transformers: - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.GeoJsonToWktTransformer - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer - config: - query: ./sparql/map.to-zone.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.SparqlConstructTransformer - config: - query: ./sparql/zone.to-oslo.rq - - name: be.vlaanderen.informatievlaanderen.ldes.ldi.VersionObjectCreator - config: - member-type: http://def.isotc211.org/iso19156/2011/SamplingSurface#SF_SamplingSurface - delimiter: "/" - date-observed-property: http://purl.org/dc/terms/modified - generatedAt-property: http://www.w3.org/ns/prov#generatedAtTime - versionOf-property: http://purl.org/dc/terms/isVersionOf + url: https://azure.crowdscan.be/ldes-scewc/zones outputs: - name: be.vlaanderen.informatievlaanderen.ldes.ldi.RepositoryMaterialiser config: diff --git a/sparql/crowdscan.add-query.rq b/sparql/crowdscan.add-query.rq index ab20ca9..8847ca4 100644 --- a/sparql/crowdscan.add-query.rq +++ b/sparql/crowdscan.add-query.rq @@ -19,7 +19,7 @@ CONSTRUCT { ?obs prov:generatedAtTime ?obstime . - bind(concat(str(?loc), "/") as ?locpart) . + bind(concat(replace(replace(str(?loc), "www.crowdscan.be", "crowdscan.be"),"box4","beacon"), "/") as ?locpart) . # set HTTP request parameters bind(bnode() as ?request) . diff --git a/sparql/crowdscan.remove-query.rq b/sparql/crowdscan.remove-query.rq new file mode 100644 index 0000000..a574af3 --- /dev/null +++ b/sparql/crowdscan.remove-query.rq @@ -0,0 +1,15 @@ +PREFIX : + + +CONSTRUCT { + ?s ?p ?o . +} +WHERE { + ?s ?p ?o . + + {SELECT ?request + WHERE { + ?request a :HttpRequest . + }} + FILTER(?s != ?request) +} \ No newline at end of file